diff --git a/Cargo.lock b/Cargo.lock index af32d89981dea..2ebacff323f5f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9694,9 +9694,9 @@ checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" [[package]] name = "wabt" -version = "0.9.2" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5c5c1286c6e578416982609f47594265f9d489f9b836157d403ad605a46693" +checksum = "00bef93d5e6c81a293bccf107cf43aa47239382f455ba14869d36695d8963b9c" dependencies = [ "serde", "serde_derive", @@ -9706,9 +9706,9 @@ dependencies = [ [[package]] name = "wabt-sys" -version = "0.7.2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01c695f98f7eb81fd4e2f6b65301ccc916a950dc2265eeefc4d376b34ce666df" +checksum = "1a4e043159f63e16986e713e9b5e1c06043df4848565bf672e27c523864c7791" dependencies = [ "cc", "cmake", diff --git a/bin/node/executor/Cargo.toml b/bin/node/executor/Cargo.toml index d8fb2e4078bd1..485671c943137 100644 --- a/bin/node/executor/Cargo.toml +++ b/bin/node/executor/Cargo.toml @@ -41,7 +41,7 @@ sp-application-crypto = { version = "2.0.0-rc6", path = "../../../primitives/app sp-runtime = { version = "2.0.0-rc6", path = "../../../primitives/runtime" } sp-externalities = { version = "0.8.0-rc6", path = "../../../primitives/externalities" } substrate-test-client = { version = "2.0.0-rc6", path = "../../../test-utils/client" } -wabt = "0.9.2" +wabt = "0.10.0" [features] wasmtime = [ diff --git a/bin/node/testing/Cargo.toml b/bin/node/testing/Cargo.toml index 23bf10336dcfe..7ae8f27dfd491 100644 --- a/bin/node/testing/Cargo.toml +++ b/bin/node/testing/Cargo.toml @@ -39,7 +39,7 @@ substrate-test-client = { version = "2.0.0-rc6", path = "../../../test-utils/cli pallet-timestamp = { version = "2.0.0-rc6", path = "../../../frame/timestamp" } pallet-transaction-payment = { version = "2.0.0-rc6", path = "../../../frame/transaction-payment" } pallet-treasury = { version = "2.0.0-rc6", path = "../../../frame/treasury" } -wabt = "0.9.2" +wabt = "0.10.0" sp-api = { version = "2.0.0-rc6", path = "../../../primitives/api" } sp-finality-tracker = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/finality-tracker" } sp-timestamp = { version = "2.0.0-rc6", default-features = false, path = "../../../primitives/timestamp" } diff --git a/client/consensus/babe/src/authorship.rs b/client/consensus/babe/src/authorship.rs index 682e04e380d7c..f6e2a9c967b5b 100644 --- a/client/consensus/babe/src/authorship.rs +++ b/client/consensus/babe/src/authorship.rs @@ -295,10 +295,10 @@ mod tests { #[test] fn claim_secondary_plain_slot_works() { let keystore = sc_keystore::Store::new_in_memory(); - let valid_public_key = dbg!(keystore.write().sr25519_generate_new( + let valid_public_key = keystore.write().sr25519_generate_new( AuthorityId::ID, Some(sp_core::crypto::DEV_PHRASE), - ).unwrap()); + ).unwrap(); let authorities = vec![ (AuthorityId::from(Pair::generate().0.public()), 5), diff --git a/client/executor/Cargo.toml b/client/executor/Cargo.toml index f963068ea37b2..79f78d229a67f 100644 --- a/client/executor/Cargo.toml +++ b/client/executor/Cargo.toml @@ -37,7 +37,7 @@ libsecp256k1 = "0.3.4" [dev-dependencies] assert_matches = "1.3.0" -wabt = "0.9.2" +wabt = "0.10.0" hex-literal = "0.3.1" sc-runtime-test = { version = "2.0.0-rc6", path = "runtime-test" } substrate-test-runtime = { version = "2.0.0-rc6", path = "../../test-utils/runtime" } diff --git a/frame/elections-phragmen/src/lib.rs b/frame/elections-phragmen/src/lib.rs index 50c5de9bc0de4..e53b0063ddac8 100644 --- a/frame/elections-phragmen/src/lib.rs +++ b/frame/elections-phragmen/src/lib.rs @@ -100,7 +100,7 @@ use frame_support::{ ContainsLengthBound, } }; -use sp_npos_elections::{build_support_map, ExtendedBalance, VoteWeight, ElectionResult}; +use sp_npos_elections::{ExtendedBalance, VoteWeight, ElectionResult}; use frame_system::{ensure_signed, ensure_root}; mod benchmarking; @@ -226,7 +226,7 @@ decl_storage! { // ---- State /// The current elected membership. Sorted based on account id. pub Members get(fn members): Vec<(T::AccountId, BalanceOf)>; - /// The current runners_up. Sorted based on low to high merit (worse to best runner). + /// The current runners_up. Sorted based on low to high merit (worse to best). pub RunnersUp get(fn runners_up): Vec<(T::AccountId, BalanceOf)>; /// The total number of vote rounds that have happened, excluding the upcoming one. pub ElectionRounds get(fn election_rounds): u32 = Zero::zero(); @@ -717,6 +717,9 @@ decl_event!( /// No (or not enough) candidates existed for this round. This is different from /// `NewTerm([])`. See the description of `NewTerm`. EmptyTerm, + /// Internal error happened while trying to perform election. + ElectionError, + /// A member has been removed. This should always be followed by either `NewTerm` ot /// A [member] has been removed. This should always be followed by either `NewTerm` ot /// `EmptyTerm`. MemberKicked(AccountId), @@ -853,11 +856,6 @@ impl Module { } } - /// The locked stake of a voter. - fn locked_stake_of(who: &T::AccountId) -> BalanceOf { - Voting::::get(who).0 - } - /// Check there's nothing to do this block. /// /// Runs phragmen election and cleans all the previous candidate state. The voter state is NOT @@ -872,7 +870,8 @@ impl Module { 0 } - /// Run the phragmen election with all required side processes and state updates. + /// Run the phragmen election with all required side processes and state updates, if election + /// succeeds. Else, it will emit an `ElectionError` event. /// /// Calls the appropriate [`ChangeMembers`] function variant internally. /// @@ -893,6 +892,11 @@ impl Module { // previous runners_up are also always candidates for the next round. candidates.append(&mut Self::runners_up_ids()); + if candidates.len().is_zero() { + Self::deposit_event(RawEvent::EmptyTerm); + return; + } + // helper closures to deal with balance/stake. let to_votes = |b: BalanceOf| -> VoteWeight { , VoteWeight>>::convert(b) @@ -900,21 +904,17 @@ impl Module { let to_balance = |e: ExtendedBalance| -> BalanceOf { >>::convert(e) }; - let stake_of = |who: &T::AccountId| -> VoteWeight { - to_votes(Self::locked_stake_of(who)) - }; let voters_and_votes = Voting::::iter() .map(|(voter, (stake, targets))| { (voter, to_votes(stake), targets) }) .collect::>(); - let maybe_phragmen_result = sp_npos_elections::seq_phragmen::( + + let _ = sp_npos_elections::seq_phragmen::( num_to_elect, - 0, candidates, voters_and_votes.clone(), - ); - - if let Some(ElectionResult { winners, assignments }) = maybe_phragmen_result { + None, + ).map(|ElectionResult { winners, assignments: _ }| { let old_members_ids = >::take().into_iter() .map(|(m, _)| m) .collect::>(); @@ -922,41 +922,17 @@ impl Module { .map(|(r, _)| r) .collect::>(); - // filter out those who had literally no votes at all. - // NOTE: the need to do this is because all candidates, even those who have no - // vote are still considered by phragmen and when good candidates are scarce, then these - // cheap ones might get elected. We might actually want to remove the filter and allow - // zero-voted candidates to also make it to the membership set. - let new_set_with_approval = winners; - let new_set = new_set_with_approval + // filter out those who end up with no backing stake. + let new_set_with_stake = winners .into_iter() - .filter_map(|(m, a)| if a.is_zero() { None } else { Some(m) } ) - .collect::>(); + .filter_map(|(m, b)| if b.is_zero() { None } else { Some((m, to_balance(b))) }) + .collect::)>>(); // OPTIMISATION NOTE: we could bail out here if `new_set.len() == 0`. There isn't much // left to do. Yet, re-arranging the code would require duplicating the slashing of // exposed candidates, cleaning any previous members, and so on. For now, in favour of // readability and veracity, we keep it simple. - let staked_assignments = sp_npos_elections::assignment_ratio_to_staked( - assignments, - stake_of, - ); - - let (support_map, _) = build_support_map::(&new_set, &staked_assignments); - - let new_set_with_stake = new_set - .into_iter() - .map(|ref m| { - let support = support_map.get(m) - .expect( - "entire new_set was given to build_support_map; en entry must be \ - created for each item; qed" - ); - (m.clone(), to_balance(support.total)) - }) - .collect::)>>(); - // split new set into winners and runners up. let split_point = desired_seats.min(new_set_with_stake.len()); let mut new_members = (&new_set_with_stake[..split_point]).to_vec(); @@ -1042,14 +1018,14 @@ impl Module { >::put(new_runners_up); Self::deposit_event(RawEvent::NewTerm(new_members.clone().to_vec())); - } else { - Self::deposit_event(RawEvent::EmptyTerm); - } - // clean candidates. - >::kill(); + // clean candidates. + >::kill(); - ElectionRounds::mutate(|v| *v += 1); + ElectionRounds::mutate(|v| *v += 1); + }).map_err(|_| { + Self::deposit_event(RawEvent::ElectionError); + }); } } @@ -1376,6 +1352,10 @@ mod tests { assert_eq!(Elections::candidates(), candidates); } + fn locked_stake_of(who: &u64) -> u64 { + Voting::::get(who).0 + } + fn ensure_members_has_approval_stake() { // we filter members that have no approval state. This means that even we have more seats // than candidates, we will never ever chose a member with no votes. @@ -1694,13 +1674,13 @@ mod tests { assert_eq!(balances(&2), (18, 2)); assert_eq!(has_lock(&2), 20); - assert_eq!(Elections::locked_stake_of(&2), 20); + assert_eq!(locked_stake_of(&2), 20); // can update; different stake; different lock and reserve. assert_ok!(vote(Origin::signed(2), vec![5, 4], 15)); assert_eq!(balances(&2), (18, 2)); assert_eq!(has_lock(&2), 15); - assert_eq!(Elections::locked_stake_of(&2), 15); + assert_eq!(locked_stake_of(&2), 15); }); } @@ -1838,7 +1818,7 @@ mod tests { assert_ok!(vote(Origin::signed(2), vec![4, 5], 30)); // you can lie but won't get away with it. - assert_eq!(Elections::locked_stake_of(&2), 20); + assert_eq!(locked_stake_of(&2), 20); assert_eq!(has_lock(&2), 20); }); } @@ -1852,8 +1832,8 @@ mod tests { assert_ok!(vote(Origin::signed(3), vec![5], 30)); assert_eq_uvec!(all_voters(), vec![2, 3]); - assert_eq!(Elections::locked_stake_of(&2), 20); - assert_eq!(Elections::locked_stake_of(&3), 30); + assert_eq!(locked_stake_of(&2), 20); + assert_eq!(locked_stake_of(&3), 30); assert_eq!(votes_of(&2), vec![5]); assert_eq!(votes_of(&3), vec![5]); @@ -1861,7 +1841,7 @@ mod tests { assert_eq_uvec!(all_voters(), vec![3]); assert_eq!(votes_of(&2), vec![]); - assert_eq!(Elections::locked_stake_of(&2), 0); + assert_eq!(locked_stake_of(&2), 0); assert_eq!(balances(&2), (20, 0)); assert_eq!(Balances::locks(&2).len(), 0); @@ -2106,6 +2086,57 @@ mod tests { }); } + #[test] + fn empty_term() { + ExtBuilder::default().build_and_execute(|| { + // no candidates, no nothing. + System::set_block_number(5); + Elections::end_block(System::block_number()); + + assert_eq!( + System::events().iter().last().unwrap().event, + Event::elections_phragmen(RawEvent::EmptyTerm), + ) + }) + } + + #[test] + fn all_outgoing() { + ExtBuilder::default().build_and_execute(|| { + assert_ok!(submit_candidacy(Origin::signed(5))); + assert_ok!(submit_candidacy(Origin::signed(4))); + + assert_ok!(vote(Origin::signed(5), vec![5], 50)); + assert_ok!(vote(Origin::signed(4), vec![4], 40)); + + System::set_block_number(5); + Elections::end_block(System::block_number()); + + assert_eq!( + System::events().iter().last().unwrap().event, + Event::elections_phragmen(RawEvent::NewTerm(vec![(4, 40), (5, 50)])), + ); + + assert_eq!(Elections::members(), vec![(4, 40), (5, 50)]); + assert_eq!(Elections::runners_up(), vec![]); + + assert_ok!(Elections::remove_voter(Origin::signed(5))); + assert_ok!(Elections::remove_voter(Origin::signed(4))); + + System::set_block_number(10); + Elections::end_block(System::block_number()); + + assert_eq!( + System::events().iter().last().unwrap().event, + Event::elections_phragmen(RawEvent::NewTerm(vec![])), + ); + + // outgoing have lost their bond. + assert_eq!(balances(&4), (37, 0)); + assert_eq!(balances(&5), (47, 0)); + }); + } + #[test] fn defunct_voter_will_be_counted() { ExtBuilder::default().build_and_execute(|| { @@ -2680,29 +2711,29 @@ mod tests { }) } - // #[test] - // fn runner_up_replacement_works_when_out_of_order() { - // ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { - // assert_ok!(submit_candidacy(Origin::signed(5))); - // assert_ok!(submit_candidacy(Origin::signed(4))); - // assert_ok!(submit_candidacy(Origin::signed(3))); - // assert_ok!(submit_candidacy(Origin::signed(2))); - - // assert_ok!(vote(Origin::signed(2), vec![5], 20)); - // assert_ok!(vote(Origin::signed(3), vec![3], 30)); - // assert_ok!(vote(Origin::signed(4), vec![4], 40)); - // assert_ok!(vote(Origin::signed(5), vec![2], 50)); - - // System::set_block_number(5); - // Elections::end_block(System::block_number()); - - // assert_eq!(Elections::members_ids(), vec![2, 4]); - // assert_eq!(ELections::runners_up_ids(), vec![3, 5]); - // assert_ok!(Elections::renounce_candidacy(Origin::signed(3), Renouncing::RunnerUp)); - // assert_eq!(Elections::members_ids(), vec![2, 4]); - // assert_eq!(ELections::runners_up_ids(), vec![5]); - // }); - // } + #[test] + fn runner_up_replacement_works_when_out_of_order() { + ExtBuilder::default().desired_runners_up(2).build_and_execute(|| { + assert_ok!(submit_candidacy(Origin::signed(5))); + assert_ok!(submit_candidacy(Origin::signed(4))); + assert_ok!(submit_candidacy(Origin::signed(3))); + assert_ok!(submit_candidacy(Origin::signed(2))); + + assert_ok!(vote(Origin::signed(2), vec![5], 20)); + assert_ok!(vote(Origin::signed(3), vec![3], 30)); + assert_ok!(vote(Origin::signed(4), vec![4], 40)); + assert_ok!(vote(Origin::signed(5), vec![2], 50)); + + System::set_block_number(5); + Elections::end_block(System::block_number()); + + assert_eq!(Elections::members_ids(), vec![2, 4]); + assert_eq!(Elections::runners_up_ids(), vec![5, 3]); + assert_ok!(Elections::renounce_candidacy(Origin::signed(3), Renouncing::RunnerUp)); + assert_eq!(Elections::members_ids(), vec![2, 4]); + assert_eq!(Elections::runners_up_ids(), vec![5]); + }); + } #[test] fn can_renounce_candidacy_candidate() { diff --git a/frame/staking/src/lib.rs b/frame/staking/src/lib.rs index 8f5b8561eb420..c7d2ef325b57a 100644 --- a/frame/staking/src/lib.rs +++ b/frame/staking/src/lib.rs @@ -281,11 +281,7 @@ pub mod offchain_election; pub mod inflation; use sp_std::{ - result, - prelude::*, - collections::btree_map::BTreeMap, - convert::{TryInto, From}, - mem::size_of, + result, prelude::*, collections::btree_map::BTreeMap, convert::{TryInto, From}, mem::size_of, }; use codec::{HasCompact, Encode, Decode}; use frame_support::{ @@ -303,7 +299,7 @@ use frame_support::{ }; use pallet_session::historical; use sp_runtime::{ - Percent, Perbill, PerU16, PerThing, RuntimeDebug, DispatchError, + Percent, Perbill, PerU16, PerThing, InnerOf, RuntimeDebug, DispatchError, curve::PiecewiseLinear, traits::{ Convert, Zero, StaticLookup, CheckedSub, Saturating, SaturatedConversion, @@ -702,18 +698,18 @@ pub struct ElectionSize { impl ElectionStatus { - fn is_open_at(&self, n: BlockNumber) -> bool { + pub fn is_open_at(&self, n: BlockNumber) -> bool { *self == Self::Open(n) } - fn is_closed(&self) -> bool { + pub fn is_closed(&self) -> bool { match self { Self::Closed => true, _ => false } } - fn is_open(&self) -> bool { + pub fn is_open(&self) -> bool { !self.is_closed() } } @@ -1468,6 +1464,22 @@ decl_module! { T::BondingDuration::get(), ) ); + + use sp_runtime::UpperOf; + // see the documentation of `Assignment::try_normalize`. Now we can ensure that this + // will always return `Ok`. + assert!( + >>::try_into(MAX_NOMINATIONS) + .unwrap() + .checked_mul(::one().deconstruct().try_into().unwrap()) + .is_some() + ); + assert!( + >>::try_into(MAX_NOMINATIONS) + .unwrap() + .checked_mul(::one().deconstruct().try_into().unwrap()) + .is_some() + ); } /// Take the origin account as a stash and lock up `value` of its balance. `controller` will @@ -2300,7 +2312,7 @@ impl Module { } /// internal impl of [`slashable_balance_of`] that returns [`VoteWeight`]. - fn slashable_balance_of_vote_weight(stash: &T::AccountId) -> VoteWeight { + pub fn slashable_balance_of_vote_weight(stash: &T::AccountId) -> VoteWeight { , VoteWeight>>::convert( Self::slashable_balance_of(stash) ) @@ -2576,6 +2588,7 @@ impl Module { ) -> DispatchResultWithPostInfo { // Do the basic checks. era, claimed score and window open. Self::pre_dispatch_checks(claimed_score, era)?; + // the weight that we will refund in case of a correct submission. We compute this now // because the data needed for it will be consumed further down. let adjusted_weight = weight::weight_for_correct_submit_solution::( @@ -2731,8 +2744,8 @@ impl Module { // write new results. >::put(ElectionResult { elected_stashes: winners, - compute, exposures, + compute, }); QueuedScore::put(submitted_score); @@ -2941,7 +2954,7 @@ impl Module { fn try_do_election() -> Option>> { // an election result from either a stored submission or locally executed one. let next_result = >::take().or_else(|| - Self::do_phragmen_with_post_processing::(ElectionCompute::OnChain) + Self::do_on_chain_phragmen() ); // either way, kill this. We remove it here to make sure it always has the exact same @@ -2958,13 +2971,8 @@ impl Module { /// `PrimitiveElectionResult` into `ElectionResult`. /// /// No storage item is updated. - fn do_phragmen_with_post_processing(compute: ElectionCompute) - -> Option>> - where - Accuracy: sp_std::ops::Mul, - ExtendedBalance: From<::Inner>, - { - if let Some(phragmen_result) = Self::do_phragmen::() { + fn do_on_chain_phragmen() -> Option>> { + if let Some(phragmen_result) = Self::do_phragmen::(0) { let elected_stashes = phragmen_result.winners.iter() .map(|(s, _)| s.clone()) .collect::>(); @@ -2990,7 +2998,7 @@ impl Module { Some(ElectionResult::> { elected_stashes, exposures, - compute, + compute: ElectionCompute::OnChain, }) } else { // There were not enough candidates for even our minimal level of functionality. This is @@ -3004,10 +3012,14 @@ impl Module { /// Execute phragmen election and return the new results. No post-processing is applied and the /// raw edge weights are returned. /// - /// Self votes are added and nominations before the most recent slashing span are reaped. + /// Self votes are added and nominations before the most recent slashing span are ignored. /// /// No storage item is updated. - fn do_phragmen() -> Option> { + pub fn do_phragmen( + iterations: usize, + ) -> Option> + where ExtendedBalance: From> + { let mut all_nominators: Vec<(T::AccountId, VoteWeight, Vec)> = Vec::new(); let mut all_validators = Vec::new(); for (validator, _) in >::iter() { @@ -3036,16 +3048,25 @@ impl Module { (n, s, ns) })); - seq_phragmen::<_, Accuracy>( - Self::validator_count() as usize, - Self::minimum_validator_count().max(1) as usize, - all_validators, - all_nominators, - ) + if all_validators.len() < Self::minimum_validator_count().max(1) as usize { + // If we don't have enough candidates, nothing to do. + None + } else { + seq_phragmen::<_, Accuracy>( + Self::validator_count() as usize, + all_validators, + all_nominators, + Some((iterations, 0)), // exactly run `iterations` rounds. + ) + .map_err(|err| log!(error, "Call to seq-phragmen failed due to {}", err)) + .ok() + } } /// Consume a set of [`Supports`] from [`sp_npos_elections`] and collect them into a [`Exposure`] - fn collect_exposure(supports: SupportMap) -> Vec<(T::AccountId, Exposure>)> { + fn collect_exposure( + supports: SupportMap, + ) -> Vec<(T::AccountId, Exposure>)> { let to_balance = |e: ExtendedBalance| >>::convert(e); diff --git a/frame/staking/src/mock.rs b/frame/staking/src/mock.rs index dcdacfbaacb04..942469adbb7e2 100644 --- a/frame/staking/src/mock.rs +++ b/frame/staking/src/mock.rs @@ -908,7 +908,7 @@ pub(crate) fn prepare_submission_with( let sp_npos_elections::ElectionResult { winners, assignments, - } = Staking::do_phragmen::().unwrap(); + } = Staking::do_phragmen::(iterations).unwrap(); let winners = sp_npos_elections::to_without_backing(winners); let stake_of = |who: &AccountId| -> VoteWeight { @@ -918,16 +918,6 @@ pub(crate) fn prepare_submission_with( }; let mut staked = sp_npos_elections::assignment_ratio_to_staked(assignments, stake_of); - let (mut support_map, _) = build_support_map::(&winners, &staked); - - if iterations > 0 { - sp_npos_elections::balance_solution( - &mut staked, - &mut support_map, - Zero::zero(), - iterations, - ); - } // apply custom tweaks. awesome for testing. tweak(&mut staked); diff --git a/frame/staking/src/offchain_election.rs b/frame/staking/src/offchain_election.rs index 79f3a5c2d94fe..2778c8fd3165a 100644 --- a/frame/staking/src/offchain_election.rs +++ b/frame/staking/src/offchain_election.rs @@ -25,10 +25,10 @@ use crate::{ use frame_system::offchain::SubmitTransaction; use sp_npos_elections::{ build_support_map, evaluate_support, reduce, Assignment, ExtendedBalance, ElectionResult, - ElectionScore, balance_solution, + ElectionScore, }; use sp_runtime::offchain::storage::StorageValueRef; -use sp_runtime::{PerThing, RuntimeDebug, traits::{TrailingZeroInput, Zero}}; +use sp_runtime::{PerThing, RuntimeDebug, traits::TrailingZeroInput}; use frame_support::traits::Get; use sp_std::{convert::TryInto, prelude::*}; @@ -106,16 +106,24 @@ pub(crate) fn set_check_offchain_execution_status( /// compacts and reduces the solution, computes the score and submits it back to the chain as an /// unsigned transaction, without any signature. pub(crate) fn compute_offchain_election() -> Result<(), OffchainElectionError> { + let iters = get_balancing_iters::(); // compute raw solution. Note that we use `OffchainAccuracy`. let ElectionResult { winners, assignments, - } = >::do_phragmen::() + } = >::do_phragmen::(iters) .ok_or(OffchainElectionError::ElectionFailed)?; // process and prepare it for submission. let (winners, compact, score, size) = prepare_submission::(assignments, winners, true)?; + crate::log!( + info, + "prepared a seq-phragmen solution with {} balancing iterations and score {:?}", + iters, + score, + ); + // defensive-only: current era can never be none except genesis. let current_era = >::current_era().unwrap_or_default(); @@ -132,6 +140,20 @@ pub(crate) fn compute_offchain_election() -> Result<(), OffchainElecti .map_err(|_| OffchainElectionError::PoolSubmissionFailed) } +/// Get a random number of iterations to run the balancing. +/// +/// Uses the offchain seed to generate a random number. +pub fn get_balancing_iters() -> usize { + match T::MaxIterations::get() { + 0 => 0, + max @ _ => { + let seed = sp_io::offchain::random_seed(); + let random = ::decode(&mut TrailingZeroInput::new(seed.as_ref())) + .expect("input is padded with zeroes; qed") % max.saturating_add(1); + random as usize + } + } +} /// Takes an election result and spits out some data that can be submitted to the chain. /// @@ -177,26 +199,6 @@ pub fn prepare_submission( >::slashable_balance_of_vote_weight, ); - let (mut support_map, _) = build_support_map::(&winners, &staked); - // balance a random number of times. - let iterations_executed = match T::MaxIterations::get() { - 0 => { - // Don't run balance_solution at all - 0 - } - iterations @ _ => { - let seed = sp_io::offchain::random_seed(); - let iterations = ::decode(&mut TrailingZeroInput::new(seed.as_ref())) - .expect("input is padded with zeroes; qed") % iterations.saturating_add(1); - balance_solution( - &mut staked, - &mut support_map, - Zero::zero(), - iterations as usize, - ) - } - }; - // reduce if do_reduce { reduce(&mut staked); @@ -250,12 +252,5 @@ pub fn prepare_submission( nominators: snapshot_nominators.len() as NominatorIndex, }; - crate::log!( - info, - "prepared solution after {} equalization iterations with score {:?}", - iterations_executed, - score, - ); - Ok((winners_indexed, compact, score, size)) } diff --git a/frame/staking/src/testing_utils.rs b/frame/staking/src/testing_utils.rs index 02acd135e63e4..7f1707d50f85e 100644 --- a/frame/staking/src/testing_utils.rs +++ b/frame/staking/src/testing_utils.rs @@ -265,10 +265,12 @@ pub fn get_weak_solution( pub fn get_seq_phragmen_solution( do_reduce: bool, ) -> (Vec, CompactAssignments, ElectionScore, ElectionSize) { + let iters = offchain_election::get_balancing_iters::(); + let sp_npos_elections::ElectionResult { winners, assignments, - } = >::do_phragmen::().unwrap(); + } = >::do_phragmen::(iters).unwrap(); offchain_election::prepare_submission::(assignments, winners, do_reduce).unwrap() } diff --git a/frame/staking/src/tests.rs b/frame/staking/src/tests.rs index 278e532387672..e79d70d7762aa 100644 --- a/frame/staking/src/tests.rs +++ b/frame/staking/src/tests.rs @@ -1749,11 +1749,10 @@ fn bond_with_duplicate_vote_should_be_ignored_by_npos_election() { assert_ok!(Staking::nominate(Origin::signed(4), vec![21, 31])); // winners should be 21 and 31. Otherwise this election is taking duplicates into account. - let sp_npos_elections::ElectionResult { winners, assignments, - } = Staking::do_phragmen::().unwrap(); + } = Staking::do_phragmen::(0).unwrap(); let winners = sp_npos_elections::to_without_backing(winners); assert_eq!(winners, vec![31, 21]); @@ -1801,7 +1800,7 @@ fn bond_with_duplicate_vote_should_be_ignored_by_npos_election_elected() { let sp_npos_elections::ElectionResult { winners, assignments, - } = Staking::do_phragmen::().unwrap(); + } = Staking::do_phragmen::(0).unwrap(); let winners = sp_npos_elections::to_without_backing(winners); assert_eq!(winners, vec![21, 11]); diff --git a/primitives/arithmetic/src/biguint.rs b/primitives/arithmetic/src/biguint.rs index 41e2c759a5967..f501fd4e082cf 100644 --- a/primitives/arithmetic/src/biguint.rs +++ b/primitives/arithmetic/src/biguint.rs @@ -17,12 +17,13 @@ //! Infinite precision unsigned integer for substrate runtime. -use num_traits::Zero; +use num_traits::{Zero, One}; use sp_std::{cmp::Ordering, ops, prelude::*, cell::RefCell, convert::TryFrom}; // A sensible value for this would be half of the dword size of the host machine. Since the // runtime is compiled to 32bit webassembly, using 32 and 64 for single and double respectively // should yield the most performance. + /// Representation of a single limb. pub type Single = u32; /// Representation of two limbs. @@ -75,7 +76,7 @@ fn div_single(a: Double, b: Single) -> (Double, Single) { /// Simple wrapper around an infinitely large integer, represented as limbs of [`Single`]. #[derive(Clone, Default)] pub struct BigUint { - /// digits (limbs) of this number (sorted as msb -> lsd). + /// digits (limbs) of this number (sorted as msb -> lsb). pub(crate) digits: Vec, } @@ -515,6 +516,12 @@ impl Zero for BigUint { } } +impl One for BigUint { + fn one() -> Self { + Self { digits: vec![Single::one()] } + } +} + macro_rules! impl_try_from_number_for { ($([$type:ty, $len:expr]),+) => { $( @@ -550,15 +557,21 @@ macro_rules! impl_from_for_smaller_than_word { })* } } -impl_from_for_smaller_than_word!(u8, u16, Single); +impl_from_for_smaller_than_word!(u8, u16, u32); -impl From for BigUint { +impl From for BigUint { fn from(a: Double) -> Self { let (ah, al) = split(a); Self { digits: vec![ah, al] } } } +impl From for BigUint { + fn from(a: u128) -> Self { + crate::helpers_128bit::to_big_uint(a) + } +} + #[cfg(test)] pub mod tests { use super::*; diff --git a/primitives/arithmetic/src/lib.rs b/primitives/arithmetic/src/lib.rs index e54c6c833d141..ff9b5074e21d5 100644 --- a/primitives/arithmetic/src/lib.rs +++ b/primitives/arithmetic/src/lib.rs @@ -36,13 +36,13 @@ macro_rules! assert_eq_error_rate { pub mod biguint; pub mod helpers_128bit; pub mod traits; -mod per_things; -mod fixed_point; -mod rational128; +pub mod per_things; +pub mod fixed_point; +pub mod rational; pub use fixed_point::{FixedPointNumber, FixedPointOperand, FixedI64, FixedI128, FixedU128}; pub use per_things::{PerThing, InnerOf, UpperOf, Percent, PerU16, Permill, Perbill, Perquintill}; -pub use rational128::Rational128; +pub use rational::{Rational128, RationalInfinite}; use sp_std::{prelude::*, cmp::Ordering, fmt::Debug, convert::TryInto}; use traits::{BaseArithmetic, One, Zero, SaturatedConversion, Unsigned}; @@ -143,8 +143,8 @@ impl Normalizable

for Vec

{ /// `leftover` value. This ensures that the result will always stay accurate, yet it might cause the /// execution to become increasingly slow, since leftovers are applied one by one. /// -/// All in all, the complicated case above is rare to happen in all substrate use cases, hence we -/// opt for it due to its simplicity. +/// All in all, the complicated case above is rare to happen in most use cases within this repo , +/// hence we opt for it due to its simplicity. /// /// This function will return an error is if length of `input` cannot fit in `T`, or if `sum(input)` /// cannot fit inside `T`. diff --git a/primitives/arithmetic/src/rational128.rs b/primitives/arithmetic/src/rational.rs similarity index 82% rename from primitives/arithmetic/src/rational128.rs rename to primitives/arithmetic/src/rational.rs index 947c7bc537d19..07556bc0e2d71 100644 --- a/primitives/arithmetic/src/rational128.rs +++ b/primitives/arithmetic/src/rational.rs @@ -17,19 +17,106 @@ use sp_std::{cmp::Ordering, prelude::*}; use crate::helpers_128bit; -use num_traits::Zero; -use sp_debug_derive::RuntimeDebug; +use num_traits::{Zero, One, Bounded}; +use crate::biguint::BigUint; + +/// A wrapper for any rational number with infinitely large numerator and denominator. +/// +/// This type exists to facilitate `cmp` operation +/// on values like `a/b < c/d` where `a, b, c, d` are all `BigUint`. +#[derive(Clone, Default, Eq)] +pub struct RationalInfinite(BigUint, BigUint); + +impl RationalInfinite { + /// Return the numerator reference. + pub fn n(&self) -> &BigUint { + &self.0 + } + + /// Return the denominator reference. + pub fn d(&self) -> &BigUint { + &self.1 + } + + /// Build from a raw `n/d`. + pub fn from(n: BigUint, d: BigUint) -> Self { + Self(n, d.max(BigUint::one())) + } + + /// Zero. + pub fn zero() -> Self { + Self(BigUint::zero(), BigUint::one()) + } + + /// One. + pub fn one() -> Self { + Self(BigUint::one(), BigUint::one()) + } +} + +impl PartialOrd for RationalInfinite { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for RationalInfinite { + fn cmp(&self, other: &Self) -> Ordering { + // handle some edge cases. + if self.d() == other.d() { + self.n().cmp(&other.n()) + } else if self.d().is_zero() { + Ordering::Greater + } else if other.d().is_zero() { + Ordering::Less + } else { + // (a/b) cmp (c/d) => (a*d) cmp (c*b) + self.n().clone().mul(&other.d()).cmp(&other.n().clone().mul(&self.d())) + } + } +} + +impl PartialEq for RationalInfinite { + fn eq(&self, other: &Self) -> bool { + self.cmp(other) == Ordering::Equal + } +} + +impl From for RationalInfinite { + fn from(t: Rational128) -> Self { + Self(t.0.into(), t.1.into()) + } +} /// A wrapper for any rational number with a 128 bit numerator and denominator. -#[derive(Clone, Copy, Default, Eq, RuntimeDebug)] +#[derive(Clone, Copy, Default, Eq)] pub struct Rational128(u128, u128); +#[cfg(feature = "std")] +impl sp_std::fmt::Debug for Rational128 { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + write!(f, "Rational128({:.4})", self.0 as f32 / self.1 as f32) + } +} + +#[cfg(not(feature = "std"))] +impl sp_std::fmt::Debug for Rational128 { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + write!(f, "Rational128(..)") + } +} + impl Rational128 { - /// Nothing. + /// Zero. pub fn zero() -> Self { Self(0, 1) } + /// One + pub fn one() -> Self { + Self(1, 1) + } + /// If it is zero or not pub fn is_zero(&self) -> bool { self.0.is_zero() @@ -122,6 +209,22 @@ impl Rational128 { } } +impl Bounded for Rational128 { + fn min_value() -> Self { + Self(0, 1) + } + + fn max_value() -> Self { + Self(Bounded::max_value(), 1) + } +} + +impl> From for Rational128 { + fn from(t: T) -> Self { + Self::from(t.into(), 1) + } +} + impl PartialOrd for Rational128 { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) diff --git a/primitives/npos-elections/fuzzer/src/balance_solution.rs b/primitives/npos-elections/fuzzer/src/balance_solution.rs index 13f9b29706aed..8f0e02f23470b 100644 --- a/primitives/npos-elections/fuzzer/src/balance_solution.rs +++ b/primitives/npos-elections/fuzzer/src/balance_solution.rs @@ -21,10 +21,11 @@ //! one. mod common; + use common::to_range; use honggfuzz::fuzz; use sp_npos_elections::{ - balance_solution, assignment_ratio_to_staked, build_support_map, to_without_backing, seq_phragmen, + assignment_ratio_to_staked, build_support_map, to_without_backing, seq_phragmen, ElectionResult, VoteWeight, evaluate_support, is_score_better, }; use sp_std::collections::btree_map::BTreeMap; @@ -33,25 +34,30 @@ use rand::{self, Rng, SeedableRng, RngCore}; type AccountId = u64; -fn generate_random_phragmen_result( +fn generate_random_npos_result( voter_count: u64, target_count: u64, to_elect: usize, edge_per_voter: u64, mut rng: impl RngCore, -) -> (ElectionResult, BTreeMap) { +) -> ( + ElectionResult, + Vec, + Vec<(AccountId, VoteWeight, Vec)>, + BTreeMap, +) { let prefix = 100_000; - // Note, it is important that stakes are always bigger than ed and + // Note, it is important that stakes are always bigger than ed. let base_stake: u64 = 1_000_000_000; let ed: u64 = base_stake; let mut candidates = Vec::with_capacity(target_count as usize); - let mut stake_of_tree: BTreeMap = BTreeMap::new(); + let mut stake_of: BTreeMap = BTreeMap::new(); (1..=target_count).for_each(|acc| { candidates.push(acc); let stake_var = rng.gen_range(ed, 100 * ed); - stake_of_tree.insert(acc, base_stake + stake_var); + stake_of.insert(acc, base_stake + stake_var); }); let mut voters = Vec::with_capacity(voter_count as usize); @@ -68,18 +74,20 @@ fn generate_random_phragmen_result( let stake_var = rng.gen_range(ed, 100 * ed) ; let stake = base_stake + stake_var; - stake_of_tree.insert(acc, stake); + stake_of.insert(acc, stake); voters.push((acc, stake, targets)); }); ( seq_phragmen::( to_elect, - 0, - candidates, - voters, + candidates.clone(), + voters.clone(), + None, ).unwrap(), - stake_of_tree, + candidates, + voters, + stake_of, ) } @@ -97,12 +105,17 @@ fn main() { let rng = rand::rngs::SmallRng::seed_from_u64(seed); target_count = to_range(target_count, 50, 2000); voter_count = to_range(voter_count, 50, 1000); - iterations = to_range(iterations, 1, 20); + iterations = to_range(iterations, 1, 50); to_elect = to_range(to_elect, 25, target_count); edge_per_voter = to_range(edge_per_voter, 1, target_count); println!("++ [{} / {} / {} / {}]", voter_count, target_count, to_elect, iterations); - let (ElectionResult { winners, assignments }, stake_of_tree) = generate_random_phragmen_result( + let ( + unbalanced, + candidates, + voters, + stake_of_tree, + ) = generate_random_npos_result( voter_count as u64, target_count as u64, to_elect, @@ -114,42 +127,45 @@ fn main() { *stake_of_tree.get(who).unwrap() }; - let mut staked = assignment_ratio_to_staked(assignments, &stake_of); - let winners = to_without_backing(winners); - let mut support = build_support_map(winners.as_ref(), staked.as_ref()).0; - - let initial_score = evaluate_support(&support); - if initial_score[0] == 0 { - // such cases cannot be improved by reduce. - return; - } - - let i = balance_solution( - &mut staked, - &mut support, - 10, - iterations, - ); + let unbalanced_score = { + let staked = assignment_ratio_to_staked(unbalanced.assignments.clone(), &stake_of); + let winners = to_without_backing(unbalanced.winners); + let support = build_support_map(winners.as_ref(), staked.as_ref()).0; + + let score = evaluate_support(&support); + if score[0] == 0 { + // such cases cannot be improved by reduce. + return; + } + score + }; - let final_score = evaluate_support(&support); - if final_score[0] == initial_score[0] { - // such solutions can only be improved by such a tiny fiction that it is most often - // wrong due to rounding errors. - return; - } + let balanced = seq_phragmen::( + to_elect, + candidates, + voters, + Some((iterations, 0)), + ).unwrap(); + + let balanced_score = { + let staked = assignment_ratio_to_staked(balanced.assignments.clone(), &stake_of); + let winners = to_without_backing(balanced.winners); + let support = build_support_map(winners.as_ref(), staked.as_ref()).0; + evaluate_support(&support) + }; - let enhance = is_score_better(final_score, initial_score, Perbill::zero()); + let enhance = is_score_better(balanced_score, unbalanced_score, Perbill::zero()); println!( "iter = {} // {:?} -> {:?} [{}]", - i, - initial_score, - final_score, + iterations, + unbalanced_score, + balanced_score, enhance, ); // if more than one iteration has been done, or they must be equal. - assert!(enhance || initial_score == final_score || i == 0) + assert!(enhance || unbalanced_score == balanced_score || iterations == 0) }); } } diff --git a/primitives/npos-elections/src/balancing.rs b/primitives/npos-elections/src/balancing.rs new file mode 100644 index 0000000000000..4729acc0d573c --- /dev/null +++ b/primitives/npos-elections/src/balancing.rs @@ -0,0 +1,190 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Balancing algorithm implementation. +//! +//! Given a committee `A` and an edge weight vector `w`, a balanced solution is one that +//! +//! 1. it maximizes the sum of member supports, i.e `Argmax { sum(support(c)) }`. for all `c` in +//! `A`. +//! 2. it minimizes the sum of supports squared, i.e `Argmin { sum(support(c).pow(2)) }` for all `c` +//! in `A`. +//! +//! See [`balance`] for more information. + +use crate::{IdentifierT, Voter, ExtendedBalance, Edge}; +use sp_arithmetic::traits::Zero; +use sp_std::prelude::*; + +/// Balance the weight distribution of a given `voters` at most `iterations` times, or up until the +/// point where the biggest difference created per iteration of all stakes is `tolerance`. If this +/// is called with `tolerance = 0`, then exactly `iterations` rounds will be executed, except if no +/// change has been made (`difference = 0`). +/// +/// In almost all cases, a balanced solution will have a better score than an unbalanced solution, +/// yet this is not 100% guaranteed because the first element of a [`ElectionScore`] does not +/// directly related to balancing. +/// +/// Note that some reference implementation adopt an approach in which voters are balanced randomly +/// per round. To advocate determinism, we don't do this. In each round, all voters are exactly +/// balanced once, in the same order. +/// +/// Also, note that due to re-distribution of weights, the outcome of this function might contain +/// edges with weight zero. The call site should filter such weight if desirable. Moreover, the +/// outcome might need balance re-normalization, see `Voter::try_normalize`. +/// +/// ### References +/// +/// - [A new approach to the maximum flow problem](https://dl.acm.org/doi/10.1145/48014.61051). +/// - [Validator election in nominated proof-of-stake](https://arxiv.org/abs/2004.12990) (Appendix +/// A.) +pub fn balance( + voters: &mut Vec>, + iterations: usize, + tolerance: ExtendedBalance, +) -> usize { + if iterations == 0 { return 0; } + + let mut iter = 0; + loop { + let mut max_diff = 0; + for voter in voters.iter_mut() { + let diff = balance_voter(voter, tolerance); + if diff > max_diff { max_diff = diff; } + } + + iter += 1; + if max_diff <= tolerance || iter >= iterations { + break iter; + } + } +} + +/// Internal implementation of balancing for one voter. +pub(crate) fn balance_voter( + voter: &mut Voter, + tolerance: ExtendedBalance, +) -> ExtendedBalance { + // create a shallow copy of the elected ones. The original one will not be used henceforth. + let mut elected_edges = voter.edges + .iter_mut() + .filter(|e| e.candidate.borrow().elected) + .collect::>>(); + + // Either empty, or a self vote. Not much to do in either case. + if elected_edges.len() <= 1 { + return Zero::zero() + } + + // amount of stake from this voter that is used in edges. + let stake_used = elected_edges + .iter() + .fold(0, |a: ExtendedBalance, e| a.saturating_add(e.weight)); + + // backed stake of each of the elected edges. + let backed_stakes = elected_edges + .iter() + .map(|e| e.candidate.borrow().backed_stake) + .collect::>(); + + // backed stake of all the edges for whom we've spent some stake. + let backing_backed_stake = elected_edges + .iter() + .filter_map(|e| + if e.weight > 0 { + Some(e.candidate.borrow().backed_stake) + } else { + None + } + ) + .collect::>(); + + let difference = if backing_backed_stake.len() > 0 { + let max_stake = backing_backed_stake + .iter() + .max() + .expect("vector with positive length will have a max; qed"); + let min_stake = backed_stakes + .iter() + .min() + .expect("iterator with positive length will have a min; qed"); + let mut difference = max_stake.saturating_sub(*min_stake); + difference = difference.saturating_add(voter.budget.saturating_sub(stake_used)); + if difference < tolerance { + return difference; + } + difference + } else { + voter.budget + }; + + // remove all backings. + for edge in elected_edges.iter_mut() { + let mut candidate = edge.candidate.borrow_mut(); + candidate.backed_stake = candidate.backed_stake.saturating_sub(edge.weight); + edge.weight = 0; + } + + elected_edges.sort_by_key(|e| e.candidate.borrow().backed_stake); + + let mut cumulative_backed_stake = Zero::zero(); + let mut last_index = elected_edges.len() - 1; + + for (index, edge) in elected_edges.iter().enumerate() { + let index = index as ExtendedBalance; + let backed_stake = edge.candidate.borrow().backed_stake; + let temp = backed_stake.saturating_mul(index); + if temp.saturating_sub(cumulative_backed_stake) > voter.budget { + // defensive only. length of elected_edges is checked to be above 1. + last_index = index.saturating_sub(1) as usize; + break + } + cumulative_backed_stake = cumulative_backed_stake.saturating_add(backed_stake); + } + + let last_stake = elected_edges.get(last_index).expect( + "length of elected_edges is greater than or equal 2; last_index index is at \ + the minimum elected_edges.len() - 1; index is within range; qed" + ).candidate.borrow().backed_stake; + let ways_to_split = last_index + 1; + let excess = voter.budget + .saturating_add(cumulative_backed_stake) + .saturating_sub(last_stake.saturating_mul(ways_to_split as ExtendedBalance)); + + // Do the final update. + for edge in elected_edges.into_iter().take(ways_to_split) { + // first, do one scoped borrow to get the previous candidate stake. + let candidate_backed_stake = { + let candidate = edge.candidate.borrow(); + candidate.backed_stake + }; + + let new_edge_weight = (excess / ways_to_split as ExtendedBalance) + .saturating_add(last_stake) + .saturating_sub(candidate_backed_stake); + + // write the new edge weight + edge.weight = new_edge_weight; + + // write the new candidate stake + let mut candidate = edge.candidate.borrow_mut(); + candidate.backed_stake = candidate.backed_stake.saturating_add(new_edge_weight); + + } + + difference +} diff --git a/primitives/npos-elections/src/helpers.rs b/primitives/npos-elections/src/helpers.rs index 063eac70c57fd..cd8c199205cab 100644 --- a/primitives/npos-elections/src/helpers.rs +++ b/primitives/npos-elections/src/helpers.rs @@ -17,7 +17,9 @@ //! Helper methods for npos-elections. -use crate::{Assignment, ExtendedBalance, VoteWeight, IdentifierT, StakedAssignment, WithApprovalOf, Error}; +use crate::{ + Assignment, ExtendedBalance, VoteWeight, IdentifierT, StakedAssignment, WithApprovalOf, Error, +}; use sp_arithmetic::{PerThing, InnerOf}; use sp_std::prelude::*; diff --git a/primitives/npos-elections/src/lib.rs b/primitives/npos-elections/src/lib.rs index 58a69a116914f..4d04927450a63 100644 --- a/primitives/npos-elections/src/lib.rs +++ b/primitives/npos-elections/src/lib.rs @@ -1,58 +1,111 @@ // This file is part of Substrate. -// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. -// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2019-2020 Parity Technologies (UK) Ltd. SPDX-License-Identifier: Apache-2.0 -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +// in compliance with the License. You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under +// the License. //! A set of election algorithms to be used with a substrate runtime, typically within the staking -//! sub-system. Notable implementation include +//! sub-system. Notable implementation include: //! //! - [`seq_phragmen`]: Implements the Phragmén Sequential Method. An un-ranked, relatively fast //! election method that ensures PJR, but does not provide a constant factor approximation of the //! maximin problem. -//! - [`balance_solution`]: Implements the star balancing algorithm. This iterative process can -//! increase a solutions score, as described in [`evaluate_support`]. +//! - [`phragmms`](phragmms::phragmms): Implements a hybrid approach inspired by Phragmén which is executed faster but +//! it can achieve a constant factor approximation of the maximin problem, similar to that of the +//! MMS algorithm. +//! - [`balance`](balancing::balance): Implements the star balancing algorithm. This iterative process can push +//! a solution toward being more `balances`, which in turn can increase its score. +//! +//! ### Terminology +//! +//! This crate uses context-independent words, not to be confused with staking. This is because the +//! election algorithms of this crate, while designed for staking, can be used in other contexts as +//! well. +//! +//! `Voter`: The entity casting some votes to a number of `Targets`. This is the same as `Nominator` +//! in the context of staking. `Target`: The entities eligible to be voted upon. This is the same as +//! `Validator` in the context of staking. `Edge`: A mapping from a `Voter` to a `Target`. +//! +//! The goal of an election algorithm is to provide an `ElectionResult`. A data composed of: +//! - `winners`: A flat list of identifiers belonging to those who have won the election, usually +//! ordered in some meaningful way. They are zipped with their total backing stake. +//! - `assignment`: A mapping from each voter to their winner-only targets, zipped with a ration +//! denoting the amount of support given to that particular target. +//! +//! ```rust +//! # use sp_npos_elections::*; +//! # use sp_runtime::Perbill; +//! // the winners. +//! let winners = vec![(1, 100), (2, 50)]; +//! let assignments = vec![ +//! // A voter, giving equal backing to both 1 and 2. +//! Assignment { +//! who: 10, +//! distribution: vec![(1, Perbill::from_percent(50)), (2, Perbill::from_percent(50))], +//! }, +//! // A voter, Only backing 1. +//! Assignment { who: 20, distribution: vec![(1, Perbill::from_percent(100))] }, +//! ]; +//! +//! // the combination of the two makes the election result. +//! let election_result = ElectionResult { winners, assignments }; +//! +//! ``` +//! +//! The `Assignment` field of the election result is voter-major, i.e. it is from the perspective of +//! the voter. The struct that represents the opposite is called a `Support`. This struct is usually +//! accessed in a map-like manner, i.e. keyed vy voters, therefor it is stored as a mapping called +//! `SupportMap`. +//! +//! Moreover, the support is built from absolute backing values, not ratios like the example above. +//! A struct similar to `Assignment` that has stake value instead of ratios is called an +//! `StakedAssignment`. +//! //! //! More information can be found at: https://arxiv.org/abs/2004.12990 #![cfg_attr(not(feature = "std"), no_std)] -use sp_std::{prelude::*, collections::btree_map::BTreeMap, fmt::Debug, cmp::Ordering, convert::TryFrom}; +use sp_std::{ + prelude::*, collections::btree_map::BTreeMap, fmt::Debug, cmp::Ordering, rc::Rc, cell::RefCell, +}; use sp_arithmetic::{ PerThing, Rational128, ThresholdOrd, InnerOf, Normalizable, - helpers_128bit::multiply_by_rational, - traits::{Zero, Saturating, Bounded, SaturatedConversion}, + traits::{Zero, Saturating, Bounded}, }; -#[cfg(test)] -mod mock; -#[cfg(test)] -mod tests; #[cfg(feature = "std")] use serde::{Serialize, Deserialize}; #[cfg(feature = "std")] use codec::{Encode, Decode}; -mod node; -mod reduce; -mod helpers; +#[cfg(test)] +mod mock; +#[cfg(test)] +mod tests; + +pub mod phragmen; +pub mod balancing; +pub mod phragmms; +pub mod node; +pub mod reduce; +pub mod helpers; +pub mod pjr; -// re-export reduce stuff. pub use reduce::reduce; - -// re-export the helpers. pub use helpers::*; +pub use phragmen::*; +pub use phragmms::*; +pub use balancing::*; +pub use pjr::*; // re-export the compact macro, with the dependencies of the macro. #[doc(hidden)] @@ -91,8 +144,8 @@ impl IdentifierT for T {} /// The errors that might occur in the this crate and compact. #[derive(Debug, Eq, PartialEq)] pub enum Error { - /// While going from compact to staked, the stake of all the edges has gone above the - /// total and the last stake cannot be assigned. + /// While going from compact to staked, the stake of all the edges has gone above the total and + /// the last stake cannot be assigned. CompactStakeOverflow, /// The compact type has a voter who's number of targets is out of bound. CompactTargetOverflow, @@ -115,57 +168,149 @@ pub type ElectionScore = [ExtendedBalance; 3]; /// A winner, with their respective approval stake. pub type WithApprovalOf = (A, ExtendedBalance); -/// The denominator used for loads. Since votes are collected as u64, the smallest ratio that we -/// might collect is `1/approval_stake` where approval stake is the sum of votes. Hence, some number -/// bigger than u64::max_value() is needed. For maximum accuracy we simply use u128; -const DEN: u128 = u128::max_value(); +/// A pointer to a candidate struct with interior mutability. +pub type CandidatePtr = Rc>>; /// A candidate entity for the election. -#[derive(Clone, Default, Debug)] -struct Candidate { +#[derive(Debug, Clone, Default)] +pub struct Candidate { /// Identifier. who: AccountId, - /// Intermediary value used to sort candidates. + /// Score of the candidate. + /// + /// Used differently in seq-phragmen and max-score. score: Rational128, - /// Sum of the stake of this candidate based on received votes. + /// Approval stake of the candidate. Merely the sum of all the voter's stake who approve this + /// candidate. approval_stake: ExtendedBalance, - /// Flag for being elected. + /// The final stake of this candidate. Will be equal to a subset of approval stake. + backed_stake: ExtendedBalance, + /// True if this candidate is already elected in the current election. elected: bool, + /// The round index at which this candidate was elected. + round: usize, +} + +impl Candidate { + pub fn to_ptr(self) -> CandidatePtr { + Rc::new(RefCell::new(self)) + } +} + +/// A vote being casted by a [`Voter`] to a [`Candidate`] is an `Edge`. +#[derive(Clone, Default)] +pub struct Edge { + /// Identifier of the target. + /// + /// This is equivalent of `self.candidate.borrow().who`, yet it helps to avoid double borrow + /// errors of the candidate pointer. + who: AccountId, + /// Load of this edge. + load: Rational128, + /// Pointer to the candidate. + candidate: CandidatePtr, + /// The weight (i.e. stake given to `who`) of this edge. + weight: ExtendedBalance, +} + +#[cfg(feature = "std")] +impl sp_std::fmt::Debug for Edge { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + write!(f, "Edge({:?}, weight = {:?})", self.who, self.weight) + } } /// A voter entity. -#[derive(Clone, Default, Debug)] -struct Voter { +#[derive(Clone, Default)] +pub struct Voter { /// Identifier. who: AccountId, - /// List of candidates proposed by this voter. + /// List of candidates approved by this voter. edges: Vec>, /// The stake of this voter. budget: ExtendedBalance, - /// Incremented each time a candidate that this voter voted for has been elected. + /// Load of the voter. load: Rational128, } -/// A candidate being backed by a voter. -#[derive(Clone, Default, Debug)] -struct Edge { - /// Identifier. - who: AccountId, - /// Load of this vote. - load: Rational128, - /// Index of the candidate stored in the 'candidates' vector. - candidate_index: usize, +#[cfg(feature = "std")] +impl std::fmt::Debug for Voter { + fn fmt(&self, f: &mut sp_std::fmt::Formatter<'_>) -> sp_std::fmt::Result { + write!(f, "Voter({:?}, budget = {}, edges = {:?})", self.who, self.budget, self.edges) + } +} + +impl Voter { + /// Create a new `Voter`. + pub fn new(who: AccountId) -> Self { + Self { who, ..Default::default() } + } + + + /// Returns `true` if `self` votes for `target`. + /// + /// Note that this does not take into account if `target` is elected (i.e. is *active*) or not. + pub fn votes_for(&self, target: &AccountId) -> bool { + self.edges.iter().any(|e| &e.who == target) + } + + /// Returns none if this voter does not have any non-zero distributions. + /// + /// Note that this might create _un-normalized_ assignments, due to accuracy loss of `P`. Call + /// site might compensate by calling `normalize()` on the returned `Assignment` as a + /// post-precessing. + pub fn into_assignment(self) -> Option> + where + ExtendedBalance: From>, + { + let who = self.who; + let budget = self.budget; + let distribution = self.edges.into_iter().filter_map(|e| { + let per_thing = P::from_rational_approximation(e.weight, budget); + // trim zero edges. + if per_thing.is_zero() { None } else { Some((e.who, per_thing)) } + }).collect::>(); + + if distribution.len() > 0 { + Some(Assignment { who, distribution }) + } else { + None + } + } + + /// Try and normalize the votes of self. + /// + /// If the normalization is successful then `true` is returned. + /// + /// ### Errors + /// + /// This will return only if the internal `normalize` fails. This can happen if the sum of the + /// weights exceeds `ExtendedBalance::max_value()`. + pub fn try_normalize(&mut self) -> Result<(), &'static str> { + let edge_weights = self.edges.iter().map(|e| e.weight).collect::>(); + edge_weights.normalize(self.budget).map(|normalized| { + // here we count on the fact that normalize does not change the order. + for (edge, corrected) in self.edges.iter_mut().zip(normalized.into_iter()) { + let mut candidate = edge.candidate.borrow_mut(); + // first, subtract the incorrect weight + candidate.backed_stake = candidate.backed_stake.saturating_sub(edge.weight); + edge.weight = corrected; + // Then add the correct one again. + candidate.backed_stake = candidate.backed_stake.saturating_add(edge.weight); + } + }) + } } /// Final result of the election. #[derive(Debug)] -pub struct ElectionResult { +pub struct ElectionResult { /// Just winners zipped with their approval stake. Note that the approval stake is merely the /// sub of their received stake and could be used for very basic sorting and approval voting. pub winners: Vec>, - /// Individual assignments. for each tuple, the first elements is a voter and the second - /// is the list of candidates that it supports. - pub assignments: Vec>, + /// Individual assignments. for each tuple, the first elements is a voter and the second is the + /// list of candidates that it supports. + pub assignments: Vec>, } /// A voter's stake assignment among a set of targets, represented as ratios. @@ -184,8 +329,8 @@ where { /// Convert from a ratio assignment into one with absolute values aka. [`StakedAssignment`]. /// - /// It needs `stake` which is the total budget of the voter. If `fill` is set to true, - /// it _tries_ to ensure that all the potential rounding errors are compensated and the + /// It needs `stake` which is the total budget of the voter. If `fill` is set to true, it + /// _tries_ to ensure that all the potential rounding errors are compensated and the /// distribution's sum is exactly equal to the total budget, by adding or subtracting the /// remainder from the last distribution. /// @@ -219,6 +364,13 @@ where /// Try and normalize this assignment. /// /// If `Ok(())` is returned, then the assignment MUST have been successfully normalized to 100%. + /// + /// ### Errors + /// + /// This will return only if the internal `normalize` fails. This can happen if sum of + /// `self.distribution.map(|p| p.deconstruct())` fails to fit inside `UpperOf

`. A user of + /// this crate may statically assert that this can never happen and safely `expect` this to + /// return `Ok`. pub fn try_normalize(&mut self) -> Result<(), &'static str> { self.distribution .iter() @@ -289,9 +441,9 @@ impl StakedAssignment { /// /// NOTE: current implementation of `.normalize` is almost safe to `expect()` upon. The only /// error case is when the input cannot fit in `T`, or the sum of input cannot fit in `T`. - /// Sadly, both of these are dependent upon the implementation of `VoteLimit`, i.e. the limit - /// of edges per voter which is enforced from upstream. Hence, at this crate, we prefer - /// returning a result and a use the name prefix `try_`. + /// Sadly, both of these are dependent upon the implementation of `VoteLimit`, i.e. the limit of + /// edges per voter which is enforced from upstream. Hence, at this crate, we prefer returning a + /// result and a use the name prefix `try_`. pub fn try_normalize(&mut self, stake: ExtendedBalance) -> Result<(), &'static str> { self.distribution .iter() @@ -317,8 +469,8 @@ impl StakedAssignment { /// /// This complements the [`ElectionResult`] and is needed to run the balancing post-processing. /// -/// This, at the current version, resembles the `Exposure` defined in the Staking pallet, yet -/// they do not necessarily have to be the same. +/// This, at the current version, resembles the `Exposure` defined in the Staking pallet, yet they +/// do not necessarily have to be the same. #[derive(Default, Debug)] #[cfg_attr(feature = "std", derive(Serialize, Deserialize, Eq, PartialEq))] pub struct Support { @@ -331,228 +483,12 @@ pub struct Support { /// A linkage from a candidate and its [`Support`]. pub type SupportMap = BTreeMap>; -/// Perform election based on Phragmén algorithm. -/// -/// Returns an `Option` the set of winners and their detailed support ratio from each voter if -/// enough candidates are provided. Returns `None` otherwise. -/// -/// * `candidate_count`: number of candidates to elect. -/// * `minimum_candidate_count`: minimum number of candidates to elect. If less candidates exist, -/// `None` is returned. -/// * `initial_candidates`: candidates list to be elected from. -/// * `initial_voters`: voters list. -/// -/// This function does not strip out candidates who do not have any backing stake. It is the -/// responsibility of the caller to make sure only those candidates who have a sensible economic -/// value are passed in. From the perspective of this function, a candidate can easily be among the -/// winner with no backing stake. -pub fn seq_phragmen( - candidate_count: usize, - minimum_candidate_count: usize, - initial_candidates: Vec, - initial_voters: Vec<(AccountId, VoteWeight, Vec)>, -) -> Option> where - AccountId: Default + Ord + Clone, - R: PerThing, -{ - // return structures - let mut elected_candidates: Vec<(AccountId, ExtendedBalance)>; - let mut assigned: Vec>; - - // used to cache and access candidates index. - let mut c_idx_cache = BTreeMap::::new(); - - // voters list. - let num_voters = initial_candidates.len() + initial_voters.len(); - let mut voters: Vec> = Vec::with_capacity(num_voters); - - // Iterate once to create a cache of candidates indexes. This could be optimized by being - // provided by the call site. - let mut candidates = initial_candidates - .into_iter() - .enumerate() - .map(|(idx, who)| { - c_idx_cache.insert(who.clone(), idx); - Candidate { who, ..Default::default() } - }) - .collect::>>(); - - // early return if we don't have enough candidates - if candidates.len() < minimum_candidate_count { return None; } - - // collect voters. use `c_idx_cache` for fast access and aggregate `approval_stake` of - // candidates. - voters.extend(initial_voters.into_iter().map(|(who, voter_stake, votes)| { - let mut edges: Vec> = Vec::with_capacity(votes.len()); - for v in votes { - if edges.iter().any(|e| e.who == v) { - // duplicate edge. - continue; - } - if let Some(idx) = c_idx_cache.get(&v) { - // This candidate is valid + already cached. - candidates[*idx].approval_stake = candidates[*idx].approval_stake - .saturating_add(voter_stake.into()); - edges.push(Edge { who: v.clone(), candidate_index: *idx, ..Default::default() }); - } // else {} would be wrong votes. We don't really care about it. - } - Voter { - who, - edges: edges, - budget: voter_stake.into(), - load: Rational128::zero(), - } - })); - - - // we have already checked that we have more candidates than minimum_candidate_count. - let to_elect = candidate_count.min(candidates.len()); - elected_candidates = Vec::with_capacity(candidate_count); - assigned = Vec::with_capacity(candidate_count); - - // main election loop - for _round in 0..to_elect { - // loop 1: initialize score - for c in &mut candidates { - if !c.elected { - // 1 / approval_stake == (DEN / approval_stake) / DEN. If approval_stake is zero, - // then the ratio should be as large as possible, essentially `infinity`. - if c.approval_stake.is_zero() { - c.score = Rational128::from_unchecked(DEN, 0); - } else { - c.score = Rational128::from(DEN / c.approval_stake, DEN); - } - } - } - - // loop 2: increment score - for n in &voters { - for e in &n.edges { - let c = &mut candidates[e.candidate_index]; - if !c.elected && !c.approval_stake.is_zero() { - let temp_n = multiply_by_rational( - n.load.n(), - n.budget, - c.approval_stake, - ).unwrap_or_else(|_| Bounded::max_value()); - let temp_d = n.load.d(); - let temp = Rational128::from(temp_n, temp_d); - c.score = c.score.lazy_saturating_add(temp); - } - } - } - - // loop 3: find the best - if let Some(winner) = candidates - .iter_mut() - .filter(|c| !c.elected) - .min_by_key(|c| c.score) - { - // loop 3: update voter and edge load - winner.elected = true; - for n in &mut voters { - for e in &mut n.edges { - if e.who == winner.who { - e.load = winner.score.lazy_saturating_sub(n.load); - n.load = winner.score; - } - } - } - - elected_candidates.push((winner.who.clone(), winner.approval_stake)); - } else { - break - } - } // end of all rounds - - // update backing stake of candidates and voters - for n in &mut voters { - let mut assignment = Assignment { - who: n.who.clone(), - ..Default::default() - }; - for e in &mut n.edges { - if elected_candidates.iter().position(|(ref c, _)| *c == e.who).is_some() { - let per_bill_parts: R::Inner = - { - if n.load == e.load { - // Full support. No need to calculate. - R::ACCURACY - } else { - if e.load.d() == n.load.d() { - // return e.load / n.load. - let desired_scale: u128 = R::ACCURACY.saturated_into(); - let parts = multiply_by_rational( - desired_scale, - e.load.n(), - n.load.n(), - ) - // If result cannot fit in u128. Not much we can do about it. - .unwrap_or_else(|_| Bounded::max_value()); - - TryFrom::try_from(parts) - // If the result cannot fit into R::Inner. Defensive only. This can - // never happen. `desired_scale * e / n`, where `e / n < 1` always - // yields a value smaller than `desired_scale`, which will fit into - // R::Inner. - .unwrap_or_else(|_| Bounded::max_value()) - } else { - // defensive only. Both edge and voter loads are built from - // scores, hence MUST have the same denominator. - Zero::zero() - } - } - }; - let per_thing = R::from_parts(per_bill_parts); - assignment.distribution.push((e.who.clone(), per_thing)); - } - } - - let len = assignment.distribution.len(); - if len > 0 { - // To ensure an assertion indicating: no stake from the voter going to waste, - // we add a minimal post-processing to equally assign all of the leftover stake ratios. - let vote_count: R::Inner = len.saturated_into(); - let accuracy = R::ACCURACY; - let mut sum: R::Inner = Zero::zero(); - assignment.distribution.iter().for_each(|a| sum = sum.saturating_add(a.1.deconstruct())); - - let diff = accuracy.saturating_sub(sum); - let diff_per_vote = (diff / vote_count).min(accuracy); - - if !diff_per_vote.is_zero() { - for i in 0..len { - let current_ratio = assignment.distribution[i % len].1; - let next_ratio = current_ratio - .saturating_add(R::from_parts(diff_per_vote)); - assignment.distribution[i % len].1 = next_ratio; - } - } - - // `remainder` is set to be less than maximum votes of a voter (currently 16). - // safe to cast it to usize. - let remainder = diff - diff_per_vote * vote_count; - for i in 0..remainder.saturated_into::() { - let current_ratio = assignment.distribution[i % len].1; - let next_ratio = current_ratio.saturating_add(R::from_parts(1u8.into())); - assignment.distribution[i % len].1 = next_ratio; - } - assigned.push(assignment); - } - } - - Some(ElectionResult { - winners: elected_candidates, - assignments: assigned, - }) -} - /// Build the support map from the given election result. It maps a flat structure like /// /// ```nocompile /// assignments: vec![ -/// voter1, vec![(candidate1, w11), (candidate2, w12)], -/// voter2, vec![(candidate1, w21), (candidate2, w22)] +/// voter1, vec![(candidate1, w11), (candidate2, w12)], +/// voter2, vec![(candidate1, w21), (candidate2, w22)] /// ] /// ``` /// @@ -560,16 +496,16 @@ pub fn seq_phragmen( /// /// ```nocompile /// SupportMap { -/// candidate1: Support { -/// own:0, -/// total: w11 + w21, -/// others: vec![(candidate1, w11), (candidate2, w21)] -/// }, -/// candidate2: Support { -/// own:0, -/// total: w12 + w22, -/// others: vec![(candidate1, w12), (candidate2, w22)] -/// }, +/// candidate1: Support { +/// own:0, +/// total: w11 + w21, +/// others: vec![(candidate1, w11), (candidate2, w21)] +/// }, +/// candidate2: Support { +/// own:0, +/// total: w12 + w22, +/// others: vec![(candidate1, w12), (candidate2, w22)] +/// }, /// } /// ``` /// @@ -582,7 +518,7 @@ pub fn build_support_map( winners: &[AccountId], assignments: &[StakedAssignment], ) -> (SupportMap, u32) where - AccountId: Default + Ord + Clone, + AccountId: IdentifierT, { let mut errors = 0; // Initialize the support of each candidate. @@ -631,8 +567,8 @@ pub fn evaluate_support( [min_support, sum, sum_squared] } -/// Compares two sets of election scores based on desirability and returns true if `this` is -/// better than `that`. +/// Compares two sets of election scores based on desirability and returns true if `this` is better +/// than `that`. /// /// Evaluation is done in a lexicographic manner, and if each element of `this` is `that * epsilon` /// greater or less than `that`. @@ -665,139 +601,55 @@ pub fn is_score_better(this: ElectionScore, that: ElectionScore, ep } } -/// Performs balancing post-processing to the output of the election algorithm. This happens in -/// rounds. The number of rounds and the maximum diff-per-round tolerance can be tuned through input -/// parameters. -/// -/// Returns the number of iterations that were preformed. +/// Converts raw inputs to types used in this crate. /// -/// - `assignments`: exactly the same as the output of [`seq_phragmen`]. -/// - `supports`: mutable reference to s `SupportMap`. This parameter is updated. -/// - `tolerance`: maximum difference that can occur before an early quite happens. -/// - `iterations`: maximum number of iterations that will be processed. -pub fn balance_solution( - assignments: &mut Vec>, - supports: &mut SupportMap, - tolerance: ExtendedBalance, - iterations: usize, -) -> usize where AccountId: Ord + Clone { - if iterations == 0 { return 0; } - - let mut i = 0 ; - loop { - let mut max_diff = 0; - for assignment in assignments.iter_mut() { - let voter_budget = assignment.total(); - let StakedAssignment { who, distribution } = assignment; - let diff = do_balancing( - who, - voter_budget, - distribution, - supports, - tolerance, - ); - if diff > max_diff { max_diff = diff; } - } - - i += 1; - if max_diff <= tolerance || i >= iterations { - break i; - } - } -} - -/// actually perform balancing. same interface is `balance_solution`. Just called in loops with a check for -/// maximum difference. -fn do_balancing( - voter: &AccountId, - budget: ExtendedBalance, - elected_edges: &mut Vec<(AccountId, ExtendedBalance)>, - support_map: &mut SupportMap, - tolerance: ExtendedBalance -) -> ExtendedBalance where AccountId: Ord + Clone { - // Nothing to do. This voter had nothing useful. - // Defensive only. Assignment list should always be populated. 1 might happen for self vote. - if elected_edges.is_empty() || elected_edges.len() == 1 { return 0; } - - let stake_used = elected_edges - .iter() - .fold(0 as ExtendedBalance, |s, e| s.saturating_add(e.1)); - - let backed_stakes_iter = elected_edges - .iter() - .filter_map(|e| support_map.get(&e.0)) - .map(|e| e.total); +/// This will perform some cleanup that are most often important: +/// - It drops any votes that are pointing to non-candidates. +/// - It drops duplicate targets within a voter. +pub(crate) fn setup_inputs( + initial_candidates: Vec, + initial_voters: Vec<(AccountId, VoteWeight, Vec)>, +) -> (Vec>, Vec>) { + // used to cache and access candidates index. + let mut c_idx_cache = BTreeMap::::new(); - let backing_backed_stake = elected_edges - .iter() - .filter(|e| e.1 > 0) - .filter_map(|e| support_map.get(&e.0)) - .map(|e| e.total) - .collect::>(); - - let mut difference; - if backing_backed_stake.len() > 0 { - let max_stake = backing_backed_stake - .iter() - .max() - .expect("vector with positive length will have a max; qed"); - let min_stake = backed_stakes_iter - .min() - .expect("iterator with positive length will have a min; qed"); - - difference = max_stake.saturating_sub(min_stake); - difference = difference.saturating_add(budget.saturating_sub(stake_used)); - if difference < tolerance { - return difference; - } - } else { - difference = budget; - } + let candidates = initial_candidates + .into_iter() + .enumerate() + .map(|(idx, who)| { + c_idx_cache.insert(who.clone(), idx); + Candidate { who, ..Default::default() }.to_ptr() + }) + .collect::>>(); - // Undo updates to support - elected_edges.iter_mut().for_each(|e| { - if let Some(support) = support_map.get_mut(&e.0) { - support.total = support.total.saturating_sub(e.1); - support.voters.retain(|i_support| i_support.0 != *voter); - } - e.1 = 0; - }); - - elected_edges.sort_by_key(|e| - if let Some(e) = support_map.get(&e.0) { e.total } else { Zero::zero() } - ); - - let mut cumulative_stake: ExtendedBalance = 0; - let mut last_index = elected_edges.len() - 1; - let mut idx = 0usize; - for e in &mut elected_edges[..] { - if let Some(support) = support_map.get_mut(&e.0) { - let stake = support.total; - let stake_mul = stake.saturating_mul(idx as ExtendedBalance); - let stake_sub = stake_mul.saturating_sub(cumulative_stake); - if stake_sub > budget { - last_index = idx.checked_sub(1).unwrap_or(0); - break; + let voters = initial_voters.into_iter().map(|(who, voter_stake, votes)| { + let mut edges: Vec> = Vec::with_capacity(votes.len()); + for v in votes { + if edges.iter().any(|e| e.who == v) { + // duplicate edge. + continue; } - cumulative_stake = cumulative_stake.saturating_add(stake); + if let Some(idx) = c_idx_cache.get(&v) { + // This candidate is valid + already cached. + let mut candidate = candidates[*idx].borrow_mut(); + candidate.approval_stake = + candidate.approval_stake.saturating_add(voter_stake.into()); + edges.push( + Edge { + who: v.clone(), + candidate: Rc::clone(&candidates[*idx]), + ..Default::default() + } + ); + } // else {} would be wrong votes. We don't really care about it. } - idx += 1; - } - - let last_stake = elected_edges[last_index].1; - let split_ways = last_index + 1; - let excess = budget - .saturating_add(cumulative_stake) - .saturating_sub(last_stake.saturating_mul(split_ways as ExtendedBalance)); - elected_edges.iter_mut().take(split_ways).for_each(|e| { - if let Some(support) = support_map.get_mut(&e.0) { - e.1 = (excess / split_ways as ExtendedBalance) - .saturating_add(last_stake) - .saturating_sub(support.total); - support.total = support.total.saturating_add(e.1); - support.voters.push((voter.clone(), e.1)); + Voter { + who, + edges: edges, + budget: voter_stake.into(), + load: Rational128::zero(), } - }); + }).collect::>(); - difference + (candidates, voters,) } diff --git a/primitives/npos-elections/src/mock.rs b/primitives/npos-elections/src/mock.rs index 9b25f6f5f2e37..32c9d1223862a 100644 --- a/primitives/npos-elections/src/mock.rs +++ b/primitives/npos-elections/src/mock.rs @@ -20,7 +20,7 @@ #![cfg(test)] use crate::{seq_phragmen, ElectionResult, Assignment, VoteWeight, ExtendedBalance}; -use sp_arithmetic::{PerThing, traits::{SaturatedConversion, Zero, One}}; +use sp_arithmetic::{PerThing, InnerOf, traits::{SaturatedConversion, Zero, One}}; use sp_std::collections::btree_map::BTreeMap; use sp_runtime::assert_eq_error_rate; @@ -71,7 +71,6 @@ pub(crate) fn auto_generate_self_voters(candidates: &[A]) -> Vec<(A, V pub(crate) fn elect_float( candidate_count: usize, - minimum_candidate_count: usize, initial_candidates: Vec, initial_voters: Vec<(A, Vec)>, stake_of: FS, @@ -94,10 +93,6 @@ pub(crate) fn elect_float( }) .collect::>>(); - if candidates.len() < minimum_candidate_count { - return None; - } - voters.extend(initial_voters.into_iter().map(|(who, votes)| { let voter_stake = stake_of(&who) as f64; let mut edges: Vec<_Edge> = Vec::with_capacity(votes.len()); @@ -314,7 +309,7 @@ pub fn check_assignments_sum(assignments: Vec( voters: Vec<(AccountId, Vec)>, stake_of: &Box VoteWeight>, to_elect: usize, - min_to_elect: usize, -) { +) where + ExtendedBalance: From>, + Output: sp_std::ops::Mul, +{ // run fixed point code. let ElectionResult { winners, assignments } = seq_phragmen::<_, Output>( to_elect, - min_to_elect, candidates.clone(), voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None ).unwrap(); // run float poc code. let truth_value = elect_float( to_elect, - min_to_elect, candidates, voters, &stake_of, @@ -354,7 +350,11 @@ pub(crate) fn run_and_compare( Output::Inner::one(), ); } else { - panic!("candidate mismatch. This should never happen.") + panic!( + "candidate mismatch. This should never happen. could not find ({:?}, {:?})", + candidate, + per_thingy, + ) } } } else { diff --git a/primitives/npos-elections/src/phragmen.rs b/primitives/npos-elections/src/phragmen.rs new file mode 100644 index 0000000000000..87fa0bde4b7ef --- /dev/null +++ b/primitives/npos-elections/src/phragmen.rs @@ -0,0 +1,203 @@ +// This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementation of the sequential-phragmen election method. +//! +//! This method is ensured to achieve PJR, yet, it does not achieve a constant factor approximation +//! to the Maximin problem. + +use crate::{ + IdentifierT, VoteWeight, Voter, CandidatePtr, ExtendedBalance, setup_inputs, ElectionResult, +}; +use sp_std::prelude::*; +use sp_arithmetic::{ + PerThing, InnerOf, Rational128, + helpers_128bit::multiply_by_rational, + traits::{Zero, Bounded}, +}; +use crate::balancing; + +/// The denominator used for loads. Since votes are collected as u64, the smallest ratio that we +/// might collect is `1/approval_stake` where approval stake is the sum of votes. Hence, some number +/// bigger than u64::max_value() is needed. For maximum accuracy we simply use u128; +const DEN: ExtendedBalance = ExtendedBalance::max_value(); + +/// Execute sequential phragmen with potentially some rounds of `balancing`. The return type is list +/// of winners and a weight distribution vector of all voters who contribute to the winners. +/// +/// - This function is a best effort to elect `rounds` members. Nonetheless, if less candidates are +/// available, it will only return what is available. It is the responsibility of the call site to +/// ensure they have provided enough members. +/// - If `balance` parameter is `Some(i, t)`, `i` iterations of balancing is with tolerance `t` is +/// performed. +/// - Returning winners are sorted based on desirability. Voters are unsorted. Nonetheless, +/// seq-phragmen is in general an un-ranked election and the desirability should not be +/// interpreted with any significance. +/// - The returning winners are zipped with their final backing stake. Yet, to get the exact final +/// weight distribution from the winner's point of view, one needs to build a support map. See +/// [`crate::SupportMap`] for more info. Note that this backing stake is computed in +/// ExtendedBalance and may be slightly different that what will be computed from the support map, +/// due to accuracy loss. +/// - The accuracy of the returning edge weight ratios can be configured via the `P` generic +/// argument. +/// - The returning weight distribution is _normalized_, meaning that it is guaranteed that the sum +/// of the ratios in each voter's distribution sums up to exactly `P::one()`. +/// +/// This can only fail of the normalization fails. This can happen if for any of the resulting +/// assignments, `assignment.distribution.map(|p| p.deconstruct()).sum()` fails to fit inside +/// `UpperOf

`. A user of this crate may statically assert that this can never happen and safely +/// `expect` this to return `Ok`. +/// +/// This can only fail if the normalization fails. +pub fn seq_phragmen( + rounds: usize, + initial_candidates: Vec, + initial_voters: Vec<(AccountId, VoteWeight, Vec)>, + balance: Option<(usize, ExtendedBalance)>, +) -> Result, &'static str> where ExtendedBalance: From> { + let (candidates, voters) = setup_inputs(initial_candidates, initial_voters); + + let (candidates, mut voters) = seq_phragmen_core::( + rounds, + candidates, + voters, + )?; + + if let Some((iterations, tolerance)) = balance { + // NOTE: might create zer-edges, but we will strip them again when we convert voter into + // assignment. + let _iters = balancing::balance::(&mut voters, iterations, tolerance); + } + + let mut winners = candidates + .into_iter() + .filter(|c_ptr| c_ptr.borrow().elected) + .take(rounds) // defensive only: seq-phragmen-core returns only up to rounds. + .collect::>(); + + // sort winners based on desirability. + winners.sort_by_key(|c_ptr| c_ptr.borrow().round); + + let mut assignments = voters.into_iter().filter_map(|v| v.into_assignment()).collect::>(); + let _ = assignments.iter_mut().map(|a| a.try_normalize()).collect::>()?; + let winners = winners.into_iter().map(|w_ptr| + (w_ptr.borrow().who.clone(), w_ptr.borrow().backed_stake) + ).collect(); + + Ok(ElectionResult { winners, assignments }) +} + +/// Core implementation of seq-phragmen. +/// +/// This is the internal implementation that works with the types defined in this crate. see +/// `seq_phragmen` for more information. This function is left public in case a crate needs to use +/// the implementation in a custom way. +/// +/// To create th inputs needed for this function, see [`crate::setup_inputs`]. +/// +/// This can only fail if the normalization fails. +pub fn seq_phragmen_core( + rounds: usize, + candidates: Vec>, + mut voters: Vec>, +) -> Result<(Vec>, Vec>), &'static str> { + // we have already checked that we have more candidates than minimum_candidate_count. + let to_elect = rounds.min(candidates.len()); + + // main election loop + for round in 0..to_elect { + // loop 1: initialize score + for c_ptr in &candidates { + let mut candidate = c_ptr.borrow_mut(); + if !candidate.elected { + // 1 / approval_stake == (DEN / approval_stake) / DEN. If approval_stake is zero, + // then the ratio should be as large as possible, essentially `infinity`. + if candidate.approval_stake.is_zero() { + candidate.score = Bounded::max_value(); + } else { + candidate.score = Rational128::from(DEN / candidate.approval_stake, DEN); + } + } + } + + // loop 2: increment score + for voter in &voters { + for edge in &voter.edges { + let mut candidate = edge.candidate.borrow_mut(); + if !candidate.elected && !candidate.approval_stake.is_zero() { + let temp_n = multiply_by_rational( + voter.load.n(), + voter.budget, + candidate.approval_stake, + ).unwrap_or(Bounded::max_value()); + let temp_d = voter.load.d(); + let temp = Rational128::from(temp_n, temp_d); + candidate.score = candidate.score.lazy_saturating_add(temp); + } + } + } + + // loop 3: find the best + if let Some(winner_ptr) = candidates + .iter() + .filter(|c| !c.borrow().elected) + .min_by_key(|c| c.borrow().score) + { + let mut winner = winner_ptr.borrow_mut(); + // loop 3: update voter and edge load + winner.elected = true; + winner.round = round; + for voter in &mut voters { + for edge in &mut voter.edges { + if edge.who == winner.who { + edge.load = winner.score.lazy_saturating_sub(voter.load); + voter.load = winner.score; + } + } + } + } else { + break + } + } + + // update backing stake of candidates and voters + for voter in &mut voters { + for edge in &mut voter.edges { + if edge.candidate.borrow().elected { + // update internal state. + edge.weight = multiply_by_rational( + voter.budget, + edge.load.n(), + voter.load.n(), + ) + // If result cannot fit in u128. Not much we can do about it. + .unwrap_or(Bounded::max_value()); + } else { + edge.weight = 0 + } + let mut candidate = edge.candidate.borrow_mut(); + candidate.backed_stake = candidate.backed_stake.saturating_add(edge.weight); + } + + // remove all zero edges. These can become phantom edges during normalization. + voter.edges.retain(|e| e.weight > 0); + // inc budget to sum the budget + voter.try_normalize()?; + } + + Ok((candidates, voters)) +} diff --git a/primitives/npos-elections/src/phragmms.rs b/primitives/npos-elections/src/phragmms.rs new file mode 100644 index 0000000000000..337d86a93aeb9 --- /dev/null +++ b/primitives/npos-elections/src/phragmms.rs @@ -0,0 +1,399 @@ + // This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implementation of the PhragMMS method. +//! +//! The naming comes from the fact that this method is highly inspired by Phragmen's method, yet it +//! _also_ provides a constant factor approximation of the Maximin problem, similar to that of the +//! MMS algorithm. + +use crate::{ + IdentifierT, ElectionResult, ExtendedBalance, setup_inputs, VoteWeight, Voter, CandidatePtr, + balance, +}; +use sp_arithmetic::{PerThing, InnerOf, Rational128, traits::Bounded}; +use sp_std::{prelude::*, rc::Rc}; + +/// Execute the phragmms method. +/// +/// This can be used interchangeably with [`seq-phragmen`] and offers a similar API, namely: +/// +/// - The resulting edge weight distribution is normalized (thus, safe to use for submission). +/// - The accuracy can be configured via the generic type `P`. +/// - The algorithm is a _best-effort_ to elect `to_elect`. If less candidates are provided, less +/// winners are returned, without an error. +/// +/// This can only fail of the normalization fails. This can happen if for any of the resulting +/// assignments, `assignment.distribution.map(|p| p.deconstruct()).sum()` fails to fit inside +/// `UpperOf

`. A user of this crate may statically assert that this can never happen and safely +/// `expect` this to return `Ok`. +pub fn phragmms( + to_elect: usize, + initial_candidates: Vec, + initial_voters: Vec<(AccountId, VoteWeight, Vec)>, + balancing_config: Option<(usize, ExtendedBalance)>, +) -> Result, &'static str> + where ExtendedBalance: From> +{ + let (candidates, mut voters) = setup_inputs(initial_candidates, initial_voters); + + let mut winners = vec![]; + for round in 0..to_elect { + if let Some(round_winner) = calculate_max_score::(&candidates, &voters) { + apply_elected::(&mut voters, Rc::clone(&round_winner)); + + round_winner.borrow_mut().round = round; + round_winner.borrow_mut().elected = true; + winners.push(round_winner); + + if let Some((iterations, tolerance)) = balancing_config { + balance(&mut voters, iterations, tolerance); + } + } else { + break; + } + } + + let mut assignments = voters.into_iter().filter_map(|v| v.into_assignment()).collect::>(); + let _ = assignments.iter_mut().map(|a| a.try_normalize()).collect::>()?; + let winners = winners.into_iter().map(|w_ptr| + (w_ptr.borrow().who.clone(), w_ptr.borrow().backed_stake) + ).collect(); + + Ok(ElectionResult { winners, assignments }) +} + +/// Find the candidate that can yield the maximum score for this round. +/// +/// Returns a new `Some(CandidatePtr)` to the winner candidate. The score of the candidate is +/// updated and can be read from the returned pointer. +/// +/// If no winner can be determined (i.e. everyone is already elected), then `None` is returned. +/// +/// This is an internal part of the [`balanced_heuristic`]. +pub(crate) fn calculate_max_score( + candidates: &[CandidatePtr], + voters: &[Voter], +) -> Option> where ExtendedBalance: From> { + for c_ptr in candidates.iter() { + let mut candidate = c_ptr.borrow_mut(); + if !candidate.elected { + candidate.score = Rational128::from(1, P::ACCURACY.into()); + } + } + + for voter in voters.iter() { + let mut denominator_contribution: ExtendedBalance = 0; + + // gather contribution from all elected edges. + for edge in voter.edges.iter() { + let edge_candidate = edge.candidate.borrow(); + if edge_candidate.elected { + let edge_contribution: ExtendedBalance = P::from_rational_approximation( + edge.weight, + edge_candidate.backed_stake, + ).deconstruct().into(); + denominator_contribution += edge_contribution; + } + } + + // distribute to all _unelected_ edges. + for edge in voter.edges.iter() { + let mut edge_candidate = edge.candidate.borrow_mut(); + if !edge_candidate.elected { + let prev_d = edge_candidate.score.d(); + edge_candidate.score = Rational128::from(1, denominator_contribution + prev_d); + } + } + } + + // finalise the score value, and find the best. + let mut best_score = Rational128::zero(); + let mut best_candidate = None; + + for c_ptr in candidates.iter() { + let mut candidate = c_ptr.borrow_mut(); + if candidate.approval_stake > 0 { + // finalise the score value. + let score_d = candidate.score.d(); + let one: ExtendedBalance = P::ACCURACY.into(); + // Note: the accuracy here is questionable. + // First, let's consider what will happen if this saturates. In this case, two very + // whale-like validators will be effectively the same and their score will be equal. + // This is, more or less fine if the threshold of saturation is high and only a small + // subset or ever likely to become saturated. Once saturated, the score of these whales + // are effectively the same. + // Let's consider when this will happen. The approval stake of a target is the sum of + // stake of all the voter who have backed this target. Given the fact that the total + // issuance of a sane chain will fit in u128, it is safe to also assume that the + // approval stake will, since it is a subset of the total issuance at most. + // Finally, the only chance of overflow is multiplication by `one`. This highly depends + // on the `P` generic argument. With a PerBill and a 12 decimal token the maximum value + // that `candidate.approval_stake` can have is: + // (2 ** 128 - 1) / 10**9 / 10**12 = 340,282,366,920,938,463 + // Assuming that each target will have 200,000 voters, then each voter's stake can be + // roughly: + // (2 ** 128 - 1) / 10**9 / 10**12 / 200000 = 1,701,411,834,604 + // + // It is worth noting that these value would be _very_ different if one were to use + // `PerQuintill` as `P`. For now, we prefer the performance of using `Rational128` here. + // For the future, a properly benchmarked pull request can prove that using + // `RationalInfinite` as the score type does not introduce significant overhead. Then we + // can switch the score type to `RationalInfinite` and ensure compatibility with any + // crazy token scale. + let score_n = candidate.approval_stake.checked_mul(one).unwrap_or_else(|| Bounded::max_value()); + candidate.score = Rational128::from(score_n, score_d); + + // check if we have a new winner. + if !candidate.elected && candidate.score > best_score { + best_score = candidate.score; + best_candidate = Some(Rc::clone(&c_ptr)); + } + } else { + candidate.score = Rational128::zero(); + } + } + + best_candidate +} + +/// Update the weights of `voters` given that `elected_ptr` has been elected in the previous round. +/// +/// Updates `voters` in place. +/// +/// This is an internal part of the [`balanced_heuristic`] and should be called after +/// [`calculate_max_score`]. +pub(crate) fn apply_elected( + voters: &mut Vec>, + elected_ptr: CandidatePtr, +) { + let elected_who = elected_ptr.borrow().who.clone(); + let cutoff = elected_ptr.borrow().score.to_den(1) + .expect("(n / d) < u128::max() and (n' / 1) == (n / d), thus n' < u128::max()'; qed.") + .n(); + + let mut elected_backed_stake = elected_ptr.borrow().backed_stake; + for voter in voters { + if let Some(new_edge_index) = voter.edges.iter().position(|e| e.who == elected_who) { + let used_budget: ExtendedBalance = voter.edges.iter().map(|e| e.weight).sum(); + + let mut new_edge_weight = voter.budget.saturating_sub(used_budget); + elected_backed_stake = elected_backed_stake.saturating_add(new_edge_weight); + + // Iterate over all other edges. + for (_, edge) in voter.edges + .iter_mut() + .enumerate() + .filter(|(edge_index, edge_inner)| *edge_index != new_edge_index && edge_inner.weight > 0) + { + let mut edge_candidate = edge.candidate.borrow_mut(); + if edge_candidate.backed_stake > cutoff { + let stake_to_take = edge.weight.saturating_mul(cutoff) / edge_candidate.backed_stake.max(1); + + // subtract this amount from this edge. + edge.weight = edge.weight.saturating_sub(stake_to_take); + edge_candidate.backed_stake = edge_candidate.backed_stake.saturating_sub(stake_to_take); + + // inject it into the outer loop's edge. + elected_backed_stake = elected_backed_stake.saturating_add(stake_to_take); + new_edge_weight = new_edge_weight.saturating_add(stake_to_take); + } + } + + voter.edges[new_edge_index].weight = new_edge_weight; + } + } + + // final update. + elected_ptr.borrow_mut().backed_stake = elected_backed_stake; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ElectionResult, Assignment}; + use sp_runtime::{Perbill, Percent}; + use sp_std::rc::Rc; + + #[test] + fn basic_election_manual_works() { + //! Manually run the internal steps of phragmms. In each round we select a new winner by + //! `max_score`, then apply this change by `apply_elected`, and finally do a `balance` round. + let candidates = vec![1, 2, 3]; + let voters = vec![ + (10, 10, vec![1, 2]), + (20, 20, vec![1, 3]), + (30, 30, vec![2, 3]), + ]; + + let (candidates, mut voters) = setup_inputs(candidates, voters); + + // Round 1 + let winner = calculate_max_score::(candidates.as_ref(), voters.as_ref()).unwrap(); + assert_eq!(winner.borrow().who, 3); + assert_eq!(winner.borrow().score, 50u32.into()); + + apply_elected(&mut voters, Rc::clone(&winner)); + assert_eq!( + voters.iter().find(|x| x.who == 30).map(|v| ( + v.who, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>() + )).unwrap(), + (30, vec![(2, 0), (3, 30)]), + ); + assert_eq!( + voters.iter().find(|x| x.who == 20).map(|v| ( + v.who, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>() + )).unwrap(), + (20, vec![(1, 0), (3, 20)]), + ); + + // finish the round. + winner.borrow_mut().elected = true; + winner.borrow_mut().round = 0; + drop(winner); + + // balancing makes no difference here but anyhow. + balance(&mut voters, 10, 0); + + // round 2 + let winner = calculate_max_score::(candidates.as_ref(), voters.as_ref()).unwrap(); + assert_eq!(winner.borrow().who, 2); + assert_eq!(winner.borrow().score, 25u32.into()); + + apply_elected(&mut voters, Rc::clone(&winner)); + assert_eq!( + voters.iter().find(|x| x.who == 30).map(|v| ( + v.who, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>() + )).unwrap(), + (30, vec![(2, 15), (3, 15)]), + ); + assert_eq!( + voters.iter().find(|x| x.who == 20).map(|v| ( + v.who, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>() + )).unwrap(), + (20, vec![(1, 0), (3, 20)]), + ); + assert_eq!( + voters.iter().find(|x| x.who == 10).map(|v| ( + v.who, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>() + )).unwrap(), + (10, vec![(1, 0), (2, 10)]), + ); + + // finish the round. + winner.borrow_mut().elected = true; + winner.borrow_mut().round = 0; + drop(winner); + + // balancing will improve stuff here. + balance(&mut voters, 10, 0); + + assert_eq!( + voters.iter().find(|x| x.who == 30).map(|v| ( + v.who, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>() + )).unwrap(), + (30, vec![(2, 20), (3, 10)]), + ); + assert_eq!( + voters.iter().find(|x| x.who == 20).map(|v| ( + v.who, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>() + )).unwrap(), + (20, vec![(1, 0), (3, 20)]), + ); + assert_eq!( + voters.iter().find(|x| x.who == 10).map(|v| ( + v.who, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>() + )).unwrap(), + (10, vec![(1, 0), (2, 10)]), + ); + } + + #[test] + fn basic_election_works() { + let candidates = vec![1, 2, 3]; + let voters = vec![ + (10, 10, vec![1, 2]), + (20, 20, vec![1, 3]), + (30, 30, vec![2, 3]), + ]; + + let ElectionResult { winners, assignments } = phragmms::<_, Perbill>(2, candidates, voters, Some((2, 0))).unwrap(); + assert_eq!(winners, vec![(3, 30), (2, 30)]); + assert_eq!( + assignments, + vec![ + Assignment { + who: 10u64, + distribution: vec![(2, Perbill::one())], + }, + Assignment { + who: 20, + distribution: vec![(3, Perbill::one())], + }, + Assignment { + who: 30, + distribution: vec![ + (2, Perbill::from_parts(666666666)), + (3, Perbill::from_parts(333333334)), + ], + }, + ] + ) + } + + #[test] + fn linear_voting_example_works() { + let candidates = vec![11, 21, 31, 41, 51, 61, 71]; + let voters = vec![ + (2, 2000, vec![11]), + (4, 1000, vec![11, 21]), + (6, 1000, vec![21, 31]), + (8, 1000, vec![31, 41]), + (110, 1000, vec![41, 51]), + (120, 1000, vec![51, 61]), + (130, 1000, vec![61, 71]), + ]; + + let ElectionResult { winners, assignments: _ } = phragmms::<_, Perbill>(4, candidates, voters, Some((2, 0))).unwrap(); + assert_eq!(winners, vec![ + (11, 3000), + (31, 2000), + (51, 1500), + (61, 1500), + ]); + } + + #[test] + fn large_balance_wont_overflow() { + let candidates = vec![1u32, 2, 3]; + let mut voters = (0..1000).map(|i| (10 + i, u64::max_value(), vec![1, 2, 3])).collect::>(); + + // give a bit more to 1 and 3. + voters.push((2, u64::max_value(), vec![1, 3])); + + let ElectionResult { winners, assignments: _ } = phragmms::<_, Perbill>(2, candidates, voters, Some((2, 0))).unwrap(); + assert_eq!(winners.into_iter().map(|(w, _)| w).collect::>(), vec![1u32, 3]); + } +} diff --git a/primitives/npos-elections/src/pjr.rs b/primitives/npos-elections/src/pjr.rs new file mode 100644 index 0000000000000..2a700a276f814 --- /dev/null +++ b/primitives/npos-elections/src/pjr.rs @@ -0,0 +1,335 @@ + // This file is part of Substrate. + +// Copyright (C) 2020 Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Implements functions and interfaces to check solutions for being t-PJR. +//! +//! PJR stands for proportional justified representation. PJR is a string absolute measure to make +//! sure an NPoS solution adheres to a minimum standard. +//! +//! See [`pjr_check`] which is the main entry point of the module. + +use crate::*; +use sp_std::rc::Rc; +use sp_std::collections::btree_map::BTreeMap; +use sp_arithmetic::{traits::Zero, Perbill}; + + +/// The type used as the threshold. +/// +/// Just some reading sugar; Must always be same as [`ExtendedBalance`]; +type Threshold = ExtendedBalance; + +/// Convert the data types that the user runtime has into ones that can be used by this module. +/// +/// It is expected that this function's interface might change over time, or multiple variants of it +/// can be provided for different use cases. +/// +/// The ultimate goal, in any case, is to convert the election data into [`Candidate`] and [`Voter`] +/// types defined by this crate, whilst setting correct value for some of their fields, namely: +/// 1. Candidate [`backing_stake`](Candidate::backing_stake) and [`elected`](Candidate::elected) if they are a winner. +/// 2. Voter edge [`weight`](Edge::weight) if they are backing a winner. +/// 3. Voter [`budget`](Voter::budget). +/// +/// None of the `load` or `score` values are used and can be ignored. This is similar to +/// [`setup_inputs`] function of this crate. +/// +/// ### Performance (Weight) Notes +/// +/// Note that the current function is rather unfortunately inefficient. The most significant +/// slowdown is the fact that a typical solution that need to be checked for PJR only contains a +/// subset of the entire NPoS edge graph, encoded as `staked_assignment`. This only encodes the +/// edges that actually contribute to a winner's backing stake and ignores the rest to save space. +/// To check PJR, we need the entire voter set, including those edges that point to non-winners. +/// This could cause the caller runtime to have to read the entire list of voters, which is assumed +/// to be expensive. +/// +/// A sensible user of this module should make sure that the PJR check is executed and checked as +/// little as possible, and take sufficient economical measures to ensure that this function cannot +/// be abused. +pub fn prepare_pjr_input( + winners: Vec, + staked_assignments: Vec>, + supports: &SupportMap, + all_candidates: Vec, + all_voters: Vec<(A, VoteWeight, Vec)>, +) -> (Vec>, Vec>) { + // collect all candidates and winners into a unified `Vec`. + let mut candidates_index: BTreeMap = BTreeMap::new(); + + // dump the staked assignments in a voter-major map for faster access down the road. + let mut assignment_map: BTreeMap> = BTreeMap::new(); + staked_assignments + .into_iter() + .for_each(|StakedAssignment { who, distribution }| { + assignment_map.insert(who, distribution); + }); + + let candidates = all_candidates.into_iter().enumerate().map(|(i, c)| { + candidates_index.insert(c.clone(), i); + + // set the backing value and elected flag if the candidate is among the winners. + let who = c; + let elected = winners.iter().any(|w| w == &who); + let backed_stake = supports.get(&who).map(|s| s.total).unwrap_or_default(); + + debug_assert!( + !(elected ^ (backed_stake > 0)), + "If a candidate is elected, then it must have a positive backing as well." + ); + + Candidate { who, elected, backed_stake, ..Default::default() }.to_ptr() + }).collect::>(); + + // collect all voters into a unified Vec. + let voters = all_voters.into_iter().map(|(v, w, ts)| { + let mut edges: Vec> = Vec::with_capacity(ts.len()); + for t in ts { + if edges.iter().any(|e| e.who == t) { + // duplicate edge. + continue; + } + + if let Some(idx) = candidates_index.get(&t) { + // if this edge is among the assignments, set the weight as well. + let weight = assignment_map + .get(&v) + .and_then(|d| d.iter().find_map(|(x, y)| if x == &t { Some(y) } else { None })) + .cloned() + .unwrap_or_default(); + edges.push(Edge { + who: t, + candidate: Rc::clone(&candidates[*idx]), + weight, + ..Default::default() + }); + } + } + + let who = v; + let budget: ExtendedBalance = w.into(); + Voter { who, budget, edges, ..Default::default() } + }).collect::>(); + + (candidates, voters) +} + +/// Check a solution to be t-PJR. +/// +/// ### Semantics +/// +/// For a solution to be t-PJR, the original condition is as such: If there is a group of `N` voters +/// who have `r` common candidates and can afford to support each of them with backing stake `t` +/// (i.e `sum(stake(v) for all voters ) == r * t`), then this committee need to be represented by at +/// least `r` elected candidates. +/// +/// Section 5 of the NPoS paper shows that this property is equal to: For a feasible solution, if +/// `Max {score(c)} < t` where c is every unelected candidate, then this solution is t-PJR. +/// +/// In this implementation we use the latter definition due to its simplicity. +/// +/// ### Interface +/// +/// In addition to data that can be computed from the [`ElectionResult`] struct, a PJR check also +/// needs to inspect un-elected candidates and edges, thus `all_candidates` and `all_voters`. +/// +/// See [`prepare_pjr_input`] for more info. +pub fn pjr_check( + winners: Vec, + staked_assignments: Vec>, + supports: &SupportMap, + all_candidates: Vec, + all_voters: Vec<(A, VoteWeight, Vec)>, + t: Threshold, +) -> bool { + // prepare data. + let (candidates, voters) = prepare_pjr_input( + winners, + staked_assignments, + supports, + all_candidates, + all_voters, + ); + // compute with threshold t. + pjr_check_core(candidates.as_ref(), voters.as_ref(), t) +} + +/// The internal implementation of the PJR check after having the data converted. +/// +/// See [`pjr_check`] for more info. +pub fn pjr_check_core( + candidates: &[CandidatePtr], + voters: &[Voter], + t: Threshold, +) -> bool { + let unelected = candidates.iter().filter(|c| !c.borrow().elected); + let maybe_max_pre_score = unelected.map(|c| pre_score(Rc::clone(c), voters, t)).max(); + // if unelected is empty then the solution is indeed PJR. + maybe_max_pre_score.map_or(true, |max_pre_score| max_pre_score < t) +} + +/// The pre-score of an unelected candidate. +/// +/// This is the amount of stake that *all voter* can spare to devote to this candidate without +/// allowing the backing stake of any other elected candidate to fall below `t`. +/// +/// In essence, it is the sum(slack(n, t)) for all `n` who vote for `unelected`. +pub fn pre_score( + unelected: CandidatePtr, + voters: &[Voter], + t: Threshold, +) -> ExtendedBalance { + debug_assert!(!unelected.borrow().elected); + voters + .iter() + .filter(|ref v| v.votes_for(&unelected.borrow().who)) + .fold(Zero::zero(), |acc: ExtendedBalance, voter| acc.saturating_add(slack(voter, t))) +} + + +/// The slack of a voter at a given state. +/// +/// The slack of each voter, with threshold `t` is the total amount of stake that this voter can +/// spare to a new potential member, whilst not dropping the backing stake of any of its currently +/// active members below `t`. In essence, for each of the current active candidates `c`, we assume +/// that we reduce the edge weight of `voter` to `c` from `w` to `w * min(1 / (t / support(c)))`. +/// +/// More accurately: +/// +/// 1. If `c` exactly has `t` backing or less, then we don't generate any slack. +/// 2. If `c` has more than `t`, then we reduce it to `t`. +pub fn slack(voter: &Voter, t: Threshold) -> ExtendedBalance { + let budget = voter.budget; + let leftover = voter.edges.iter().fold(Zero::zero(), |acc: ExtendedBalance, edge| { + let candidate = edge.candidate.borrow(); + if candidate.elected { + // TODO: using perbill here is just going to cause annoyance, why not just subtract? + let extra = + Perbill::one().min(Perbill::from_rational_approximation(t, candidate.backed_stake)) + * edge.weight; + acc.saturating_add(extra) + } else { + // No slack generated here. + acc + } + }); + + // NOTE: candidate for saturating_log_sub() + budget.saturating_sub(leftover) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn setup_voter(who: u32, votes: Vec<(u32, u128, bool)>) -> Voter { + let mut voter = Voter::new(who); + let mut budget = 0u128; + let candidates = votes.into_iter().map(|(t, w, e)| { + budget += w; + Candidate { who: t, elected: e, backed_stake: w, ..Default::default() } + }).collect::>(); + let edges = candidates.into_iter().map(|c| + Edge { who: c.who, weight: c.backed_stake, candidate: c.to_ptr(), ..Default::default() } + ).collect::>(); + voter.edges = edges; + voter.budget = budget; + voter + } + + #[test] + fn slack_works() { + let voter = setup_voter(10, vec![(1, 10, true), (2, 20, true)]); + + assert_eq!(slack(&voter, 15), 5); + assert_eq!(slack(&voter, 17), 3); + assert_eq!(slack(&voter, 10), 10); + assert_eq!(slack(&voter, 5), 20); + + } + + #[test] + fn pre_score_works() { + // will give 5 slack + let v1 = setup_voter(10, vec![(1, 10, true), (2, 20, true), (3, 0, false)]); + // will give no slack + let v2 = setup_voter(20, vec![(1, 5, true), (2, 5, true)]); + // will give 10 slack. + let v3 = setup_voter(30, vec![(1, 20, true), (2, 20, true), (3, 0, false)]); + + let unelected = Candidate { who: 3u32, elected: false, ..Default::default() }.to_ptr(); + let score = pre_score(unelected, &vec![v1, v2, v3], 15); + + assert_eq!(score, 15); + } + + #[test] + fn can_convert_data_from_external_api() { + let winners = vec![20u32, 40]; + let all_candidates = vec![10, 20, 30, 40]; + let staked_assignments = vec![ + StakedAssignment { who: 1, distribution: vec![(20, 5), (40, 5)] }, + StakedAssignment { who: 2, distribution: vec![(20, 10), (40, 10)] }, + ]; + let all_voters = vec![ + (1, 10, vec![10, 20, 30, 40]), + (2, 20, vec![10, 20, 30, 40]), + (3, 30, vec![10, 30]), + ]; + let mut supports = SupportMap::::new(); + supports.insert(20, Support { total: 15, voters: vec![(5, 1), (10, 2)]} ); + supports.insert(40, Support { total: 15, voters: vec![(5, 1), (10, 2)]} ); + + let (candidates, voters) = prepare_pjr_input( + winners, + staked_assignments, + &supports, + all_candidates, + all_voters, + ); + + // elected flag and backing must be set correctly + assert_eq!( + candidates + .iter() + .map(|c| (c.borrow().who.clone(), c.borrow().elected, c.borrow().backed_stake)) + .collect::>(), + vec![(10, false, 0), (20, true, 15), (30, false, 0), (40, true, 15)], + ); + + // edge weight must be set correctly + assert_eq!( + voters + .iter() + .map(|v| ( + v.who, + v.budget, + v.edges.iter().map(|e| (e.who, e.weight)).collect::>(), + )).collect::>(), + vec![ + (1, 10, vec![(10, 0), (20, 5), (30, 0), (40, 5)]), + (2, 20, vec![(10, 0), (20, 10), (30, 0), (40, 10)]), + (3, 30, vec![(10, 0), (30, 0)]), + ], + ); + + // fyi. this is not PJR, obviously because the votes of 3 can bump the stake a lot but they + // are being ignored. + assert!(!pjr_check_core(&candidates, &voters, 1)); + assert!(!pjr_check_core(&candidates, &voters, 10)); + assert!(!pjr_check_core(&candidates, &voters, 20)); + } +} diff --git a/primitives/npos-elections/src/tests.rs b/primitives/npos-elections/src/tests.rs index 8e99d2222e885..d1b88c357f470 100644 --- a/primitives/npos-elections/src/tests.rs +++ b/primitives/npos-elections/src/tests.rs @@ -19,8 +19,9 @@ use crate::mock::*; use crate::{ - seq_phragmen, balance_solution, build_support_map, is_score_better, helpers::*, - Support, StakedAssignment, Assignment, ElectionResult, ExtendedBalance, + seq_phragmen, balancing, build_support_map, is_score_better, helpers::*, + Support, StakedAssignment, Assignment, ElectionResult, ExtendedBalance, setup_inputs, + seq_phragmen_core, }; use substrate_test_utils::assert_eq_uvec; use sp_arithmetic::{Perbill, Permill, Percent, PerU16}; @@ -34,7 +35,7 @@ fn float_phragmen_poc_works() { (30, vec![2, 3]), ]; let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30), (1, 0), (2, 0), (3, 0)]); - let mut phragmen_result = elect_float(2, 2, candidates, voters, &stake_of).unwrap(); + let mut phragmen_result = elect_float(2, candidates, voters, &stake_of).unwrap(); let winners = phragmen_result.clone().winners; let assignments = phragmen_result.clone().assignments; @@ -71,6 +72,107 @@ fn float_phragmen_poc_works() { ); } +#[test] +fn phragmen_core_poc_works() { + let candidates = vec![1, 2, 3]; + let voters = vec![ + (10, 10, vec![1, 2]), + (20, 20, vec![1, 3]), + (30, 30, vec![2, 3]), + ]; + + let (candidates, voters) = setup_inputs(candidates, voters); + let (candidates, voters) = seq_phragmen_core(2, candidates, voters).unwrap(); + + assert_eq!( + voters + .iter() + .map(|v| ( + v.who, + v.budget, + (v.edges.iter().map(|e| (e.who, e.weight)).collect::>()), + )) + .collect::>(), + vec![ + (10, 10, vec![(2, 10)]), + (20, 20, vec![(3, 20)]), + (30, 30, vec![(2, 15), (3, 15)]), + ] + ); + + assert_eq!( + candidates + .iter() + .map(|c_ptr| ( + c_ptr.borrow().who, + c_ptr.borrow().elected, + c_ptr.borrow().round, + c_ptr.borrow().backed_stake, + )).collect::>(), + vec![ + (1, false, 0, 0), + (2, true, 1, 25), + (3, true, 0, 35), + ] + ); +} + +#[test] +fn balancing_core_works() { + let candidates = vec![1, 2, 3, 4, 5]; + let voters = vec![ + (10, 10, vec![1, 2]), + (20, 20, vec![1, 3]), + (30, 30, vec![1, 2, 3, 4]), + (40, 40, vec![1, 3, 4, 5]), + (50, 50, vec![2, 4, 5]), + ]; + + let (candidates, voters) = setup_inputs(candidates, voters); + let (candidates, mut voters) = seq_phragmen_core(4, candidates, voters).unwrap(); + let iters = balancing::balance::(&mut voters, 4, 0); + + assert!(iters > 0); + + assert_eq!( + voters + .iter() + .map(|v| ( + v.who, + v.budget, + (v.edges.iter().map(|e| (e.who, e.weight)).collect::>()), + )) + .collect::>(), + vec![ + // note the 0 edge. This is know and not an issue per se. Also note that the stakes are + // not normalized. + (10, 10, vec![(1, 9), (2, 0)]), + (20, 20, vec![(1, 8), (3, 11)]), + (30, 30, vec![(1, 9), (2, 6), (3, 8), (4, 7)]), + (40, 40, vec![(1, 11), (3, 18), (4, 11)]), + (50, 50, vec![(2, 31), (4, 19)]), + ] + ); + + assert_eq!( + candidates + .iter() + .map(|c_ptr| ( + c_ptr.borrow().who, + c_ptr.borrow().elected, + c_ptr.borrow().round, + c_ptr.borrow().backed_stake, + )).collect::>(), + vec![ + (1, true, 1, 37), + (2, true, 2, 37), + (3, true, 3, 37), + (4, true, 0, 37), + (5, false, 0, 0), + ] + ); +} + #[test] fn phragmen_poc_works() { let candidates = vec![1, 2, 3]; @@ -82,13 +184,13 @@ fn phragmen_poc_works() { let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30)]); let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( - 2, 2, candidates, voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None, ).unwrap(); - assert_eq_uvec!(winners, vec![(2, 40), (3, 50)]); + assert_eq_uvec!(winners, vec![(2, 25), (3, 35)]); assert_eq_uvec!( assignments, vec![ @@ -110,9 +212,9 @@ fn phragmen_poc_works() { ] ); - let mut staked = assignment_ratio_to_staked(assignments, &stake_of); + let staked = assignment_ratio_to_staked(assignments, &stake_of); let winners = to_without_backing(winners); - let mut support_map = build_support_map::(&winners, &staked).0; + let support_map = build_support_map::(&winners, &staked).0; assert_eq_uvec!( staked, @@ -143,14 +245,51 @@ fn phragmen_poc_works() { *support_map.get(&3).unwrap(), Support:: { total: 35, voters: vec![(20, 20), (30, 15)] }, ); +} + +#[test] +fn phragmen_poc_works_with_balancing() { + let candidates = vec![1, 2, 3]; + let voters = vec![ + (10, vec![1, 2]), + (20, vec![1, 3]), + (30, vec![2, 3]), + ]; - balance_solution( - &mut staked, - &mut support_map, - 0, + let stake_of = create_stake_of(&[(10, 10), (20, 20), (30, 30)]); + let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( 2, + candidates, + voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + Some((4, 0)), + ).unwrap(); + + assert_eq_uvec!(winners, vec![(2, 30), (3, 30)]); + assert_eq_uvec!( + assignments, + vec![ + Assignment { + who: 10u64, + distribution: vec![(2, Perbill::from_percent(100))], + }, + Assignment { + who: 20, + distribution: vec![(3, Perbill::from_percent(100))], + }, + Assignment { + who: 30, + distribution: vec![ + (2, Perbill::from_parts(666666666)), + (3, Perbill::from_parts(333333334)), + ], + }, + ] ); + let staked = assignment_ratio_to_staked(assignments, &stake_of); + let winners = to_without_backing(winners); + let support_map = build_support_map::(&winners, &staked).0; + assert_eq_uvec!( staked, vec![ @@ -182,6 +321,7 @@ fn phragmen_poc_works() { ); } + #[test] fn phragmen_poc_2_works() { let candidates = vec![10, 20, 30]; @@ -198,10 +338,10 @@ fn phragmen_poc_2_works() { (4, 500), ]); - run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); - run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); - run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); - run_and_compare::(candidates, voters, &stake_of, 2, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2); + run_and_compare::(candidates, voters, &stake_of, 2); } #[test] @@ -219,14 +359,14 @@ fn phragmen_poc_3_works() { (4, 1000), ]); - run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); - run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); - run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2, 2); - run_and_compare::(candidates, voters, &stake_of, 2, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2); + run_and_compare::(candidates.clone(), voters.clone(), &stake_of, 2); + run_and_compare::(candidates, voters, &stake_of, 2); } #[test] -fn phragmen_accuracy_on_large_scale_only_validators() { +fn phragmen_accuracy_on_large_scale_only_candidates() { // because of this particular situation we had per_u128 and now rational128. In practice, a // candidate can have the maximum amount of tokens, and also supported by the maximum. let candidates = vec![1, 2, 3, 4, 5]; @@ -239,13 +379,13 @@ fn phragmen_accuracy_on_large_scale_only_validators() { ]); let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( - 2, 2, candidates.clone(), auto_generate_self_voters(&candidates) .iter() .map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())) .collect::>(), + None, ).unwrap(); assert_eq_uvec!(winners, vec![(1, 18446744073709551614u128), (5, 18446744073709551613u128)]); @@ -254,7 +394,7 @@ fn phragmen_accuracy_on_large_scale_only_validators() { } #[test] -fn phragmen_accuracy_on_large_scale_validators_and_nominators() { +fn phragmen_accuracy_on_large_scale_voters_and_candidates() { let candidates = vec![1, 2, 3, 4, 5]; let mut voters = vec![ (13, vec![1, 3, 5]), @@ -272,13 +412,14 @@ fn phragmen_accuracy_on_large_scale_validators_and_nominators() { ]); let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( - 2, 2, candidates, voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None, ).unwrap(); assert_eq_uvec!(winners, vec![(2, 36893488147419103226u128), (1, 36893488147419103219u128)]); + assert_eq!( assignments, vec![ @@ -300,6 +441,7 @@ fn phragmen_accuracy_on_large_scale_validators_and_nominators() { }, ] ); + check_assignments_sum(assignments); } @@ -314,14 +456,15 @@ fn phragmen_accuracy_on_small_scale_self_vote() { (30, 1), ]); - let ElectionResult { winners, assignments: _ } = seq_phragmen::<_, Perbill>( - 3, + let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( 3, candidates, voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None, ).unwrap(); assert_eq_uvec!(winners, vec![(20, 2), (10, 1), (30, 1)]); + check_assignments_sum(assignments); } #[test] @@ -344,14 +487,16 @@ fn phragmen_accuracy_on_small_scale_no_self_vote() { (3, 1), ]); - let ElectionResult { winners, assignments: _ } = seq_phragmen::<_, Perbill>( - 3, + let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( 3, candidates, voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None, ).unwrap(); assert_eq_uvec!(winners, vec![(20, 2), (10, 1), (30, 1)]); + check_assignments_sum(assignments); + } #[test] @@ -378,13 +523,13 @@ fn phragmen_large_scale_test() { ]); let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( - 2, 2, candidates, voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None, ).unwrap(); - assert_eq_uvec!(winners, vec![(24, 1490000000000200000u128), (22, 1490000000000100000u128)]); + assert_eq_uvec!(to_without_backing(winners.clone()), vec![24, 22]); check_assignments_sum(assignments); } @@ -404,21 +549,22 @@ fn phragmen_large_scale_test_2() { ]); let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( - 2, 2, candidates, voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None, ).unwrap(); - assert_eq_uvec!(winners, vec![(2, 1000000000004000000u128), (4, 1000000000004000000u128)]); - assert_eq!( + assert_eq_uvec!(winners, vec![(2, 500000000005000000u128), (4, 500000000003000000)]); + + assert_eq_uvec!( assignments, vec![ Assignment { who: 50u64, distribution: vec![ - (2, Perbill::from_parts(500000001)), - (4, Perbill::from_parts(499999999)) + (2, Perbill::from_parts(500000000)), + (4, Perbill::from_parts(500000000)), ], }, Assignment { @@ -431,6 +577,7 @@ fn phragmen_large_scale_test_2() { }, ], ); + check_assignments_sum(assignments); } @@ -464,7 +611,7 @@ fn phragmen_linear_equalize() { (130, 1000), ]); - run_and_compare::(candidates, voters, &stake_of, 2, 2); + run_and_compare::(candidates, voters, &stake_of, 2); } #[test] @@ -480,10 +627,10 @@ fn elect_has_no_entry_barrier() { ]); let ElectionResult { winners, assignments: _ } = seq_phragmen::<_, Perbill>( - 3, 3, candidates, voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None, ).unwrap(); // 30 is elected with stake 0. The caller is responsible for stripping this. @@ -495,29 +642,7 @@ fn elect_has_no_entry_barrier() { } #[test] -fn minimum_to_elect_is_respected() { - let candidates = vec![10, 20, 30]; - let voters = vec![ - (1, vec![10]), - (2, vec![20]), - ]; - let stake_of = create_stake_of(&[ - (1, 10), - (2, 10), - ]); - - let maybe_result = seq_phragmen::<_, Perbill>( - 10, - 10, - candidates, - voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), - ); - - assert!(maybe_result.is_none()); -} - -#[test] -fn self_votes_should_be_kept() { +fn phragmen_self_votes_should_be_kept() { let candidates = vec![5, 10, 20, 30]; let voters = vec![ (5, vec![5]), @@ -533,33 +658,29 @@ fn self_votes_should_be_kept() { ]); let result = seq_phragmen::<_, Perbill>( - 2, 2, candidates, voters.iter().map(|(ref v, ref vs)| (v.clone(), stake_of(v), vs.clone())).collect::>(), + None, ).unwrap(); - assert_eq!(result.winners, vec![(20, 28), (10, 18)]); - assert_eq!( + assert_eq!(result.winners, vec![(20, 24), (10, 14)]); + assert_eq_uvec!( result.assignments, vec![ - Assignment { who: 10, distribution: vec![(10, Perbill::from_percent(100))] }, - Assignment { who: 20, distribution: vec![(20, Perbill::from_percent(100))] }, Assignment { who: 1, distribution: vec![ (10, Perbill::from_percent(50)), - (20, Perbill::from_percent(50)) + (20, Perbill::from_percent(50)), ] }, - ], + Assignment { who: 10, distribution: vec![(10, Perbill::from_percent(100))] }, + Assignment { who: 20, distribution: vec![(20, Perbill::from_percent(100))] }, + ] ); - let mut staked_assignments = assignment_ratio_to_staked(result.assignments, &stake_of); + let staked_assignments = assignment_ratio_to_staked(result.assignments, &stake_of); let winners = to_without_backing(result.winners); - - let (mut supports, _) = build_support_map::( - &winners, - &staked_assignments, - ); + let (supports, _) = build_support_map::(&winners, &staked_assignments); assert_eq!(supports.get(&5u64), None); assert_eq!( @@ -570,22 +691,6 @@ fn self_votes_should_be_kept() { supports.get(&20u64).unwrap(), &Support { total: 24u128, voters: vec![(20u64, 20u128), (1u64, 4u128)] }, ); - - balance_solution( - &mut staked_assignments, - &mut supports, - 0, - 2usize, - ); - - assert_eq!( - supports.get(&10u64).unwrap(), - &Support { total: 18u128, voters: vec![(10u64, 10u128), (1u64, 8u128)] }, - ); - assert_eq!( - supports.get(&20u64).unwrap(), - &Support { total: 20u128, voters: vec![(20u64, 20u128)] }, - ); } #[test] @@ -598,10 +703,10 @@ fn duplicate_target_is_ignored() { ]; let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( - 2, 2, candidates, voters, + None, ).unwrap(); let winners = to_without_backing(winners); @@ -628,10 +733,10 @@ fn duplicate_target_is_ignored_when_winner() { ]; let ElectionResult { winners, assignments } = seq_phragmen::<_, Perbill>( - 2, 2, candidates, voters, + None, ).unwrap(); let winners = to_without_backing(winners); @@ -979,7 +1084,6 @@ mod solution_type { compact.encode().len() }; - dbg!(with_compact, without_compact); assert!(with_compact < without_compact); } diff --git a/primitives/runtime/src/lib.rs b/primitives/runtime/src/lib.rs index eb8bbb38a6ffe..c1eb37506958b 100644 --- a/primitives/runtime/src/lib.rs +++ b/primitives/runtime/src/lib.rs @@ -71,8 +71,9 @@ pub use sp_core::RuntimeDebug; /// Re-export top-level arithmetic stuff. pub use sp_arithmetic::{ - PerThing, traits::SaturatedConversion, Perquintill, Perbill, Permill, Percent, PerU16, InnerOf, + PerThing, Perquintill, Perbill, Permill, Percent, PerU16, InnerOf, UpperOf, Rational128, FixedI64, FixedI128, FixedU128, FixedPointNumber, FixedPointOperand, + traits::SaturatedConversion, }; /// Re-export 128 bit helpers. pub use sp_arithmetic::helpers_128bit; diff --git a/primitives/sandbox/Cargo.toml b/primitives/sandbox/Cargo.toml index 98376c77464d5..4d5fc43005cd1 100755 --- a/primitives/sandbox/Cargo.toml +++ b/primitives/sandbox/Cargo.toml @@ -20,7 +20,7 @@ sp-wasm-interface = { version = "2.0.0-rc6", default-features = false, path = ". codec = { package = "parity-scale-codec", version = "1.3.1", default-features = false } [dev-dependencies] -wabt = "0.9.2" +wabt = "0.10.0" assert_matches = "1.3.0" [features]