Skip to content

Reorganize partitioned epoch rewards runtime code, 1 of 5 #511

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
CriesofCarrots merged 5 commits into from
Apr 2, 2024
Merged

Reorganize partitioned epoch rewards runtime code, 1 of 5 #511

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
352bff0
Add bank::partitioned_epoch_rewards submodule
CriesofCarrots Mar 29, 2024
f59b7b0
Move helper structs and types to submodule
CriesofCarrots Mar 29, 2024
19d4fc4
Move partitioned-rewards-specific Bank methods to submodule
CriesofCarrots Mar 29, 2024
42d0382
Move unit tests into submodule
CriesofCarrots Mar 29, 2024
4d668bb
Update BankAbiTestWrapperNewer frozen_abi hash
CriesofCarrots Mar 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 7 additions & 167 deletions runtime/src/bank.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,11 @@ use {
bank::{
builtins::{BuiltinPrototype, BUILTINS},
metrics::*,
partitioned_epoch_rewards::{
CalculateRewardsAndDistributeVoteRewardsResult, EpochRewardCalculateParamInfo,
EpochRewardStatus, PartitionedRewardsCalculation, RewardInterval,
StakeRewardCalculationPartitioned, StakeRewards, VoteRewardsAccounts,
},
},
bank_forks::BankForks,
epoch_rewards_hasher::hash_rewards_into_partitions,
Expand Down Expand Up @@ -83,7 +88,6 @@ use {
ancestors::{Ancestors, AncestorsForSerialization},
blockhash_queue::BlockhashQueue,
epoch_accounts_hash::EpochAccountsHash,
partitioned_rewards::PartitionedEpochRewardsConfig,
sorted_storages::SortedStorages,
stake_rewards::StakeReward,
storable_accounts::StorableAccounts,
Expand Down Expand Up @@ -173,7 +177,7 @@ use {
},
},
solana_system_program::{get_system_account_kind, SystemAccountKind},
solana_vote::vote_account::{VoteAccount, VoteAccounts, VoteAccountsHashMap},
solana_vote::vote_account::{VoteAccount, VoteAccountsHashMap},
solana_vote_program::vote_state::VoteState,
std::{
borrow::Cow,
Expand Down Expand Up @@ -220,6 +224,7 @@ pub mod builtins;
pub mod epoch_accounts_hash_utils;
mod fee_distribution;
mod metrics;
pub(crate) mod partitioned_epoch_rewards;
mod serde_snapshot;
mod sysvar_cache;
#[cfg(test)]
Expand Down Expand Up @@ -642,27 +647,6 @@ impl AbiExample for OptionalDropCallback {
}
}

#[derive(AbiExample, Debug, Clone, PartialEq, Serialize, Deserialize)]
pub(crate) struct StartBlockHeightAndRewards {
/// the block height of the slot at which rewards distribution began
pub(crate) start_block_height: u64,
/// calculated epoch rewards pending distribution, outer Vec is by partition (one partition per block)
pub(crate) stake_rewards_by_partition: Arc<Vec<StakeRewards>>,
}

/// Represent whether bank is in the reward phase or not.
#[derive(AbiExample, AbiEnumVisitor, Debug, Clone, PartialEq, Serialize, Deserialize, Default)]
pub(crate) enum EpochRewardStatus {
/// this bank is in the reward phase.
/// Contents are the start point for epoch reward calculation,
/// i.e. parent_slot and parent_block height for the starting
/// block of the current epoch.
Active(StartBlockHeightAndRewards),
/// this bank is outside of the rewarding phase.
#[default]
Inactive,
}

/// Manager for the state of all accounts and programs after processing its entries.
/// AbiExample is needed even without Serialize/Deserialize; actual (de-)serialization
/// are implemented elsewhere for versioning
Expand Down Expand Up @@ -865,56 +849,6 @@ struct VoteReward {
}

type VoteRewards = DashMap<Pubkey, VoteReward>;
#[derive(Debug, Default)]
struct VoteRewardsAccounts {
/// reward info for each vote account pubkey.
/// This type is used by `update_reward_history()`
rewards: Vec<(Pubkey, RewardInfo)>,
/// corresponds to pubkey in `rewards`
/// Some if account is to be stored.
/// None if to be skipped.
accounts_to_store: Vec<Option<AccountSharedData>>,
}

/// hold reward calc info to avoid recalculation across functions
struct EpochRewardCalculateParamInfo<'a> {
stake_history: StakeHistory,
stake_delegations: Vec<(&'a Pubkey, &'a StakeAccount<Delegation>)>,
cached_vote_accounts: &'a VoteAccounts,
}

/// Hold all results from calculating the rewards for partitioned distribution.
/// This struct exists so we can have a function which does all the calculation with no
/// side effects.
struct PartitionedRewardsCalculation {
vote_account_rewards: VoteRewardsAccounts,
stake_rewards_by_partition: StakeRewardCalculationPartitioned,
old_vote_balance_and_staked: u64,
validator_rewards: u64,
validator_rate: f64,
foundation_rate: f64,
prev_epoch_duration_in_years: f64,
capitalization: u64,
}

/// result of calculating the stake rewards at beginning of new epoch
struct StakeRewardCalculationPartitioned {
/// each individual stake account to reward, grouped by partition
stake_rewards_by_partition: Vec<StakeRewards>,
/// total lamports across all `stake_rewards`
total_stake_rewards_lamports: u64,
}

struct CalculateRewardsAndDistributeVoteRewardsResult {
/// total rewards for the epoch (including both vote rewards and stake rewards)
total_rewards: u64,
/// distributed vote rewards
distributed_rewards: u64,
/// stake rewards that still need to be distributed, grouped by partition
stake_rewards_by_partition: Vec<StakeRewards>,
}

pub(crate) type StakeRewards = Vec<StakeReward>;

#[derive(Debug, Default)]
pub struct NewBankOptions {
Expand Down Expand Up @@ -950,14 +884,6 @@ struct StakeRewardCalculation {
total_stake_rewards_lamports: u64,
}

#[derive(Debug, PartialEq, Eq, Copy, Clone)]
pub(super) enum RewardInterval {
/// the slot within the epoch is INSIDE the reward distribution interval
InsideInterval,
/// the slot within the epoch is OUTSIDE the reward distribution interval
OutsideInterval,
}

impl Bank {
fn default_with_accounts(accounts: Accounts) -> Self {
let mut bank = Self {
Expand Down Expand Up @@ -1144,76 +1070,6 @@ impl Bank {
rent_collector.clone_with_epoch(epoch)
}

fn is_partitioned_rewards_feature_enabled(&self) -> bool {
self.feature_set
.is_active(&feature_set::enable_partitioned_epoch_reward::id())
}

pub(crate) fn set_epoch_reward_status_active(
&mut self,
stake_rewards_by_partition: Vec<StakeRewards>,
) {
self.epoch_reward_status = EpochRewardStatus::Active(StartBlockHeightAndRewards {
start_block_height: self.block_height,
stake_rewards_by_partition: Arc::new(stake_rewards_by_partition),
});
}

fn partitioned_epoch_rewards_config(&self) -> &PartitionedEpochRewardsConfig {
&self
.rc
.accounts
.accounts_db
.partitioned_epoch_rewards_config
}

/// # stake accounts to store in one block during partitioned reward interval
fn partitioned_rewards_stake_account_stores_per_block(&self) -> u64 {
self.partitioned_epoch_rewards_config()
.stake_account_stores_per_block
}

/// reward calculation happens synchronously during the first block of the epoch boundary.
/// So, # blocks for reward calculation is 1.
fn get_reward_calculation_num_blocks(&self) -> Slot {
self.partitioned_epoch_rewards_config()
.reward_calculation_num_blocks
}

/// Calculate the number of blocks required to distribute rewards to all stake accounts.
fn get_reward_distribution_num_blocks(&self, rewards: &StakeRewards) -> u64 {
let total_stake_accounts = rewards.len();
if self.epoch_schedule.warmup && self.epoch < self.first_normal_epoch() {
1
} else {
const MAX_FACTOR_OF_REWARD_BLOCKS_IN_EPOCH: u64 = 10;
let num_chunks = solana_accounts_db::accounts_hash::AccountsHasher::div_ceil(
total_stake_accounts,
self.partitioned_rewards_stake_account_stores_per_block() as usize,
) as u64;

// Limit the reward credit interval to 10% of the total number of slots in a epoch
num_chunks.clamp(
1,
(self.epoch_schedule.slots_per_epoch / MAX_FACTOR_OF_REWARD_BLOCKS_IN_EPOCH).max(1),
)
}
}

/// Return `RewardInterval` enum for current bank
fn get_reward_interval(&self) -> RewardInterval {
if matches!(self.epoch_reward_status, EpochRewardStatus::Active(_)) {
RewardInterval::InsideInterval
} else {
RewardInterval::OutsideInterval
}
}

/// For testing only
pub fn force_reward_interval_end_for_tests(&mut self) {
self.epoch_reward_status = EpochRewardStatus::Inactive;
}

fn _new_from_parent(
parent: Arc<Bank>,
collector_id: &Pubkey,
Expand Down Expand Up @@ -1642,13 +1498,6 @@ impl Bank {
}
}

fn force_partition_rewards_in_first_block_of_epoch(&self) -> bool {
self.partitioned_epoch_rewards_config()
.test_enable_partitioned_rewards
&& self.get_reward_calculation_num_blocks() == 0
&& self.partitioned_rewards_stake_account_stores_per_block() == u64::MAX
}

/// Begin the process of calculating and distributing rewards.
/// This process can take multiple slots.
fn begin_partitioned_rewards(
Expand Down Expand Up @@ -3591,15 +3440,6 @@ impl Bank {
report_partitioned_reward_metrics(self, metrics);
}

/// true if it is ok to run partitioned rewards code.
/// This means the feature is activated or certain testing situations.
fn is_partitioned_rewards_code_enabled(&self) -> bool {
self.is_partitioned_rewards_feature_enabled()
|| self
.partitioned_epoch_rewards_config()
.test_enable_partitioned_rewards
}

/// Helper fn to log epoch_rewards sysvar
fn log_epoch_rewards_sysvar(&self, prefix: &str) {
if let Some(account) = self.get_account(&sysvar::epoch_rewards::id()) {
Expand Down
Loading