Prototype: accumulate hash of all accounts

accounts-db/src/accounts_db.rs

@@ -7916,10 +7916,40 @@ impl AccountsDb {
     ///
     /// As part of calculating the accounts delta hash, get a list of accounts modified this slot
     /// (aka dirty pubkeys) and add them to `self.uncleaned_pubkeys` for future cleaning.
-    pub fn calculate_accounts_delta_hash(&self, slot: Slot) -> AccountsDeltaHash {
+    pub fn calculate_accounts_delta_hash(&self, slot: Slot) -> (AccountsDeltaHash, Vec<(Pubkey, AccountHash)>) {
         self.calculate_accounts_delta_hash_internal(slot, None, HashMap::default())
     }
 
+    pub fn accumulate_accounts_hash(
+        &self,
+        slot: Slot,
+        mut ancestors: Ancestors,
+        accumulated_accounts_hash: &mut Hash,
+        pubkey_hash: Vec<(Pubkey, AccountHash)>,
+        old_written_accounts: &RwLock<HashMap<Pubkey, (Option<AccountSharedData>, Option<AccountHash>)>>
+    ) {
+        // we are assuming it was easy to lookup a hash for everything written in `slot` when we were calculating the delta hash. So, caller passes in `pubkey_hash`
+        // note we don't need rewrites in `pubkey_hash`. these accounts had the same hash before and after. So, we only have to consider what was written that changed.
+        ancestors.remove(&slot);
+        let old_written_accounts = old_written_accounts.read().unwrap();
+        // if we want to look it up ourselves: let (hashes, _scan_us, _accumulate) = self.get_pubkey_hash_for_slot(slot);
+        let old = pubkey_hash.iter().map(|(k, _)| {
+            if let Some((account, hash)) = old_written_accounts.get(k) {
+                Some(hash.unwrap()) // todo on demand calculate, calculate in bg
+            }
+            else {
+                self.load_with_fixed_root(&ancestors, k).map(|(account, _)| Self::hash_account(&account, k))
+            }
+        }).collect::<Vec<_>>();
+        pubkey_hash.into_iter().zip(old.into_iter()).for_each(|((k, new_hash), old_hash)| {
+            if let Some(old) = old_hash {
+                // todo if old == new, then we can avoid this update altogether
+                // todo subtract accumulated_accounts_hash -= old_hash
+            }
+            // todo add accumulated_accounts_hash += new_hash
+        });
+    }
+
     /// Calculate accounts delta hash for `slot`
     ///
     /// As part of calculating the accounts delta hash, get a list of accounts modified this slot
@@ -7929,8 +7959,9 @@ impl AccountsDb {
         slot: Slot,
         ignore: Option<Pubkey>,
         mut skipped_rewrites: HashMap<Pubkey, AccountHash>,
-    ) -> AccountsDeltaHash {
+    ) -> (AccountsDeltaHash, Vec<(Pubkey, AccountHash)>) {
         let (mut hashes, scan_us, mut accumulate) = self.get_pubkey_hash_for_slot(slot);
+        let original_pubkey_hash = hashes.clone();
         let dirty_keys = hashes.iter().map(|(pubkey, _hash)| *pubkey).collect();
 
         hashes.iter().for_each(|(k, _h)| {
@@ -7969,7 +8000,7 @@ impl AccountsDb {
             .skipped_rewrites_num
             .fetch_add(num_skipped_rewrites, Ordering::Relaxed);
 
-        accounts_delta_hash
+        (accounts_delta_hash, original_pubkey_hash)
     }
 
     /// Set the accounts delta hash for `slot` in the `accounts_delta_hashes` map

accounts-db/src/accounts_hash.rs

@@ -1162,6 +1162,8 @@ impl<'a> AccountsHasher<'a> {
                 overall_sum = Self::checked_cast_for_capitalization(
                     item.lamports as u128 + overall_sum as u128,
                 );
+                // note that we DO have to dedup and avoid zero lamport hashes...
+                // todo: probably we could accumulate here instead of writing every hash here and accumulating each hash later (in a map/fold reduce)
                 hashes.write(&item.hash.0);
             } else {
                 // if lamports == 0, check if they should be included
@@ -1170,6 +1172,7 @@ impl<'a> AccountsHasher<'a> {
                     // the hash of its pubkey
                     let hash = blake3::hash(bytemuck::bytes_of(&item.pubkey));
                     let hash = Hash::new_from_array(hash.into());
+                    // todo: same as above
                     hashes.write(&hash);
                 }
             }
@@ -1204,6 +1207,16 @@ impl<'a> AccountsHasher<'a> {
 
         let _guard = self.active_stats.activate(ActiveStatItem::HashMerkleTree);
         let mut hash_time = Measure::start("hash");
+        let mut accumulated = Hash::default();
+        let mut i = 0;
+        while i < cumulative.total_count() {
+            let slice = cumulative.get_slice(i);
+            slice.iter().for_each(|hash| {
+                // todo: accumulate here if we weren't able to do it earlier
+                // accumulated += hash
+            });
+            i += slice.len();
+        }
         let (hash, _) = Self::compute_merkle_root_from_slices(
             cumulative.total_count(),
             MERKLE_FANOUT,

runtime/src/accounts/mod.rs

@@ -46,6 +46,8 @@ use {
     solana_system_program::{get_system_account_kind, SystemAccountKind},
     std::{collections::HashMap, num::NonZeroUsize},
 };
+use solana_accounts_db::accounts_hash::AccountHash;
+use std::sync::RwLock;
 
 #[allow(clippy::too_many_arguments)]
 pub(super) fn load_accounts(
@@ -63,6 +65,7 @@ pub(super) fn load_accounts(
     program_accounts: &HashMap<Pubkey, (&Pubkey, u64)>,
     loaded_programs: &LoadedProgramsForTxBatch,
     should_collect_rent: bool,
+    old_written_accounts: &RwLock<HashMap<Pubkey, (Option<AccountSharedData>, Option<AccountHash>)>>,
 ) -> Vec<TransactionLoadResult> {
     txs.iter()
         .zip(lock_results)
@@ -106,6 +109,7 @@ pub(super) fn load_accounts(
                     program_accounts,
                     loaded_programs,
                     should_collect_rent,
+                    old_written_accounts,
                 ) {
                     Ok(loaded_transaction) => loaded_transaction,
                     Err(e) => return (Err(e), None),
@@ -147,6 +151,7 @@ fn load_transaction_accounts(
     program_accounts: &HashMap<Pubkey, (&Pubkey, u64)>,
     loaded_programs: &LoadedProgramsForTxBatch,
     should_collect_rent: bool,
+    old_written_accounts: &RwLock<HashMap<Pubkey, (Option<AccountSharedData>, Option<AccountHash>)>>,
 ) -> Result<LoadedTransaction> {
     let in_reward_interval = reward_interval == RewardInterval::InsideInterval;
 
@@ -215,6 +220,13 @@ fn load_transaction_accounts(
                         .load_with_fixed_root(ancestors, key)
                         .map(|(mut account, _)| {
                             if message.is_writable(i) {
+                                {
+                                    // todo: find all the places this has to happen
+                                    let mut old_written_accounts= old_written_accounts.write().unwrap();
+                                    if !old_written_accounts.contains_key(key) {
+                                        old_written_accounts.insert(*key, (Some(account.clone()), None));
+                                    }
+                                }
                                 if should_collect_rent {
                                     let rent_due = rent_collector
                                         .collect_from_existing_account(

runtime/src/bank.rs

@@ -511,6 +511,9 @@ impl PartialEq for Bank {
             return true;
         }
         let Self {
+            accumulated_accounts_hash: _,
+            old_written_accounts_from_last_slot: _,
+            old_written_accounts: _,
             skipped_rewrites: _,
             rc: _,
             status_cache: _,
@@ -833,6 +836,12 @@ pub struct Bank {
 
     pub check_program_modification_slot: bool,
 
+    pub accumulated_accounts_hash: RwLock<Option<Hash>>,
+    /// the pubkey, (account, hash) pairs that were loaded in this slot in order to be written. Ideally, when the bank is done, this contains a calculated hash value for all accounts which are written in THIS slot.
+    pub old_written_accounts: RwLock<HashMap<Pubkey, (Option<AccountSharedData>, Option<AccountHash>)>>,
+    /// the pubkey, hash pairs that were written by the last slot. MANY accounts are written EVERY slot. This avoids a re-hashing.
+    pub old_written_accounts_from_last_slot: RwLock<HashMap<Pubkey, (Option<AccountSharedData>, Option<AccountHash>)>>,
+
     epoch_reward_status: EpochRewardStatus,
 }
 
@@ -976,6 +985,9 @@ impl Bank {
 
     fn default_with_accounts(accounts: Accounts) -> Self {
         let mut bank = Self {
+            accumulated_accounts_hash: RwLock::default(),
+            old_written_accounts_from_last_slot: RwLock::default(),
+            old_written_accounts: RwLock::default(),
             skipped_rewrites: Mutex::default(),
             incremental_snapshot_persistence: None,
             rc: BankRc::new(accounts, Slot::default()),
@@ -1312,6 +1324,10 @@ impl Bank {
 
         let accounts_data_size_initial = parent.load_accounts_data_size();
         let mut new = Self {
+            accumulated_accounts_hash: RwLock::default(),
+            // start this slot's old written accounts with the accounts written in the last slot. many accounts are written every slot (like votes)
+            old_written_accounts: RwLock::new(std::mem::take(&mut parent.old_written_accounts_from_last_slot.write().unwrap())),
+            old_written_accounts_from_last_slot: RwLock::default(),
             skipped_rewrites: Mutex::default(),
             incremental_snapshot_persistence: None,
             rc,
@@ -1827,6 +1843,10 @@ impl Bank {
         );
         let stakes_accounts_load_duration = now.elapsed();
         let mut bank = Self {
+            old_written_accounts: RwLock::default(),
+            old_written_accounts_from_last_slot: RwLock::default(),
+            // todo: this has to be saved and loaded in bank persistence somehow
+            accumulated_accounts_hash: RwLock::default(),
             skipped_rewrites: Mutex::default(),
             incremental_snapshot_persistence: fields.incremental_snapshot_persistence,
             rc: bank_rc,
@@ -5215,6 +5235,7 @@ impl Bank {
             &program_accounts_map,
             &programs_loaded_for_tx_batch.borrow(),
             self.should_collect_rent(),
+            &self.old_written_accounts,
         );
         load_time.stop();
 
@@ -6984,7 +7005,7 @@ impl Bank {
                 && self.get_reward_calculation_num_blocks() == 0
                 && self.partitioned_rewards_stake_account_stores_per_block() == u64::MAX))
             .then_some(sysvar::epoch_rewards::id());
-        let accounts_delta_hash = self
+        let (accounts_delta_hash, pubkey_hash) = self
             .rc
             .accounts
             .accounts_db
@@ -6994,6 +7015,40 @@ impl Bank {
                 self.skipped_rewrites.lock().unwrap().clone(),
             );
 
+        // todo: note that this could be slow. My theory was we'd want to run this in the bg and have it ready later.
+        // It could be ready for the snapshot, accounts hash abs loop, publishing to gossip later, or even to include in the bank hash NEXT slot.
+        // having it ready THIS slot is a lot of work and adds loads into the dependency chain or makes every load more expensive (especially considering an account written multiple times in the same slot).
+        // We'd have to add infrastructure to keep hash values in the read only cache, for example.
+        // We could also build a HashMap<Pubkey, (AccountSharedData, RwLock<AccountHash>)> which is populated the FIRST time a writable account is loaded in this slot. a bg thread could hash the contents. Then, we would have the hash of each written account prior to the first load.
+        // We could even leave the RwLock<AccountsHash> out of it and just re-hash inside `accumulate_accounts_hash`. Depends on how much time loading from prior slots, cloning the account, hashing the account, and looking up the hash (in hashmap) take relative to each other.
+        // I think we want to avoid passing the hash value (or some type of Arc<RwLock<AccountHash>> all around to everyone at every load just for this feature). We need (impl ReadableAccount) + pubkey in order to calculate a hash, so we need something like &AccountSharedData, which we'd like to avoid cloning just so we can hash it later.
+        // All of this concern can get mitigated if we just wait a single slot and do this hash in the bg.
+
+
+        {
+            // every account that was written and hashed in this slot is likely going to be written again in the next slot. So, remember the pubkey and hash values as of the end of this slot.
+            // This will become the initial expected (pubkey, hash) pairs needed to subtract out these hashes from the accumulated hash next time.
+            let mut old_written_accounts_from_last_slot = self.old_written_accounts_from_last_slot.write().unwrap();
+            pubkey_hash.iter().for_each(|(k, h)| {
+                old_written_accounts_from_last_slot.insert(*k, (None, Some(*h)));
+            })
+        }
+
+        let mut accumulated = self.parent().map(|bank| bank.accumulated_accounts_hash.read().unwrap().unwrap_or_default()).unwrap_or_default(); // todo probably not default here
+        self
+            .rc
+            .accounts
+            .accounts_db
+            .accumulate_accounts_hash(
+                slot,
+                self.ancestors.clone(),
+                &mut accumulated,
+                pubkey_hash,
+                &self.old_written_accounts,
+            );
+
+        *self.accumulated_accounts_hash.write().unwrap() = Some(accumulated);
+
         let mut signature_count_buf = [0u8; 8];
         LittleEndian::write_u64(&mut signature_count_buf[..], self.signature_count());