Skip to content

Introduce AccountsFileProvider #334

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
yhchiang-sol wants to merge 1 commit into from
Closed

Introduce AccountsFileProvider #334

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions accounts-db/src/account_storage.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ impl Default for AccountStorageStatus {

#[cfg(test)]
pub(crate) mod tests {
use {super::*, std::path::Path};
use {super::*, crate::accounts_file::AppendVecProvider, std::path::Path};

#[test]
fn test_shrink_in_progress() {
Expand All @@ -292,13 +292,13 @@ pub(crate) mod tests {
let store_file_size = 4000;
let store_file_size2 = store_file_size * 2;
// 2 append vecs with same id, but different sizes
let entry = Arc::new(AccountStorageEntry::new(
let entry = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot,
id,
store_file_size,
));
let entry2 = Arc::new(AccountStorageEntry::new(
let entry2 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot,
id,
Expand Down Expand Up @@ -348,7 +348,7 @@ pub(crate) mod tests {
// add a map store
let common_store_path = Path::new("");
let store_file_size = 4000;
Arc::new(AccountStorageEntry::new(
Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot,
id,
Expand Down
58 changes: 33 additions & 25 deletions accounts-db/src/accounts_db.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ use {
},
accounts_cache::{AccountsCache, CachedAccount, SlotCache},
accounts_file::{
AccountsFile, AccountsFileError, MatchAccountOwnerError, ALIGN_BOUNDARY_OFFSET,
AccountsFile, AccountsFileError, AccountsFileProvider, AppendVecProvider,
MatchAccountOwnerError, ALIGN_BOUNDARY_OFFSET,
},
accounts_hash::{
AccountHash, AccountsDeltaHash, AccountsHash, AccountsHashKind, AccountsHasher,
Expand All @@ -55,9 +56,7 @@ use {
ancient_append_vecs::{
get_ancient_append_vec_capacity, is_ancient, AccountsToStore, StorageSelector,
},
append_vec::{
aligned_stored_size, AppendVec, APPEND_VEC_MMAPPED_FILES_OPEN, STORE_META_OVERHEAD,
},
append_vec::{aligned_stored_size, APPEND_VEC_MMAPPED_FILES_OPEN, STORE_META_OVERHEAD},
cache_hash_data::{
CacheHashData, CacheHashDataFileReference, DeletionPolicy as CacheHashDeletionPolicy,
},
Expand Down Expand Up @@ -1031,10 +1030,15 @@ pub struct AccountStorageEntry {
}

impl AccountStorageEntry {
pub fn new(path: &Path, slot: Slot, id: AccountsFileId, file_size: u64) -> Self {
pub fn new<AFP: AccountsFileProvider>(
path: &Path,
slot: Slot,
id: AccountsFileId,
file_size: u64,
) -> Self {
let tail = AccountsFile::file_name(slot, id);
let path = Path::new(path).join(tail);
let accounts = AccountsFile::AppendVec(AppendVec::new(path, true, file_size as usize));
let accounts = AFP::new_writable(path, file_size);

Self {
id,
Expand Down Expand Up @@ -2514,7 +2518,7 @@ impl AccountsDb {
}

fn new_storage_entry(&self, slot: Slot, path: &Path, size: u64) -> AccountStorageEntry {
AccountStorageEntry::new(path, slot, self.next_id(), size)
AccountStorageEntry::new::<AppendVecProvider>(path, slot, self.next_id(), size)
}

pub fn expected_cluster_type(&self) -> ClusterType {
Expand Down Expand Up @@ -6330,7 +6334,7 @@ impl AccountsDb {
slot_stores: &HashMap<AccountsFileId, Arc<AccountStorageEntry>>,
) -> Arc<AccountStorageEntry> {
let size = slot_stores.values().map(|storage| storage.capacity()).sum();
let storage = AccountStorageEntry::new(path, slot, id, size);
let storage = AccountStorageEntry::new::<AppendVecProvider>(path, slot, id, size);

// get unique accounts, most recent version by write_version
let mut accum = HashMap::<Pubkey, StoredAccountMeta<'_>>::default();
Expand Down Expand Up @@ -9508,7 +9512,7 @@ pub mod tests {
accounts_hash::MERKLE_FANOUT,
accounts_index::{tests::*, AccountSecondaryIndexesIncludeExclude},
ancient_append_vecs,
append_vec::{test_utils::TempFile, AppendVecStoredAccountMeta},
append_vec::{test_utils::TempFile, AppendVec, AppendVecStoredAccountMeta},
cache_hash_data::CacheHashDataFile,
inline_spl_token,
},
Expand Down Expand Up @@ -10547,7 +10551,8 @@ pub mod tests {
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let slot_expected: Slot = 0;
let size: usize = 123;
let data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
let data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot_expected, 0, size as u64);

let arc = Arc::new(data);
let storages = vec![arc];
Expand Down Expand Up @@ -10598,7 +10603,8 @@ pub mod tests {
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let slot_expected: Slot = 0;
let size: usize = 123;
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
let mut data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot_expected, 0, size as u64);
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
data.accounts = av;

Expand Down Expand Up @@ -10714,7 +10720,8 @@ pub mod tests {
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let slot_expected: Slot = 0;
let size: usize = 123;
let mut data = AccountStorageEntry::new(&paths[0], slot_expected, 0, size as u64);
let mut data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot_expected, 0, size as u64);
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
data.accounts = av;

Expand Down Expand Up @@ -10793,7 +10800,8 @@ pub mod tests {
let (_temp_dirs, paths) = get_temp_accounts_paths(1).unwrap();
let file_size = account_data_size.unwrap_or(123) * 100 / fill_percentage;
let size_aligned: usize = aligned_stored_size(file_size as usize);
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size_aligned as u64);
let mut data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot, id, size_aligned as u64);
let av = AccountsFile::AppendVec(AppendVec::new(
&tf.path,
true,
Expand Down Expand Up @@ -12897,7 +12905,7 @@ pub mod tests {
let store_file_size = 2 * PAGE_SIZE;

let store1_id = 22;
let store1 = Arc::new(AccountStorageEntry::new(
let store1 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_1,
store1_id,
Expand All @@ -12910,7 +12918,7 @@ pub mod tests {
let slot_id_2 = 13;

let store2_id = 44;
let store2 = Arc::new(AccountStorageEntry::new(
let store2 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_2,
store2_id,
Expand All @@ -12926,7 +12934,7 @@ pub mod tests {

let slot_id_3 = 14;
let store3_id = 55;
let entry3 = Arc::new(AccountStorageEntry::new(
let entry3 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_3,
store3_id,
Expand Down Expand Up @@ -12970,7 +12978,7 @@ pub mod tests {
let store_file_size = 2 * PAGE_SIZE;

let store1_id = 22;
let store1 = Arc::new(AccountStorageEntry::new(
let store1 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_1,
store1_id,
Expand All @@ -12982,7 +12990,7 @@ pub mod tests {

let slot_id_2 = 13;
let store2_id = 44;
let store2 = Arc::new(AccountStorageEntry::new(
let store2 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_2,
store2_id,
Expand All @@ -12999,7 +13007,7 @@ pub mod tests {

let slot_id_3 = 14;
let store3_id = 55;
let entry3 = Arc::new(AccountStorageEntry::new(
let entry3 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot_id_3,
store3_id,
Expand Down Expand Up @@ -13036,7 +13044,7 @@ pub mod tests {

let store_file_size = 4 * PAGE_SIZE;
let store1_id = 22;
let store1 = Arc::new(AccountStorageEntry::new(
let store1 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot1,
store1_id,
Expand All @@ -13054,7 +13062,7 @@ pub mod tests {

let store2_id = 44;
let slot2 = 44;
let store2 = Arc::new(AccountStorageEntry::new(
let store2 = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
slot2,
store2_id,
Expand Down Expand Up @@ -15066,11 +15074,11 @@ pub mod tests {
#[test]
fn test_shrink_productive() {
solana_logger::setup();
let s1 = AccountStorageEntry::new(Path::new("."), 0, 0, 1024);
let s1 = AccountStorageEntry::new::<AppendVecProvider>(Path::new("."), 0, 0, 1024);
let store = Arc::new(s1);
assert!(!AccountsDb::is_shrinking_productive(0, &store));

let s1 = AccountStorageEntry::new(Path::new("."), 0, 0, PAGE_SIZE * 4);
let s1 = AccountStorageEntry::new::<AppendVecProvider>(Path::new("."), 0, 0, PAGE_SIZE * 4);
let store = Arc::new(s1);
store.add_account((3 * PAGE_SIZE as usize) - 1);
store.add_account(10);
Expand All @@ -15088,7 +15096,7 @@ pub mod tests {
let mut accounts = AccountsDb::new_single_for_tests();
let common_store_path = Path::new("");
let store_file_size = 2 * PAGE_SIZE;
let entry = Arc::new(AccountStorageEntry::new(
let entry = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
0,
1,
Expand Down Expand Up @@ -17424,7 +17432,7 @@ pub mod tests {
solana_logger::setup();
let common_store_path = Path::new("");
let store_file_size = 2 * PAGE_SIZE;
let entry = Arc::new(AccountStorageEntry::new(
let entry = Arc::new(AccountStorageEntry::new::<AppendVecProvider>(
common_store_path,
0,
1,
Expand Down
23 changes: 23 additions & 0 deletions accounts-db/src/accounts_file.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,29 @@ impl<'a> Iterator for AccountsFileIter<'a> {
}
}

/// A trait that creates AccountsFile instance with the specified format.
pub trait AccountsFileProvider {
fn new_writable(path: impl Into<PathBuf>, file_size: u64) -> AccountsFile;
}

/// A struct that creates AccountsFile instance under AppendVec format.
#[derive(Debug)]
pub struct AppendVecProvider;
Comment thread
brooksprumo marked this conversation as resolved.
impl AccountsFileProvider for AppendVecProvider {
fn new_writable(path: impl Into<PathBuf>, file_size: u64) -> AccountsFile {
AccountsFile::AppendVec(AppendVec::new(path, true, file_size as usize))
}
}

/// A struct that creates AccountsFile instance under HotStorage format.
#[derive(Debug)]
pub struct HotStorageProvider;
impl AccountsFileProvider for HotStorageProvider {
fn new_writable(path: impl Into<PathBuf>, _file_size: u64) -> AccountsFile {
AccountsFile::TieredStorage(TieredStorage::new_writable(path))
}
}

#[cfg(test)]
pub mod tests {
use crate::accounts_file::AccountsFile;
Expand Down
5 changes: 3 additions & 2 deletions accounts-db/src/sorted_storages.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ mod tests {
super::*,
crate::{
accounts_db::{AccountStorageEntry, AccountsFileId},
accounts_file::AccountsFile,
accounts_file::{AccountsFile, AppendVecProvider},
append_vec::AppendVec,
},
std::sync::Arc,
Expand Down Expand Up @@ -445,7 +445,8 @@ mod tests {
let (_temp_dirs, paths) = crate::accounts_db::get_temp_accounts_paths(1).unwrap();
let size: usize = 123;
let slot = 0;
let mut data = AccountStorageEntry::new(&paths[0], slot, id, size as u64);
let mut data =
AccountStorageEntry::new::<AppendVecProvider>(&paths[0], slot, id, size as u64);
let av = AccountsFile::AppendVec(AppendVec::new(&tf.path, true, 1024 * 1024));
data.accounts = av;

Expand Down