Skip to content
This repository was archived by the owner on Nov 15, 2023. It is now read-only.

Add DB Read/Write Tracking to Benchmarking Pipeline #6386

Merged
gavofyork merged 22 commits into from
Jun 24, 2020
Merged

Add DB Read/Write Tracking to Benchmarking Pipeline #6386

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
7468464
initial mockup
shawntabrizi Jun 17, 2020
947d68c
add and wipe
shawntabrizi Jun 17, 2020
9c77f50
track writes
shawntabrizi Jun 17, 2020
2f198b9
start to add to pipeline
shawntabrizi Jun 18, 2020
12395ae
return all reads/writes
shawntabrizi Jun 18, 2020
69f137d
Log reads and writes from bench db
shawntabrizi Jun 18, 2020
60f22c3
causes panic
shawntabrizi Jun 18, 2020
e08ffef
Allow multiple commits
arkpar Jun 18, 2020
86ecbeb
commit before ending benchmark
shawntabrizi Jun 18, 2020
f7305a2
Merge branch 'master' into shawntabrizi-bench-db-tracking
shawntabrizi Jun 18, 2020
004bea1
doesn't work???
shawntabrizi Jun 18, 2020
1b25a8b
fix
shawntabrizi Jun 18, 2020
c995215
Update lib.rs
shawntabrizi Jun 18, 2020
2eef7fa
switch to struct for `BenchmarkResults`
shawntabrizi Jun 18, 2020
473f492
add to output
shawntabrizi Jun 18, 2020
0ae5300
fix test
shawntabrizi Jun 18, 2020
fd99bda
line width
shawntabrizi Jun 18, 2020
79cd015
@kianenigma review
shawntabrizi Jun 18, 2020
9464475
Add Whitelist to DB Tracking in Benchmarks Pipeline (#6405)
shawntabrizi Jun 20, 2020
956cda1
Add regression analysis to DB Tracking (#6475)
shawntabrizi Jun 22, 2020
893bc52
Update client/db/src/bench.rs
shawntabrizi Jun 24, 2020
00cb79b
Merge branch 'master' into shawntabrizi-bench-db-tracking
shawntabrizi Jun 24, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
145 changes: 138 additions & 7 deletions client/db/src/bench.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,10 @@ use std::collections::HashMap;

use hash_db::{Prefix, Hasher};
use sp_trie::{MemoryDB, prefixed_key};
use sp_core::storage::ChildInfo;
use sp_core::{storage::ChildInfo, hexdisplay::HexDisplay};
use sp_runtime::traits::{Block as BlockT, HashFor};
use sp_runtime::Storage;
use sp_state_machine::{DBValue, backend::Backend as StateBackend};
use sp_state_machine::{DBValue, backend::Backend as StateBackend, StorageCollection};
use kvdb::{KeyValueDB, DBTransaction};
use crate::storage_cache::{CachingState, SharedCache, new_shared_cache};

Expand All @@ -50,6 +50,40 @@ impl<Block: BlockT> sp_state_machine::Storage<HashFor<Block>> for StorageDb<Bloc
}
}

/// Track whether a specific key has already been read or written to.
#[derive(Default, Clone, Copy)]
pub struct KeyTracker {
has_been_read: bool,
has_been_written: bool,
}

/// A simple object that counts the reads and writes at the key level to the underlying state db.
#[derive(Default, Clone, Copy, Debug)]
pub struct ReadWriteTracker {
reads: u32,
repeat_reads: u32,
writes: u32,
repeat_writes: u32,
}

impl ReadWriteTracker {
fn add_read(&mut self) {
self.reads += 1;
}

fn add_repeat_read(&mut self) {
self.repeat_reads += 1;
}

fn add_write(&mut self) {
self.writes += 1;
}

fn add_repeat_write(&mut self) {
self.repeat_writes += 1;
}
}

/// State that manages the backend database reference. Allows runtime to control the database.
pub struct BenchmarkingState<B: BlockT> {
root: Cell<B::Hash>,
Expand All @@ -59,6 +93,8 @@ pub struct BenchmarkingState<B: BlockT> {
genesis: HashMap<Vec<u8>, (Vec<u8>, i32)>,
record: Cell<Vec<Vec<u8>>>,
shared_cache: SharedCache<B>, // shared cache is always empty
key_tracker: RefCell<HashMap<Vec<u8>, KeyTracker>>,
read_write_tracker: RefCell<ReadWriteTracker>,
}

impl<B: BlockT> BenchmarkingState<B> {
Expand All @@ -76,6 +112,8 @@ impl<B: BlockT> BenchmarkingState<B> {
genesis_root: Default::default(),
record: Default::default(),
shared_cache: new_shared_cache(0, (1, 10)),
key_tracker: Default::default(),
read_write_tracker: Default::default(),
};

state.reopen()?;
Expand All @@ -89,7 +127,7 @@ impl<B: BlockT> BenchmarkingState<B> {
);
state.genesis = transaction.clone().drain();
state.genesis_root = root.clone();
state.commit(root, transaction)?;
state.commit(root, transaction, Vec::new())?;
state.record.take();
Ok(state)
}
Expand All @@ -109,6 +147,70 @@ impl<B: BlockT> BenchmarkingState<B> {
));
Ok(())
}

fn wipe_tracker(&self) {
*self.key_tracker.borrow_mut() = HashMap::new();
*self.read_write_tracker.borrow_mut() = Default::default();
}

fn add_read_key(&self, key: &[u8]) {
log::trace!(target: "benchmark", "Read: {}", HexDisplay::from(&key));

let mut key_tracker = self.key_tracker.borrow_mut();
let mut read_write_tracker = self.read_write_tracker.borrow_mut();

let maybe_tracker = key_tracker.get(&key.to_vec());
Comment thread
shawntabrizi marked this conversation as resolved.
Outdated

let has_been_read = KeyTracker {
has_been_read: true,
has_been_written: false,
};

match maybe_tracker {
None => {
key_tracker.insert(key.to_vec(), has_been_read);
read_write_tracker.add_read();
},
Some(tracker) => {
if !tracker.has_been_read {
key_tracker.insert(key.to_vec(), has_been_read);
read_write_tracker.add_read();
} else {
read_write_tracker.add_repeat_read();
}
}
}
}

fn add_write_key(&self, key: &[u8]) {
log::trace!(target: "benchmark", "Write: {}", HexDisplay::from(&key));

let mut key_tracker = self.key_tracker.borrow_mut();
let mut read_write_tracker = self.read_write_tracker.borrow_mut();

let maybe_tracker = key_tracker.get(&key.to_vec());

// If we have written to the key, we also consider that we have read from it.
let has_been_written = KeyTracker {
has_been_read: true,
has_been_written: true,
};

match maybe_tracker {
None => {
key_tracker.insert(key.to_vec(), has_been_written);
read_write_tracker.add_write();
},
Some(tracker) => {
if !tracker.has_been_written {
key_tracker.insert(key.to_vec(), has_been_written);
read_write_tracker.add_write();
} else {
read_write_tracker.add_repeat_write();
}
}
}
}
}

fn state_err() -> String {
Expand All @@ -121,10 +223,12 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
type TrieBackendStorage = <DbState<B> as StateBackend<HashFor<B>>>::TrieBackendStorage;

fn storage(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.add_read_key(key);
Comment thread
shawntabrizi marked this conversation as resolved.
self.state.borrow().as_ref().ok_or_else(state_err)?.storage(key)
}

fn storage_hash(&self, key: &[u8]) -> Result<Option<B::Hash>, Self::Error> {
self.add_read_key(key);
self.state.borrow().as_ref().ok_or_else(state_err)?.storage_hash(key)
}

Expand All @@ -133,10 +237,12 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
child_info: &ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
self.add_read_key(key);
self.state.borrow().as_ref().ok_or_else(state_err)?.child_storage(child_info, key)
}

fn exists_storage(&self, key: &[u8]) -> Result<bool, Self::Error> {
self.add_read_key(key);
self.state.borrow().as_ref().ok_or_else(state_err)?.exists_storage(key)
}

Expand All @@ -145,10 +251,12 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
child_info: &ChildInfo,
key: &[u8],
) -> Result<bool, Self::Error> {
self.add_read_key(key);
self.state.borrow().as_ref().ok_or_else(state_err)?.exists_child_storage(child_info, key)
}

fn next_storage_key(&self, key: &[u8]) -> Result<Option<Vec<u8>>, Self::Error> {
self.add_read_key(key);
self.state.borrow().as_ref().ok_or_else(state_err)?.next_storage_key(key)
}

Expand All @@ -157,6 +265,7 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
child_info: &ChildInfo,
key: &[u8],
) -> Result<Option<Vec<u8>>, Self::Error> {
self.add_read_key(key);
self.state.borrow().as_ref().ok_or_else(state_err)?.next_child_storage_key(child_info, key)
}

Expand Down Expand Up @@ -230,8 +339,11 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
None
}

fn commit(&self, storage_root: <HashFor<B> as Hasher>::Out, mut transaction: Self::Transaction)
-> Result<(), Self::Error>
fn commit(&self,
storage_root: <HashFor<B> as Hasher>::Out,
mut transaction: Self::Transaction,
storage_changes: StorageCollection,
) -> Result<(), Self::Error>
{
if let Some(db) = self.db.take() {
let mut db_transaction = DBTransaction::new();
Expand All @@ -245,10 +357,17 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {
}
keys.push(key);
}
self.record.set(keys);
let mut record = self.record.take();
record.extend(keys);
self.record.set(record);
db.write(db_transaction).map_err(|_| String::from("Error committing transaction"))?;
self.root.set(storage_root);
self.db.set(Some(db))
self.db.set(Some(db));

// Track DB Writes
storage_changes.iter().for_each(|(key, _)| {
self.add_write_key(key);
});
} else {
return Err("Trying to commit to a closed db".into())
}
Expand All @@ -272,9 +391,21 @@ impl<B: BlockT> StateBackend<HashFor<B>> for BenchmarkingState<B> {

self.root.set(self.genesis_root.clone());
self.reopen()?;
self.wipe_tracker();
Ok(())
}

/// Get the key tracking information for the state db.
fn read_write_count(&self) -> (u32, u32, u32, u32) {
let count = *self.read_write_tracker.borrow_mut();
(count.reads, count.repeat_reads, count.writes, count.repeat_writes)
}

/// Reset the key tracking information for the state db.
fn reset_read_write_count(&self) {
self.wipe_tracker()
}

fn register_overlay_stats(&mut self, stats: &sp_state_machine::StateMachineStats) {
self.state.borrow_mut().as_mut().map(|s| s.register_overlay_stats(stats));
}
Expand Down
66 changes: 41 additions & 25 deletions frame/benchmarking/src/analysis.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,22 +31,22 @@ pub struct Analysis {

impl Analysis {
pub fn median_slopes(r: &Vec<BenchmarkResults>) -> Option<Self> {
let results = r[0].0.iter().enumerate().map(|(i, &(param, _))| {
let results = r[0].components.iter().enumerate().map(|(i, &(param, _))| {
let mut counted = BTreeMap::<Vec<u32>, usize>::new();
for (params, _, _) in r.iter() {
let mut p = params.iter().map(|x| x.1).collect::<Vec<_>>();
for result in r.iter() {
let mut p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
p[i] = 0;
*counted.entry(p).or_default() += 1;
}
let others: Vec<u32> = counted.iter().max_by_key(|i| i.1).expect("r is not empty; qed").0.clone();
let values = r.iter()
.filter(|v|
v.0.iter()
v.components.iter()
.map(|x| x.1)
.zip(others.iter())
.enumerate()
.all(|(j, (v1, v2))| j == i || v1 == *v2)
).map(|(ps, v, _)| (ps[i].1, *v))
).map(|result| (result.components[i].1, result.extrinsic_time))
.collect::<Vec<_>>();
(format!("{:?}", param), i, others, values)
}).collect::<Vec<_>>();
Expand Down Expand Up @@ -99,9 +99,9 @@ impl Analysis {

pub fn min_squares_iqr(r: &Vec<BenchmarkResults>) -> Option<Self> {
let mut results = BTreeMap::<Vec<u32>, Vec<u128>>::new();
for &(ref params, t, _) in r.iter() {
let p = params.iter().map(|x| x.1).collect::<Vec<_>>();
results.entry(p).or_default().push(t);
for result in r.iter() {
let p = result.components.iter().map(|x| x.1).collect::<Vec<_>>();
results.entry(p).or_default().push(result.extrinsic_time);
}
for (_, rs) in results.iter_mut() {
rs.sort();
Expand All @@ -111,7 +111,7 @@ impl Analysis {

let mut data = vec![("Y", results.iter().flat_map(|x| x.1.iter().map(|v| *v as f64)).collect())];

let names = r[0].0.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
let names = r[0].components.iter().map(|x| format!("{:?}", x.0)).collect::<Vec<_>>();
data.extend(names.iter()
.enumerate()
.map(|(i, p)| (
Expand Down Expand Up @@ -222,17 +222,33 @@ mod tests {
use super::*;
use crate::BenchmarkParameter;

fn benchmark_result(
components: Vec<(BenchmarkParameter, u32)>,
extrinsic_time: u128,
storage_root_time: u128,
) -> BenchmarkResults {
BenchmarkResults {
components,
extrinsic_time,
storage_root_time,
reads: 0,
repeat_reads: 0,
writes: 0,
repeat_writes: 0,
}
}

#[test]
fn analysis_median_slopes_should_work() {
let a = Analysis::median_slopes(&vec![
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
]).unwrap();
assert_eq!(a.base, 10_000_000);
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
Expand All @@ -241,14 +257,14 @@ mod tests {
#[test]
fn analysis_median_min_squares_should_work() {
let a = Analysis::min_squares_iqr(&vec![
(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 1), (BenchmarkParameter::m, 5)], 11_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 2), (BenchmarkParameter::m, 5)], 12_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 5)], 13_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 4), (BenchmarkParameter::m, 5)], 14_500_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 1)], 13_100_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 3)], 13_300_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 7)], 13_700_000, 0),
benchmark_result(vec![(BenchmarkParameter::n, 3), (BenchmarkParameter::m, 10)], 14_000_000, 0),
]).unwrap();
assert_eq!(a.base, 10_000_000);
assert_eq!(a.slopes, vec![1_000_000, 100_000]);
Expand Down
Loading