Skip to content

[r2r] IndexedDB Cursor Iterator #1678

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
shamardy merged 7 commits into from
Mar 7, 2023
Merged

[r2r] IndexedDB Cursor Iterator #1678

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
fbcfe46
Save dev state
sergeyboyko0791 Feb 10, 2023
c7792c2
Refactor IndexedDB Cursor
sergeyboyko0791 Feb 23, 2023
bded40c
Merge branch 'dev' into cursor-iter
sergeyboyko0791 Feb 23, 2023
b4acf2c
Fix merge conflicts
sergeyboyko0791 Feb 23, 2023
4deacf5
Fix and optimize `IndexedDbBlockHeaderStorage`
sergeyboyko0791 Feb 24, 2023
3b4a6e0
Fix TODOs
sergeyboyko0791 Feb 24, 2023
d8bc96c
Fix PR issues
sergeyboyko0791 Mar 2, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 3 additions & 4 deletions mm2src/coins/utxo/utxo_block_header_storage/mod.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,6 @@ impl BlockHeaderStorage {
})
}

#[cfg(test)]
pub(crate) fn into_inner(self) -> Box<dyn BlockHeaderStorageOps> { self.inner }
}

Expand Down Expand Up @@ -108,7 +107,7 @@ impl BlockHeaderStorageOps for BlockHeaderStorage {
async fn is_table_empty(&self) -> Result<(), BlockHeaderStorageError> { self.inner.is_table_empty().await }
}

#[cfg(test)]
#[cfg(any(test, target_arch = "wasm32"))]
mod block_headers_storage_tests {
use super::*;
use chain::BlockHeaderBits;
Expand Down Expand Up @@ -313,7 +312,7 @@ mod native_tests {
fn test_remove_headers_up_to_height() { block_on(test_remove_headers_up_to_height_impl(FOR_COIN_GET)) }
}

#[cfg(all(test, target_arch = "wasm32"))]
#[cfg(target_arch = "wasm32")]
mod wasm_test {
use super::*;
use crate::utxo::utxo_block_header_storage::block_headers_storage_tests::*;
Expand All @@ -323,7 +322,7 @@ mod wasm_test {

wasm_bindgen_test_configure!(run_in_browser);

const FOR_COIN: &str = "RICK";
const FOR_COIN: &str = "tBTC";
Comment thread
shamardy marked this conversation as resolved.

#[wasm_bindgen_test]
async fn test_storage_init() {
Expand Down
8 changes: 4 additions & 4 deletions mm2src/coins/utxo/utxo_block_header_storage/wasm/block_header_table.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
use mm2_db::indexed_db::{DbUpgrader, OnUpgradeResult, TableSignature};
use mm2_db::indexed_db::{BeBigUint, DbUpgrader, OnUpgradeResult, TableSignature};

#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct BlockHeaderStorageTable {
pub height: u64,
pub height: BeBigUint,
pub bits: u32,
pub hash: String,
pub raw_header: String,
pub ticker: String,
}

impl BlockHeaderStorageTable {
pub const HEIGHT_TICKER_INDEX: &str = "block_height_ticker_index";
pub const TICKER_HEIGHT_INDEX: &str = "block_height_ticker_index";
pub const HASH_TICKER_INDEX: &str = "block_hash_ticker_index";
}

Expand All @@ -21,7 +21,7 @@ impl TableSignature for BlockHeaderStorageTable {
match (old_version, new_version) {
(0, 1) => {
let table = upgrader.create_table(Self::table_name())?;
table.create_multi_index(Self::HEIGHT_TICKER_INDEX, &["height", "ticker"], true)?;
table.create_multi_index(Self::TICKER_HEIGHT_INDEX, &["ticker", "height"], true)?;
table.create_multi_index(Self::HASH_TICKER_INDEX, &["hash", "ticker"], true)?;
table.create_index("ticker", false)?;
},
Expand Down
100 changes: 60 additions & 40 deletions mm2src/coins/utxo/utxo_block_header_storage/wasm/indexeddb_block_header_storage.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ use super::BlockHeaderStorageTable;
use async_trait::async_trait;
use chain::BlockHeader;
use mm2_core::mm_ctx::MmArc;
use mm2_db::indexed_db::cursor_prelude::{CollectCursor, WithOnly};
use mm2_db::indexed_db::{ConstructibleDb, DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder,
use mm2_db::indexed_db::{BeBigUint, ConstructibleDb, DbIdentifier, DbInstance, DbLocked, IndexedDb, IndexedDbBuilder,
InitDbResult, MultiIndex, SharedDb};
use mm2_err_handle::prelude::*;
use num_traits::ToPrimitive;
use primitives::hash::H256;
use serialization::Reader;
use spv_validation::storage::{BlockHeaderStorageError, BlockHeaderStorageOps};
Expand Down Expand Up @@ -93,15 +93,15 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
let bits: u32 = header.bits.into();
let headers_to_store = BlockHeaderStorageTable {
ticker: ticker.clone(),
height,
height: BeBigUint::from(height),
bits,
hash,
raw_header,
};
let index_keys = MultiIndex::new(BlockHeaderStorageTable::HEIGHT_TICKER_INDEX)
.with_value(&height)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
let index_keys = MultiIndex::new(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.with_value(&ticker)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
.with_value(&BeBigUint::from(height))
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

block_headers_db
Expand Down Expand Up @@ -149,10 +149,10 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.table::<BlockHeaderStorageTable>()
.await
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;
let index_keys = MultiIndex::new(BlockHeaderStorageTable::HEIGHT_TICKER_INDEX)
.with_value(&height)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
let index_keys = MultiIndex::new(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.with_value(&ticker)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
.with_value(&BeBigUint::from(height))
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

Ok(block_headers_db
Expand All @@ -178,21 +178,30 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.await
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

// Todo: use open_cursor with direction to optimze this process.
let res = block_headers_db
.open_cursor("ticker")
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
let maybe_item = block_headers_db
.cursor_builder()
.only("ticker", ticker.clone())
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.collect()
// We need to provide any constraint on the `height` property
// since `ticker_height` consists of both `ticker` and `height` properties.
.bound("height", BeBigUint::from(0u64), BeBigUint::from(u64::MAX))
Comment thread
shamardy marked this conversation as resolved.
// Cursor returns values from the lowest to highest key indexes.
// But we need to get the most highest height, so reverse the cursor direction.
.reverse()
.open_cursor(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.into_iter()
.map(|(_item_id, item)| item.height)
.collect::<Vec<_>>();
.next()
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?;

Ok(res.into_iter().max())
maybe_item
.map(|(_, item)| {
item.height
.to_u64()
.ok_or_else(|| BlockHeaderStorageError::get_err(&ticker, "height is too large".to_string()))
})
.transpose()
}

async fn get_last_block_header_with_non_max_bits(
Expand All @@ -214,25 +223,29 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.await
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

// Todo: use open_cursor with direction to optimze this process.
let res = block_headers_db
.open_cursor("ticker")
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
let mut cursor = block_headers_db
.cursor_builder()
.only("ticker", ticker.clone())
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.collect()
// We need to provide any constraint on the `height` property
// since `ticker_height` consists of both `ticker` and `height` properties.
.bound("height", BeBigUint::from(0u64), BeBigUint::from(u64::MAX))
// Cursor returns values from the lowest to highest key indexes.
// But we need to get the most highest height, so reverse the cursor direction.
.reverse()
.open_cursor(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?;

while let Some((_item_id, header)) = cursor
.next()
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.into_iter()
.map(|(_item_id, item)| item)
.collect::<Vec<_>>();
let res = res
.into_iter()
.filter_map(|e| if e.bits != max_bits { Some(e) } else { None })
.collect::<Vec<_>>();

for header in res {
{
if header.bits == max_bits {
continue;
}

let serialized = &hex::decode(header.raw_header).map_err(|e| BlockHeaderStorageError::DecodeError {
coin: ticker.clone(),
reason: e.to_string(),
Expand Down Expand Up @@ -273,11 +286,18 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.with_value(&ticker)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

Ok(block_headers_db
let maybe_item = block_headers_db
.get_item_by_unique_multi_index(index_keys)
.await
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?
.map(|raw| raw.1.height as i64))
.map_err(|err| BlockHeaderStorageError::get_err(&ticker, err.to_string()))?;

maybe_item
.map(|(_, item)| {
item.height
.to_i64()
.ok_or_else(|| BlockHeaderStorageError::get_err(&ticker, "height is too large".to_string()))
})
.transpose()
}

async fn remove_headers_up_to_height(&self, to_height: u64) -> Result<(), BlockHeaderStorageError> {
Expand All @@ -297,10 +317,10 @@ impl BlockHeaderStorageOps for IDBBlockHeadersStorage {
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

for height in 0..=to_height {
let index_keys = MultiIndex::new(BlockHeaderStorageTable::HEIGHT_TICKER_INDEX)
.with_value(&height)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
let index_keys = MultiIndex::new(BlockHeaderStorageTable::TICKER_HEIGHT_INDEX)
.with_value(&ticker)
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?
.with_value(&BeBigUint::from(height))
.map_err(|err| BlockHeaderStorageError::table_err(&ticker, err.to_string()))?;

block_headers_db
Expand Down
3 changes: 2 additions & 1 deletion mm2src/mm2_db/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ doctest = false
async-trait = "0.1"
common = { path = "../common" }
derive_more = "0.99"
enum_from = { path = "../derives/enum_from" }
futures = { version = "0.3", package = "futures", features = ["compat", "async-await", "thread-pool"] }
itertools = "0.10"
hex = "0.4.2"
Expand All @@ -26,4 +27,4 @@ serde_json = { version = "1.0", features = ["preserve_order", "raw_value"] }
wasm-bindgen = { version = "0.2.50", features = ["nightly"] }
wasm-bindgen-futures = { version = "0.4.1" }
wasm-bindgen-test = { version = "0.3.2" }
web-sys = { version = "0.3.55", features = ["console", "CloseEvent", "DomException", "ErrorEvent", "IdbDatabase", "IdbCursor", "IdbCursorWithValue", "IdbFactory", "IdbIndex", "IdbIndexParameters", "IdbObjectStore", "IdbObjectStoreParameters", "IdbOpenDbRequest", "IdbKeyRange", "IdbTransaction", "IdbTransactionMode", "IdbVersionChangeEvent", "MessageEvent", "WebSocket"] }
web-sys = { version = "0.3.55", features = ["console", "CloseEvent", "DomException", "ErrorEvent", "IdbDatabase", "IdbCursor", "IdbCursorWithValue", "IdbCursorDirection", "IdbFactory", "IdbIndex", "IdbIndexParameters", "IdbObjectStore", "IdbObjectStoreParameters", "IdbOpenDbRequest", "IdbKeyRange", "IdbTransaction", "IdbTransactionMode", "IdbVersionChangeEvent", "MessageEvent", "WebSocket"] }
Loading