Skip to content
Merged
1,164 changes: 1,164 additions & 0 deletions activity.md

Large diffs are not rendered by default.

230 changes: 114 additions & 116 deletions src/database/identities.rs
Original file line number Diff line number Diff line change
Expand Up @@ -156,54 +156,54 @@ impl Database {

let conn = self.conn.lock().unwrap();

// Prepare the main statement to select identities, including wallet_index
// Use a LEFT JOIN to load identities and their top-ups in a single query,
// avoiding the N+1 query pattern of querying top_up per identity.
let mut stmt = conn.prepare(
"SELECT data, alias, wallet_index, status FROM identity WHERE is_local = 1 AND network = ? AND data IS NOT NULL",
"SELECT i.data, i.alias, i.wallet_index, i.status, i.id, t.top_up_index, t.amount
FROM identity i
LEFT JOIN top_up t ON i.id = t.identity_id
WHERE i.is_local = 1 AND i.network = ? AND i.data IS NOT NULL
ORDER BY i.id",
)?;

// Prepare the statement to select top-ups (will be used multiple times)
let mut top_up_stmt =
conn.prepare("SELECT top_up_index, amount FROM top_up WHERE identity_id = ?")?;
let mut rows = stmt.query(params![network])?;

// Iterate over each identity
let identity_iter = stmt.query_map(params![network], |row| {
let data: Vec<u8> = row.get(0)?;
let alias: Option<String> = row.get(1)?;
let wallet_index: Option<u32> = row.get(2)?;
// Handle NULL status values from older database entries by defaulting to Active (2)
let status: Option<u8> = row.get(3)?;

let mut identity: QualifiedIdentity = QualifiedIdentity::from_bytes(&data);
identity.alias = alias;
identity.wallet_index = wallet_index;

identity.status = IdentityStatus::from_u8(status.unwrap_or(2));
identity.network = app_context.network;
let mut identities: Vec<QualifiedIdentity> = Vec::new();
let mut current_id: Option<Vec<u8>> = None;

// Associate wallets
identity.associated_wallets = wallets.clone(); //todo: use less wallets

// Retrieve the identity_id as bytes
let identity_id = identity.identity.id().to_buffer();
while let Some(row) = rows.next()? {
let id: Vec<u8> = row.get(4)?;

// Query the top_up table for this identity_id
let mut top_ups = BTreeMap::new();
let mut rows = top_up_stmt.query(params![identity_id])?;
if current_id.as_ref() != Some(&id) {
// New identity row
let data: Vec<u8> = row.get(0)?;
let alias: Option<String> = row.get(1)?;
let wallet_index: Option<u32> = row.get(2)?;
let status: Option<u8> = row.get(3)?;

let mut identity = QualifiedIdentity::from_bytes(&data);
Comment thread
lklimek marked this conversation as resolved.
Resolved
identity.alias = alias;
identity.wallet_index = wallet_index;
identity.status = IdentityStatus::from_u8(status.unwrap_or(2));
identity.network = app_context.network;
identity.associated_wallets = wallets.clone();
identity.top_ups = BTreeMap::new();

while let Some(top_up_row) = rows.next()? {
let top_up_index: u32 = top_up_row.get(0)?;
let amount: u64 = top_up_row.get(1)?;
top_ups.insert(top_up_index, amount);
identities.push(identity);
current_id = Some(id);
}

// Assign the top_ups to the identity
identity.top_ups = top_ups;

Ok(identity)
})?;
// Add top-up entry if present (NULL when identity has no top-ups)
let top_up_index: Option<u32> = row.get(5)?;
let amount: Option<u64> = row.get(6)?;
if let (Some(idx), Some(amt)) = (top_up_index, amount)
&& let Some(identity) = identities.last_mut()
{
identity.top_ups.insert(idx, amt);
}
}

let identities: rusqlite::Result<Vec<QualifiedIdentity>> = identity_iter.collect();
identities
Ok(identities)
}

#[allow(dead_code)] // May be used for filtering identities that belong to specific wallets
Expand All @@ -216,50 +216,51 @@ impl Database {

let conn = self.conn.lock().unwrap();

// Prepare the main statement to select identities, including wallet_index
// Use a LEFT JOIN to load identities and their top-ups in a single query.
let mut stmt = conn.prepare(
"SELECT data, alias, wallet_index FROM identity WHERE is_local = 1 AND network = ? AND data IS NOT NULL AND wallet_index IS NOT NULL",
"SELECT i.data, i.alias, i.wallet_index, i.id, t.top_up_index, t.amount
FROM identity i
LEFT JOIN top_up t ON i.id = t.identity_id
WHERE i.is_local = 1 AND i.network = ? AND i.data IS NOT NULL AND i.wallet_index IS NOT NULL
ORDER BY i.id",
)?;

// Prepare the statement to select top-ups (will be used multiple times)
let mut top_up_stmt =
conn.prepare("SELECT top_up_index, amount FROM top_up WHERE identity_id = ?")?;

// Iterate over each identity
let identity_iter = stmt.query_map(params![network], |row| {
let data: Vec<u8> = row.get(0)?;
let alias: Option<String> = row.get(1)?;
let wallet_index: Option<u32> = row.get(2)?;
let mut rows = stmt.query(params![network])?;

let mut identity: QualifiedIdentity = QualifiedIdentity::from_bytes(&data);
identity.alias = alias;
identity.wallet_index = wallet_index;
identity.network = app_context.network;
let mut identities: Vec<QualifiedIdentity> = Vec::new();
let mut current_id: Option<Vec<u8>> = None;

// Associate wallets
identity.associated_wallets = wallets.clone(); //todo: use less wallets
while let Some(row) = rows.next()? {
let id: Vec<u8> = row.get(3)?;

// Retrieve the identity_id as bytes
let identity_id = identity.identity.id().to_buffer();
if current_id.as_ref() != Some(&id) {
// New identity row
let data: Vec<u8> = row.get(0)?;
let alias: Option<String> = row.get(1)?;
let wallet_index: Option<u32> = row.get(2)?;

// Query the top_up table for this identity_id
let mut top_ups = BTreeMap::new();
let mut rows = top_up_stmt.query(params![identity_id])?;
let mut identity = QualifiedIdentity::from_bytes(&data);
identity.alias = alias;
identity.wallet_index = wallet_index;
identity.network = app_context.network;
identity.associated_wallets = wallets.clone();
identity.top_ups = BTreeMap::new();

while let Some(top_up_row) = rows.next()? {
let top_up_index: u32 = top_up_row.get(0)?;
let amount: u64 = top_up_row.get(1)?;
top_ups.insert(top_up_index, amount);
identities.push(identity);
current_id = Some(id);
}

// Assign the top_ups to the identity
identity.top_ups = top_ups;

Ok(identity)
})?;
// Add top-up entry if present
let top_up_index: Option<u32> = row.get(4)?;
let amount: Option<u64> = row.get(5)?;
if let (Some(idx), Some(amt)) = (top_up_index, amount)
&& let Some(identity) = identities.last_mut()
{
identity.top_ups.insert(idx, amt);
}
}

let identities: rusqlite::Result<Vec<QualifiedIdentity>> = identity_iter.collect();
identities
Ok(identities)
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Resolved
}

pub fn get_identity_by_id(
Expand All @@ -272,50 +273,45 @@ impl Database {

let conn = self.conn.lock().unwrap();

// Prepare the main statement to select identities, including wallet_index
// Use a LEFT JOIN to load identity and its top-ups in a single query.
let mut stmt = conn.prepare(
"SELECT data, alias, wallet_index FROM identity WHERE id = ? AND is_local = 1 AND network = ? AND data IS NOT NULL",
"SELECT i.data, i.alias, i.wallet_index, t.top_up_index, t.amount
FROM identity i
LEFT JOIN top_up t ON i.id = t.identity_id
WHERE i.id = ? AND i.is_local = 1 AND i.network = ? AND i.data IS NOT NULL",
)?;

// Prepare the statement to select top-ups (will be used multiple times)
let mut top_up_stmt =
conn.prepare("SELECT top_up_index, amount FROM top_up WHERE identity_id = ?")?;

// Iterate over each identity
let identity_iter = stmt.query_map(params![identifier.to_buffer(), network], |row| {
let data: Vec<u8> = row.get(0)?;
let alias: Option<String> = row.get(1)?;
let wallet_index: Option<u32> = row.get(2)?;

let mut identity: QualifiedIdentity = QualifiedIdentity::from_bytes(&data);
identity.alias = alias;
identity.wallet_index = wallet_index;
identity.network = app_context.network;
let mut rows = stmt.query(params![identifier.to_buffer(), network])?;

// Associate wallets
identity.associated_wallets = wallets.clone(); //todo: use less wallets
let mut identity: Option<QualifiedIdentity> = None;

// Retrieve the identity_id as bytes
let identity_id = identity.identity.id().to_buffer();
while let Some(row) = rows.next()? {
if identity.is_none() {
let data: Vec<u8> = row.get(0)?;
let alias: Option<String> = row.get(1)?;
let wallet_index: Option<u32> = row.get(2)?;

// Query the top_up table for this identity_id
let mut top_ups = BTreeMap::new();
let mut rows = top_up_stmt.query(params![identity_id])?;
let mut qi = QualifiedIdentity::from_bytes(&data);
qi.alias = alias;
qi.wallet_index = wallet_index;
qi.network = app_context.network;
qi.associated_wallets = wallets.clone();
qi.top_ups = BTreeMap::new();

while let Some(top_up_row) = rows.next()? {
let top_up_index: u32 = top_up_row.get(0)?;
let amount: u64 = top_up_row.get(1)?;
top_ups.insert(top_up_index, amount);
identity = Some(qi);
}

// Assign the top_ups to the identity
identity.top_ups = top_ups;

Ok(identity)
})?;
// Add top-up entry if present
let top_up_index: Option<u32> = row.get(3)?;
let amount: Option<u64> = row.get(4)?;
if let (Some(idx), Some(amt)) = (top_up_index, amount)
&& let Some(ref mut qi) = identity
{
qi.top_ups.insert(idx, amt);
}
}

let identities: rusqlite::Result<Vec<QualifiedIdentity>> = identity_iter.collect();
Ok(identities?.into_iter().next())
Ok(identity)
}

pub fn get_local_voting_identities(
Expand Down Expand Up @@ -469,21 +465,28 @@ impl Database {
Ok(())
}

/// Loads the user’s custom identity order (the entire list).
/// Loads the user's custom identity order (the entire list).
/// If an identity in the order doesn't exist in the identity table, it is removed.
pub fn load_identity_order(&self) -> rusqlite::Result<Vec<Identifier>> {
let conn = self.conn.lock().unwrap();

// Read all rows sorted by pos
let mut stmt = conn.prepare("SELECT identity_id FROM identity_order ORDER BY pos ASC")?;
// Use a LEFT JOIN to get all order entries and detect dangling references
// in a single query instead of per-row EXISTS checks.
let mut stmt = conn.prepare(
"SELECT io.identity_id, i.id IS NOT NULL AS exists_in_identity
FROM identity_order io
LEFT JOIN identity i ON io.identity_id = i.id
ORDER BY io.pos ASC",
)?;

let mut rows = stmt.query([])?;
let mut final_list = Vec::new();
let mut to_remove = Vec::new();

while let Some(row) = rows.next()? {
let id_bytes: Vec<u8> = row.get(0)?;
// Convert from raw bytes to an Identifier
let exists: bool = row.get(1)?;

let identifier = match Identifier::from_vec(id_bytes.clone()) {
Ok(id) => id,
Err(_) => {
Expand All @@ -493,20 +496,15 @@ impl Database {
}
};

// Check if the identity is still in 'identity' table
let mut check_stmt =
conn.prepare("SELECT EXISTS(SELECT 1 FROM identity WHERE id = ?)")?;
let exists: i64 = check_stmt.query_row(params![identifier.to_vec()], |r| r.get(0))?;
if exists == 1 {
// Keep it
if exists {
final_list.push(identifier);
} else {
// Queue for removal because it doesn't exist in the identity table
to_remove.push(identifier.to_vec());
}
}

// Remove any “dangling” references
// Remove any "dangling" references
for id in to_remove {
conn.execute(
"DELETE FROM identity_order WHERE identity_id = ?",
Expand Down
41 changes: 40 additions & 1 deletion src/database/initialization.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ use rusqlite::{Connection, params};
use std::fs;
use std::path::Path;

pub const DEFAULT_DB_VERSION: u16 = 26;
pub const DEFAULT_DB_VERSION: u16 = 27;

pub const DEFAULT_NETWORK: &str = "dash";

Expand Down Expand Up @@ -51,6 +51,9 @@ impl Database {

fn apply_version_changes(&self, version: u16, tx: &Connection) -> rusqlite::Result<()> {
match version {
27 => {
self.add_network_indexes(tx)?;
}
26 => {
self.add_last_full_sync_balance_column(tx)?;
}
Expand Down Expand Up @@ -316,6 +319,11 @@ impl Database {
[],
)?;

conn.execute(
"CREATE INDEX IF NOT EXISTS idx_wallet_network ON wallet (network)",
[],
)?;

// Create wallet addresses
conn.execute(
"CREATE TABLE IF NOT EXISTS wallet_addresses (
Expand Down Expand Up @@ -398,6 +406,11 @@ impl Database {
[],
)?;

conn.execute(
"CREATE INDEX IF NOT EXISTS idx_asset_lock_transaction_network ON asset_lock_transaction (network)",
[],
)?;

// Create the identities table
conn.execute(
"CREATE TABLE IF NOT EXISTS identity (
Expand Down Expand Up @@ -822,6 +835,32 @@ impl Database {

Ok(())
}

/// Migration: Add network indexes to high-traffic tables (version 27).
/// These tables are frequently queried with WHERE network = ? but lacked indexes.
fn add_network_indexes(&self, conn: &Connection) -> rusqlite::Result<()> {
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_wallet_network ON wallet (network)",
[],
)?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_token_network ON token (network)",
[],
)?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_identity_token_balances_network ON identity_token_balances (network)",
[],
)?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_scheduled_votes_network ON scheduled_votes (network)",
[],
)?;
conn.execute(
"CREATE INDEX IF NOT EXISTS idx_asset_lock_transaction_network ON asset_lock_transaction (network)",
[],
)?;
Ok(())
}
}

#[cfg(test)]
Expand Down
Loading
Loading