From 8e678058f12d5136b13b35e70a8ff2dfc3178f4e Mon Sep 17 00:00:00 2001 From: Yeastplume Date: Fri, 14 Dec 2018 16:24:53 +0000 Subject: [PATCH] [WIP] Store completed transactions in files instead of DB (#2148) Store completed transactions in files instead of DB --- src/bin/cmd/wallet_tests.rs | 4 +- wallet/src/command.rs | 2 +- wallet/src/controller.rs | 5 +- wallet/src/display.rs | 2 +- wallet/src/libwallet/api.rs | 22 ++++-- wallet/src/libwallet/internal/selection.rs | 78 ++++++++++++---------- wallet/src/libwallet/internal/tx.rs | 22 ++---- wallet/src/libwallet/types.rs | 21 +++--- wallet/src/lmdb_wallet.rs | 58 +++++++++++++++- wallet/tests/repost.rs | 6 +- wallet/tests/transaction.rs | 3 +- 11 files changed, 142 insertions(+), 81 deletions(-) diff --git a/src/bin/cmd/wallet_tests.rs b/src/bin/cmd/wallet_tests.rs index 4ab313d045..cc3bc49f22 100644 --- a/src/bin/cmd/wallet_tests.rs +++ b/src/bin/cmd/wallet_tests.rs @@ -408,7 +408,7 @@ mod wallet_tests { Ok(()) })?; - // Try using the self-send method + // Try using the self-send method, splitting up outputs for the fun of it let arg_vec = vec![ "grin", "wallet", @@ -423,6 +423,8 @@ mod wallet_tests { "mining", "-g", "Self love", + "-o", + "75", "-s", "smallest", "10", diff --git a/wallet/src/command.rs b/wallet/src/command.rs index d1e1529e48..35d9b9fc3f 100644 --- a/wallet/src/command.rs +++ b/wallet/src/command.rs @@ -409,7 +409,7 @@ pub fn repost( ) -> Result<(), Error> { controller::owner_single_use(wallet.clone(), |api| { let (_, txs) = api.retrieve_txs(true, Some(args.id), None)?; - let stored_tx = txs[0].get_stored_tx(); + let stored_tx = api.get_stored_tx(&txs[0])?; if stored_tx.is_none() { error!( "Transaction with id {} does not have transaction data. Not reposting.", diff --git a/wallet/src/controller.rs b/wallet/src/controller.rs index ed91f6e914..b19014a499 100644 --- a/wallet/src/controller.rs +++ b/wallet/src/controller.rs @@ -231,7 +231,10 @@ where if let Some(id_string) = params.get("id") { match id_string[0].parse() { Ok(id) => match api.retrieve_txs(true, Some(id), None) { - Ok((_, txs)) => Ok((txs[0].confirmed, txs[0].get_stored_tx())), + Ok((_, txs)) => { + let stored_tx = api.get_stored_tx(&txs[0])?; + Ok((txs[0].confirmed, stored_tx)) + } Err(e) => { error!("retrieve_stored_tx: failed with error: {}", e); Err(e) diff --git a/wallet/src/display.rs b/wallet/src/display.rs index 86e1e331c6..f5e24dbae6 100644 --- a/wallet/src/display.rs +++ b/wallet/src/display.rs @@ -178,7 +178,7 @@ pub fn txs( ) }; let tx_data = match t.tx_hex { - Some(_) => format!("Exists"), + Some(t) => format!("{}", t), None => "None".to_owned(), }; if dark_background_color_scheme { diff --git a/wallet/src/libwallet/api.rs b/wallet/src/libwallet/api.rs index 3447d41d46..2dd9f88d12 100644 --- a/wallet/src/libwallet/api.rs +++ b/wallet/src/libwallet/api.rs @@ -612,7 +612,13 @@ where num_change_outputs: usize, selection_strategy_is_use_all: bool, message: Option, - ) -> Result<(Slate, impl FnOnce(&mut W, &str) -> Result<(), Error>), Error> { + ) -> Result< + ( + Slate, + impl FnOnce(&mut W, &Transaction) -> Result<(), Error>, + ), + Error, + > { let mut w = self.wallet.lock(); w.open_with_credentials()?; let parent_key_id = match src_acct_name { @@ -653,12 +659,11 @@ where pub fn tx_lock_outputs( &mut self, slate: &Slate, - lock_fn: impl FnOnce(&mut W, &str) -> Result<(), Error>, + lock_fn: impl FnOnce(&mut W, &Transaction) -> Result<(), Error>, ) -> Result<(), Error> { let mut w = self.wallet.lock(); w.open_with_credentials()?; - let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap()); - lock_fn(&mut *w, &tx_hex)?; + lock_fn(&mut *w, &slate.tx)?; Ok(()) } @@ -668,11 +673,10 @@ where /// propagation. pub fn finalize_tx(&mut self, slate: &mut Slate) -> Result<(), Error> { let mut w = self.wallet.lock(); - let parent_key_id = w.parent_key_id(); w.open_with_credentials()?; let context = w.get_private_context(slate.id.as_bytes())?; tx::complete_tx(&mut *w, slate, &context)?; - tx::update_tx_hex(&mut *w, &parent_key_id, slate)?; + tx::update_stored_tx(&mut *w, slate)?; { let mut batch = w.batch()?; batch.delete_private_context(slate.id.as_bytes())?; @@ -706,6 +710,12 @@ where Ok(()) } + /// Retrieves a stored transaction from a TxLogEntry + pub fn get_stored_tx(&self, entry: &TxLogEntry) -> Result, Error> { + let w = self.wallet.lock(); + w.get_stored_tx(entry) + } + /// Posts a transaction to the chain pub fn post_tx(&self, tx: &Transaction, fluff: bool) -> Result<(), Error> { let tx_hex = util::to_hex(ser::ser_vec(tx).unwrap()); diff --git a/wallet/src/libwallet/internal/selection.rs b/wallet/src/libwallet/internal/selection.rs index 2a747dc879..be2160ecb7 100644 --- a/wallet/src/libwallet/internal/selection.rs +++ b/wallet/src/libwallet/internal/selection.rs @@ -14,6 +14,7 @@ //! Selection of inputs for building transactions +use crate::core::core::Transaction; use crate::core::libtx::{build, slate::Slate, tx_fee}; use crate::keychain::{Identifier, Keychain}; use crate::libwallet::error::{Error, ErrorKind}; @@ -40,7 +41,7 @@ pub fn build_send_tx_slate( ( Slate, Context, - impl FnOnce(&mut T, &str) -> Result<(), Error>, + impl FnOnce(&mut T, &Transaction) -> Result<(), Error>, ), Error, > @@ -94,42 +95,47 @@ where // Return a closure to acquire wallet lock and lock the coins being spent // so we avoid accidental double spend attempt. - let update_sender_wallet_fn = move |wallet: &mut T, tx_hex: &str| { - let mut batch = wallet.batch()?; - let log_id = batch.next_tx_log_id(&parent_key_id)?; - let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id); - t.tx_slate_id = Some(slate_id); - t.fee = Some(fee); - t.tx_hex = Some(tx_hex.to_owned()); - let mut amount_debited = 0; - t.num_inputs = lock_inputs.len(); - for id in lock_inputs { - let mut coin = batch.get(&id).unwrap(); - coin.tx_log_entry = Some(log_id); - amount_debited = amount_debited + coin.value; - batch.lock_output(&mut coin)?; - } + let update_sender_wallet_fn = move |wallet: &mut T, tx: &Transaction| { + let tx_entry = { + let mut batch = wallet.batch()?; + let log_id = batch.next_tx_log_id(&parent_key_id)?; + let mut t = TxLogEntry::new(parent_key_id.clone(), TxLogEntryType::TxSent, log_id); + t.tx_slate_id = Some(slate_id); + let filename = format!("{}.grintx", slate_id); + t.tx_hex = Some(filename); + t.fee = Some(fee); + let mut amount_debited = 0; + t.num_inputs = lock_inputs.len(); + for id in lock_inputs { + let mut coin = batch.get(&id).unwrap(); + coin.tx_log_entry = Some(log_id); + amount_debited = amount_debited + coin.value; + batch.lock_output(&mut coin)?; + } - t.amount_debited = amount_debited; - - // write the output representing our change - for (change_amount, id) in &change_amounts_derivations { - t.num_outputs += 1; - t.amount_credited += change_amount; - batch.save(OutputData { - root_key_id: parent_key_id.clone(), - key_id: id.clone(), - n_child: id.to_path().last_path_index(), - value: change_amount.clone(), - status: OutputStatus::Unconfirmed, - height: current_height, - lock_height: 0, - is_coinbase: false, - tx_log_entry: Some(log_id), - })?; - } - batch.save_tx_log_entry(t, &parent_key_id)?; - batch.commit()?; + t.amount_debited = amount_debited; + + // write the output representing our change + for (change_amount, id) in &change_amounts_derivations { + t.num_outputs += 1; + t.amount_credited += change_amount; + batch.save(OutputData { + root_key_id: parent_key_id.clone(), + key_id: id.clone(), + n_child: id.to_path().last_path_index(), + value: change_amount.clone(), + status: OutputStatus::Unconfirmed, + height: current_height, + lock_height: 0, + is_coinbase: false, + tx_log_entry: Some(log_id), + })?; + } + batch.save_tx_log_entry(t.clone(), &parent_key_id)?; + batch.commit()?; + t + }; + wallet.store_tx(&format!("{}", tx_entry.tx_slate_id.unwrap()), tx)?; Ok(()) }; diff --git a/wallet/src/libwallet/internal/tx.rs b/wallet/src/libwallet/internal/tx.rs index acbca76752..caac3359fe 100644 --- a/wallet/src/libwallet/internal/tx.rs +++ b/wallet/src/libwallet/internal/tx.rs @@ -14,11 +14,10 @@ //! Transaction building functions -use crate::util; use uuid::Uuid; +use crate::core::core::Transaction; use crate::core::libtx::slate::Slate; -use crate::core::ser; use crate::keychain::{Identifier, Keychain}; use crate::libwallet::internal::{selection, updater}; use crate::libwallet::types::{Context, NodeClient, TxLogEntryType, WalletBackend}; @@ -74,7 +73,7 @@ pub fn create_send_tx( ( Slate, Context, - impl FnOnce(&mut T, &str) -> Result<(), Error>, + impl FnOnce(&mut T, &Transaction) -> Result<(), Error>, ), Error, > @@ -200,19 +199,13 @@ where Ok((tx.confirmed, tx.tx_hex)) } -/// Update the stored hex transaction (this update needs to happen when the TX is finalised) -pub fn update_tx_hex( - wallet: &mut T, - _parent_key_id: &Identifier, - slate: &Slate, -) -> Result<(), Error> +/// Update the stored transaction (this update needs to happen when the TX is finalised) +pub fn update_stored_tx(wallet: &mut T, slate: &Slate) -> Result<(), Error> where T: WalletBackend, C: NodeClient, K: Keychain, { - let tx_hex = util::to_hex(ser::ser_vec(&slate.tx).unwrap()); - // This will ignore the parent key, so no need to specify account on the // finalize command let tx_vec = updater::retrieve_txs(wallet, None, Some(slate.id), None)?; let mut tx = None; @@ -223,14 +216,11 @@ where break; } } - let mut tx = match tx { + let tx = match tx { Some(t) => t, None => return Err(ErrorKind::TransactionDoesntExist(slate.id.to_string()))?, }; - tx.tx_hex = Some(tx_hex); - let batch = wallet.batch()?; - batch.save_tx_log_entry(tx.clone(), &tx.parent_key_id)?; - batch.commit()?; + wallet.store_tx(&format!("{}", tx.tx_slate_id.unwrap()), &slate.tx)?; Ok(()) } diff --git a/wallet/src/libwallet/types.rs b/wallet/src/libwallet/types.rs index 88978cde5f..e499981118 100644 --- a/wallet/src/libwallet/types.rs +++ b/wallet/src/libwallet/types.rs @@ -21,7 +21,6 @@ use crate::core::libtx::aggsig; use crate::core::ser; use crate::keychain::{Identifier, Keychain}; use crate::libwallet::error::{Error, ErrorKind}; -use crate::util; use crate::util::secp::key::{PublicKey, SecretKey}; use crate::util::secp::{self, pedersen, Secp256k1}; use chrono::prelude::*; @@ -102,6 +101,12 @@ where /// Gets an account path for a given label fn get_acct_path(&self, label: String) -> Result, Error>; + /// Stores a transaction + fn store_tx(&self, uuid: &str, tx: &Transaction) -> Result<(), Error>; + + /// Retrieves a stored transaction from a TxLogEntry + fn get_stored_tx(&self, entry: &TxLogEntry) -> Result, Error>; + /// Create a new write batch to update or remove output data fn batch<'a>(&'a mut self) -> Result + 'a>, Error>; @@ -156,7 +161,7 @@ where fn tx_log_iter(&self) -> Box>; /// save a tx log entry - fn save_tx_log_entry(&self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>; + fn save_tx_log_entry(&mut self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error>; /// save an account label -> path mapping fn save_acct_path(&mut self, mapping: AcctPathMapping) -> Result<(), Error>; @@ -595,6 +600,7 @@ pub struct TxLogEntry { pub amount_debited: u64, /// Fee pub fee: Option, + // TODO: rename this to 'stored_tx_file' or something for mainnet /// The transaction json itself, stored for reference or resending pub tx_hex: Option, } @@ -636,17 +642,6 @@ impl TxLogEntry { pub fn update_confirmation_ts(&mut self) { self.confirmation_ts = Some(Utc::now()); } - - /// Retrieve the stored transaction, if any - pub fn get_stored_tx(&self) -> Option { - match self.tx_hex.as_ref() { - None => None, - Some(t) => { - let tx_bin = util::from_hex(t.clone()).unwrap(); - Some(ser::deserialize::(&mut &tx_bin[..]).unwrap()) - } - } - } } /// Map of named accounts to BIP32 paths diff --git a/wallet/src/lmdb_wallet.rs b/wallet/src/lmdb_wallet.rs index 306136e2c4..acadf3651f 100644 --- a/wallet/src/lmdb_wallet.rs +++ b/wallet/src/lmdb_wallet.rs @@ -16,18 +16,29 @@ use std::cell::RefCell; use std::sync::Arc; use std::{fs, path}; +// for writing storedtransaction files +use std::fs::File; +use std::io::{Read, Write}; +use std::path::Path; + +use serde_json; + use failure::ResultExt; use uuid::Uuid; use crate::keychain::{ChildNumber, ExtKeychain, Identifier, Keychain}; use crate::store::{self, option_to_not_found, to_key, to_key_u64}; +use crate::core::core::Transaction; +use crate::core::ser; use crate::libwallet::types::*; use crate::libwallet::{internal, Error, ErrorKind}; use crate::types::{WalletConfig, WalletSeed}; +use crate::util; use crate::util::secp::pedersen; pub const DB_DIR: &'static str = "db"; +pub const TX_SAVE_DIR: &'static str = "saved_txs"; const COMMITMENT_PREFIX: u8 = 'C' as u8; const OUTPUT_PREFIX: u8 = 'o' as u8; @@ -69,10 +80,16 @@ impl LMDBBackend { let db_path = path::Path::new(&config.data_file_dir).join(DB_DIR); fs::create_dir_all(&db_path).expect("Couldn't create wallet backend directory!"); + let stored_tx_path = path::Path::new(&config.data_file_dir).join(TX_SAVE_DIR); + fs::create_dir_all(&stored_tx_path) + .expect("Couldn't create wallet backend tx storage directory!"); + let lmdb_env = Arc::new(store::new_env(db_path.to_str().unwrap().to_string())); let store = store::Store::open(lmdb_env, DB_DIR); // Make sure default wallet derivation path always exists + // as well as path (so it can be retrieved by batches to know where to store + // completed transactions, for reference let default_account = AcctPathMapping { label: "default".to_owned(), path: LMDBBackend::::default_path(), @@ -228,6 +245,37 @@ where self.db.get_ser(&acct_key).map_err(|e| e.into()) } + fn store_tx(&self, uuid: &str, tx: &Transaction) -> Result<(), Error> { + let filename = format!("{}.grintx", uuid); + let path = path::Path::new(&self.config.data_file_dir) + .join(TX_SAVE_DIR) + .join(filename); + let path_buf = Path::new(&path).to_path_buf(); + let mut stored_tx = File::create(path_buf)?; + let tx_hex = util::to_hex(ser::ser_vec(tx).unwrap());; + stored_tx.write_all(&tx_hex.as_bytes())?; + stored_tx.sync_all()?; + Ok(()) + } + + fn get_stored_tx(&self, entry: &TxLogEntry) -> Result, Error> { + let filename = match entry.tx_hex.clone() { + Some(f) => f, + None => return Ok(None), + }; + let path = path::Path::new(&self.config.data_file_dir) + .join(TX_SAVE_DIR) + .join(filename); + let tx_file = Path::new(&path).to_path_buf(); + let mut tx_f = File::open(tx_file)?; + let mut content = String::new(); + tx_f.read_to_string(&mut content)?; + let tx_bin = util::from_hex(content).unwrap(); + Ok(Some( + ser::deserialize::(&mut &tx_bin[..]).unwrap(), + )) + } + fn batch<'a>(&'a mut self) -> Result + 'a>, Error> { Ok(Box::new(Batch { _store: self, @@ -403,17 +451,21 @@ where Ok(()) } - fn save_tx_log_entry(&self, t: TxLogEntry, parent_id: &Identifier) -> Result<(), Error> { + fn save_tx_log_entry( + &mut self, + tx_in: TxLogEntry, + parent_id: &Identifier, + ) -> Result<(), Error> { let tx_log_key = to_key_u64( TX_LOG_ENTRY_PREFIX, &mut parent_id.to_bytes().to_vec(), - t.id as u64, + tx_in.id as u64, ); self.db .borrow() .as_ref() .unwrap() - .put_ser(&tx_log_key, &t)?; + .put_ser(&tx_log_key, &tx_in)?; Ok(()) } diff --git a/wallet/tests/repost.rs b/wallet/tests/repost.rs index 69f0820f7e..c2cc8ad818 100644 --- a/wallet/tests/repost.rs +++ b/wallet/tests/repost.rs @@ -148,7 +148,8 @@ fn file_repost_test_impl(test_dir: &str) -> Result<(), libwallet::Error> { // Now repost from cached wallet::controller::owner_single_use(wallet1.clone(), |api| { let (_, txs) = api.retrieve_txs(true, None, Some(slate.id))?; - api.post_tx(&txs[0].get_stored_tx().unwrap(), false)?; + let stored_tx = api.get_stored_tx(&txs[0])?; + api.post_tx(&stored_tx.unwrap(), false)?; bh += 1; Ok(()) })?; @@ -214,7 +215,8 @@ fn file_repost_test_impl(test_dir: &str) -> Result<(), libwallet::Error> { // Now repost from cached wallet::controller::owner_single_use(wallet1.clone(), |api| { let (_, txs) = api.retrieve_txs(true, None, Some(slate.id))?; - api.post_tx(&txs[0].get_stored_tx().unwrap(), false)?; + let stored_tx = api.get_stored_tx(&txs[0])?; + api.post_tx(&stored_tx.unwrap(), false)?; bh += 1; Ok(()) })?; diff --git a/wallet/tests/transaction.rs b/wallet/tests/transaction.rs index 8e725d4f42..9ad880ad56 100644 --- a/wallet/tests/transaction.rs +++ b/wallet/tests/transaction.rs @@ -252,7 +252,8 @@ fn basic_transaction_api(test_dir: &str) -> Result<(), libwallet::Error> { .iter() .find(|t| t.tx_slate_id == Some(slate.id)) .unwrap(); - sender_api.post_tx(&tx.get_stored_tx().unwrap(), false)?; + let stored_tx = sender_api.get_stored_tx(&tx)?; + sender_api.post_tx(&stored_tx.unwrap(), false)?; let (_, wallet1_info) = sender_api.retrieve_summary_info(true, 1)?; // should be mined now assert_eq!(