Dynamic LMDB mapsize allocation [1.1.0]

chain/Cargo.toml

@@ -12,7 +12,6 @@ edition = "2018"
 [dependencies]
 bitflags = "1"
 byteorder = "1"
-lmdb-zero = "0.4.4"
 failure = "0.1"
 failure_derive = "0.1"
 croaring = "0.3"

chain/src/chain.rs

@@ -24,7 +24,6 @@ use crate::core::core::{
 use crate::core::global;
 use crate::core::pow;
 use crate::error::{Error, ErrorKind};
-use crate::lmdb;
 use crate::pipe;
 use crate::store;
 use crate::txhashset;
@@ -160,7 +159,6 @@ impl Chain {
 	/// based on the genesis block if necessary.
 	pub fn init(
 		db_root: String,
-		db_env: Arc<lmdb::Environment>,
 		adapter: Arc<dyn ChainAdapter + Send + Sync>,
 		genesis: Block,
 		pow_verifier: fn(&BlockHeader) -> Result<(), pow::Error>,
@@ -177,7 +175,7 @@ impl Chain {
 				return Err(ErrorKind::Stopped.into());
 			}
 
-			let store = Arc::new(store::ChainStore::new(db_env)?);
+			let store = Arc::new(store::ChainStore::new(&db_root)?);
 
 			// open the txhashset, creating a new one if necessary
 			let mut txhashset = txhashset::TxHashSet::open(db_root.clone(), store.clone(), None)?;
@@ -985,6 +983,7 @@ impl Chain {
 		let mut current = self.get_header_by_height(head.height - horizon - 1)?;
 
 		let batch = self.store.batch()?;
+
 		loop {
 			// Go to the store directly so we can handle NotFoundErr robustly.
 			match self.store.get_block(&current.hash()) {

chain/src/lib.rs

@@ -23,8 +23,6 @@
 #[macro_use]
 extern crate bitflags;
 
-use lmdb_zero as lmdb;
-
 #[macro_use]
 extern crate serde_derive;
 #[macro_use]

chain/src/store.rs

@@ -18,7 +18,6 @@ use crate::core::consensus::HeaderInfo;
 use crate::core::core::hash::{Hash, Hashed};
 use crate::core::core::{Block, BlockHeader, BlockSums};
 use crate::core::pow::Difficulty;
-use crate::lmdb;
 use crate::types::Tip;
 use crate::util::secp::pedersen::Commitment;
 use croaring::Bitmap;
@@ -45,8 +44,8 @@ pub struct ChainStore {
 
 impl ChainStore {
 	/// Create new chain store
-	pub fn new(db_env: Arc<lmdb::Environment>) -> Result<ChainStore, Error> {
-		let db = store::Store::open(db_env, STORE_SUBPATH);
+	pub fn new(db_root: &str) -> Result<ChainStore, Error> {
+		let db = store::Store::new(db_root, Some(STORE_SUBPATH.clone()), None)?;
 		Ok(ChainStore { db })
 	}
 }

chain/tests/data_file_integrity.rs

@@ -26,7 +26,6 @@ use chrono::Duration;
 use grin_chain as chain;
 use grin_core as core;
 use grin_keychain as keychain;
-use grin_store as store;
 use grin_util as util;
 use std::fs;
 use std::sync::Arc;
@@ -41,10 +40,8 @@ fn setup(dir_name: &str) -> Chain {
 	global::set_mining_mode(ChainTypes::AutomatedTesting);
 	let genesis_block = pow::mine_genesis_block().unwrap();
 	let verifier_cache = Arc::new(RwLock::new(LruVerifierCache::new()));
-	let db_env = Arc::new(store::new_env(dir_name.to_string()));
 	chain::Chain::init(
 		dir_name.to_string(),
-		db_env,
 		Arc::new(NoopAdapter {}),
 		genesis_block,
 		pow::verify_size,
@@ -57,10 +54,8 @@ fn setup(dir_name: &str) -> Chain {
 
 fn reload_chain(dir_name: &str) -> Chain {
 	let verifier_cache = Arc::new(RwLock::new(LruVerifierCache::new()));
-	let db_env = Arc::new(store::new_env(dir_name.to_string()));
 	chain::Chain::init(
 		dir_name.to_string(),
-		db_env,
 		Arc::new(NoopAdapter {}),
 		genesis::genesis_dev(),
 		pow::verify_size,

chain/tests/mine_simple_chain.rs

@@ -28,7 +28,6 @@ use chrono::Duration;
 use grin_chain as chain;
 use grin_core as core;
 use grin_keychain as keychain;
-use grin_store as store;
 use grin_util as util;
 use std::fs;
 use std::sync::Arc;
@@ -41,10 +40,8 @@ fn setup(dir_name: &str, genesis: Block) -> Chain {
 	util::init_test_logger();
 	clean_output_dir(dir_name);
 	let verifier_cache = Arc::new(RwLock::new(LruVerifierCache::new()));
-	let db_env = Arc::new(store::new_env(dir_name.to_string()));
 	chain::Chain::init(
 		dir_name.to_string(),
-		db_env,
 		Arc::new(NoopAdapter {}),
 		genesis,
 		pow::verify_size,
@@ -532,11 +529,9 @@ where
 fn actual_diff_iter_output() {
 	global::set_mining_mode(ChainTypes::AutomatedTesting);
 	let genesis_block = pow::mine_genesis_block().unwrap();
-	let db_env = Arc::new(store::new_env(".grin".to_string()));
 	let verifier_cache = Arc::new(RwLock::new(LruVerifierCache::new()));
 	let chain = chain::Chain::init(
 		"../.grin".to_string(),
-		db_env,
 		Arc::new(NoopAdapter {}),
 		genesis_block,
 		pow::verify_size,

chain/tests/store_indices.rs

@@ -23,7 +23,6 @@ use env_logger;
 use grin_chain as chain;
 use grin_core as core;
 use grin_keychain as keychain;
-use grin_store as store;
 use std::fs;
 use std::sync::Arc;
 
@@ -53,9 +52,8 @@ fn test_various_store_indices() {
 
 	let keychain = ExtKeychain::from_random_seed(false).unwrap();
 	let key_id = ExtKeychainPath::new(1, 1, 0, 0, 0).to_identifier();
-	let db_env = Arc::new(store::new_env(chain_dir.to_string()));
 
-	let chain_store = Arc::new(chain::store::ChainStore::new(db_env).unwrap());
+	let chain_store = Arc::new(chain::store::ChainStore::new(chain_dir).unwrap());
 
 	global::set_mining_mode(ChainTypes::AutomatedTesting);
 	let genesis = pow::mine_genesis_block().unwrap();

chain/tests/test_coinbase_maturity.rs

@@ -26,7 +26,6 @@ use env_logger;
 use grin_chain as chain;
 use grin_core as core;
 use grin_keychain as keychain;
-use grin_store as store;
 use grin_util as util;
 use std::fs;
 use std::sync::Arc;
@@ -45,10 +44,8 @@ fn test_coinbase_maturity() {
 
 	let verifier_cache = Arc::new(RwLock::new(LruVerifierCache::new()));
 
-	let db_env = Arc::new(store::new_env(".grin".to_string()));
 	let chain = chain::Chain::init(
 		".grin".to_string(),
-		db_env,
 		Arc::new(NoopAdapter {}),
 		genesis_block,
 		pow::verify_size,

chain/tests/test_txhashset.rs

@@ -15,7 +15,6 @@
 use grin_chain as chain;
 use grin_core as core;
 
-use grin_store as store;
 use grin_util as util;
 
 use std::collections::HashSet;
@@ -38,8 +37,7 @@ fn clean_output_dir(dir_name: &str) {
 fn test_unexpected_zip() {
 	let db_root = format!(".grin_txhashset_zip");
 	clean_output_dir(&db_root);
-	let db_env = Arc::new(store::new_env(db_root.clone()));
-	let chain_store = ChainStore::new(db_env).unwrap();
+	let chain_store = ChainStore::new(&db_root).unwrap();
 	let store = Arc::new(chain_store);
 	txhashset::TxHashSet::open(db_root.clone(), store.clone(), None).unwrap();
 	let head = BlockHeader::default();

p2p/Cargo.toml

@@ -13,7 +13,6 @@ edition = "2018"
 bitflags = "1"
 bytes = "0.4"
 enum_primitive = "0.1"
-lmdb-zero = "0.4.4"
 net2 = "0.2"
 num = "0.1"
 rand = "0.5"

p2p/src/lib.rs

@@ -25,7 +25,6 @@ extern crate bitflags;
 
 #[macro_use]
 extern crate enum_primitive;
-use lmdb_zero as lmdb;
 
 #[macro_use]
 extern crate grin_core as core;

p2p/src/serv.rs

@@ -18,8 +18,6 @@ use std::sync::Arc;
 use std::time::Duration;
 use std::{io, thread};
 
-use crate::lmdb;
-
 use crate::core::core;
 use crate::core::core::hash::Hash;
 use crate::core::global;
@@ -48,7 +46,7 @@ pub struct Server {
 impl Server {
 	/// Creates a new idle p2p server with no peers
 	pub fn new(
-		db_env: Arc<lmdb::Environment>,
+		db_root: &str,
 		capab: Capabilities,
 		config: P2PConfig,
 		adapter: Arc<dyn ChainAdapter>,
@@ -59,7 +57,7 @@ impl Server {
 			config: config.clone(),
 			capabilities: capab,
 			handshake: Arc::new(Handshake::new(genesis, config.clone())),
-			peers: Arc::new(Peers::new(PeerStore::new(db_env)?, adapter, config)),
+			peers: Arc::new(Peers::new(PeerStore::new(db_root)?, adapter, config)),
 			stop_state,
 		})
 	}

p2p/src/store.rs

@@ -17,9 +17,6 @@
 use chrono::Utc;
 use num::FromPrimitive;
 use rand::{thread_rng, Rng};
-use std::sync::Arc;
-
-use crate::lmdb;
 
 use crate::core::ser::{self, Readable, Reader, Writeable, Writer};
 use crate::types::{Capabilities, PeerAddr, ReasonForBan};
@@ -117,8 +114,8 @@ pub struct PeerStore {
 
 impl PeerStore {
 	/// Instantiates a new peer store under the provided root path.
-	pub fn new(db_env: Arc<lmdb::Environment>) -> Result<PeerStore, Error> {
-		let db = grin_store::Store::open(db_env, STORE_SUBPATH);
+	pub fn new(db_root: &str) -> Result<PeerStore, Error> {
+		let db = grin_store::Store::new(db_root, Some(STORE_SUBPATH), None)?;
 		Ok(PeerStore { db: db })
 	}
 

p2p/tests/peer_handshake.rs

@@ -15,7 +15,6 @@
 use grin_core as core;
 use grin_p2p as p2p;
 
-use grin_store as store;
 use grin_util as util;
 use grin_util::{Mutex, StopState};
 
@@ -50,10 +49,9 @@ fn peer_handshake() {
 		..p2p::P2PConfig::default()
 	};
 	let net_adapter = Arc::new(p2p::DummyAdapter {});
-	let db_env = Arc::new(store::new_env(".grin".to_string()));
 	let server = Arc::new(
 		p2p::Server::new(
-			db_env,
+			".grin",
 			p2p::Capabilities::UNKNOWN,
 			p2p_config.clone(),
 			net_adapter.clone(),

pool/tests/common.rs

@@ -29,25 +29,23 @@ use grin_chain as chain;
 use grin_core as core;
 use grin_keychain as keychain;
 use grin_pool as pool;
-use grin_store as store;
 use grin_util as util;
 use std::collections::HashSet;
 use std::fs;
 use std::sync::Arc;
 
 #[derive(Clone)]
 pub struct ChainAdapter {
-	pub store: Arc<ChainStore>,
+	pub store: Arc<RwLock<ChainStore>>,
 	pub utxo: Arc<RwLock<HashSet<Commitment>>>,
 }
 
 impl ChainAdapter {
 	pub fn init(db_root: String) -> Result<ChainAdapter, String> {
 		let target_dir = format!("target/{}", db_root);
-		let db_env = Arc::new(store::new_env(target_dir.clone()));
-		let chain_store =
-			ChainStore::new(db_env).map_err(|e| format!("failed to init chain_store, {:?}", e))?;
-		let store = Arc::new(chain_store);
+		let chain_store = ChainStore::new(&target_dir)
+			.map_err(|e| format!("failed to init chain_store, {:?}", e))?;
+		let store = Arc::new(RwLock::new(chain_store));
 		let utxo = Arc::new(RwLock::new(HashSet::new()));
 
 		Ok(ChainAdapter { store, utxo })
@@ -56,7 +54,8 @@ impl ChainAdapter {
 	pub fn update_db_for_block(&self, block: &Block) {
 		let header = &block.header;
 		let tip = Tip::from_header(header);
-		let batch = self.store.batch().unwrap();
+		let mut s = self.store.write();
+		let batch = s.batch().unwrap();
 
 		batch.save_block_header(header).unwrap();
 		batch.save_head(&tip).unwrap();
@@ -102,20 +101,20 @@ impl ChainAdapter {
 
 impl BlockChain for ChainAdapter {
 	fn chain_head(&self) -> Result<BlockHeader, PoolError> {
-		self.store
-			.head_header()
+		let s = self.store.read();
+		s.head_header()
 			.map_err(|_| PoolError::Other(format!("failed to get chain head")))
 	}
 
 	fn get_block_header(&self, hash: &Hash) -> Result<BlockHeader, PoolError> {
-		self.store
-			.get_block_header(hash)
+		let s = self.store.read();
+		s.get_block_header(hash)
 			.map_err(|_| PoolError::Other(format!("failed to get block header")))
 	}
 
 	fn get_block_sums(&self, hash: &Hash) -> Result<BlockSums, PoolError> {
-		self.store
-			.get_block_sums(hash)
+		let s = self.store.read();
+		s.get_block_sums(hash)
 			.map_err(|_| PoolError::Other(format!("failed to get block sums")))
 	}
 

servers/src/grin/server.rs

@@ -42,7 +42,6 @@ use crate::mining::test_miner::Miner;
 use crate::p2p;
 use crate::p2p::types::PeerAddr;
 use crate::pool;
-use crate::store;
 use crate::util::file::get_first_line;
 use crate::util::{Mutex, RwLock, StopState};
 
@@ -175,10 +174,8 @@ impl Server {
 
 		info!("Starting server, genesis block: {}", genesis.hash());
 
-		let db_env = Arc::new(store::new_env(config.db_root.clone()));
 		let shared_chain = Arc::new(chain::Chain::init(
 			config.db_root.clone(),
-			db_env,
 			chain_adapter.clone(),
 			genesis.clone(),
 			pow::verify_size,
@@ -197,13 +194,8 @@ impl Server {
 			config.clone(),
 		));
 
-		let peer_db_env = Arc::new(store::new_named_env(
-			config.db_root.clone(),
-			"peer".into(),
-			config.p2p_config.peer_max_count,
-		));
 		let p2p_server = Arc::new(p2p::Server::new(
-			peer_db_env,
+			&config.db_root,
 			config.p2p_config.capabilities,
 			config.p2p_config.clone(),
 			net_adapter.clone(),