Rust Cuckatoo for verifier and test-miner (#1558)

* cuck placeholder

* rustfmt

* cuckatoo, early days

* rustfmt

* data structures are in place, siphash key creation is consistent with @tromp

* solver in place, (not yet working)

* cuckatoo test solver working with test nonce

* rustfmt

* update solver to remove adjacency list removals

* verifier functioning

* rustfmt

* Proper error handing in Cuckatoo module, couple of tests

* modify cuckoo/cuckatoo solvers and verifiers to function identically, in advance of trait refactoring

* rustfmt

* refactor PoW context into trait, default to using cuckoo context

* rustfmt

* create macros for integer casting/unwraps

* don't instantiate structs when just verifying, add test validation vector for cuckatoo 29

* rustfmt

* don't init cuckoo structs if just validating

* test fix

* ensure BH hashing for POW is only done within miner/validators

chain/src/chain.rs

@@ -28,7 +28,7 @@ use core::core::merkle_proof::MerkleProof;
 use core::core::verifier_cache::VerifierCache;
 use core::core::{Block, BlockHeader, BlockSums, Output, OutputIdentifier, Transaction, TxKernel};
 use core::global;
-use core::pow::Difficulty;
+use core::pow::{self, Difficulty};
 use error::{Error, ErrorKind};
 use grin_store::Error::NotFoundErr;
 use pipe;
@@ -153,7 +153,7 @@ pub struct Chain {
 	block_hashes_cache: Arc<RwLock<VecDeque<Hash>>>,
 	verifier_cache: Arc<RwLock<VerifierCache>>,
 	// POW verification function
-	pow_verifier: fn(&BlockHeader, u8) -> bool,
+	pow_verifier: fn(&BlockHeader, u8) -> Result<(), pow::Error>,
 	archive_mode: bool,
 }
 
@@ -169,7 +169,7 @@ impl Chain {
 		db_env: Arc<lmdb::Environment>,
 		adapter: Arc<ChainAdapter>,
 		genesis: Block,
-		pow_verifier: fn(&BlockHeader, u8) -> bool,
+		pow_verifier: fn(&BlockHeader, u8) -> Result<(), pow::Error>,
 		verifier_cache: Arc<RwLock<VerifierCache>>,
 		archive_mode: bool,
 	) -> Result<Chain, Error> {

chain/src/pipe.rs

@@ -27,7 +27,7 @@ use core::core::verifier_cache::VerifierCache;
 use core::core::Committed;
 use core::core::{Block, BlockHeader, BlockSums};
 use core::global;
-use core::pow::Difficulty;
+use core::pow::{self, Difficulty};
 use error::{Error, ErrorKind};
 use grin_store;
 use store;
@@ -49,7 +49,7 @@ pub struct BlockContext {
 	/// The sync head
 	pub sync_head: Tip,
 	/// The POW verification function
-	pub pow_verifier: fn(&BlockHeader, u8) -> bool,
+	pub pow_verifier: fn(&BlockHeader, u8) -> Result<(), pow::Error>,
 	/// MMR sum tree states
 	pub txhashset: Arc<RwLock<txhashset::TxHashSet>>,
 	/// Recently processed blocks to avoid double-processing
@@ -439,7 +439,7 @@ fn validate_header(
 		if shift != consensus::SECOND_POW_SIZESHIFT && header.pow.scaling_difficulty != 1 {
 			return Err(ErrorKind::InvalidScaling.into());
 		}
-		if !(ctx.pow_verifier)(header, shift) {
+		if !(ctx.pow_verifier)(header, shift).is_ok() {
 			error!(
 				LOGGER,
 				"pipe: validate_header bad cuckoo shift size {}", shift
@@ -527,7 +527,8 @@ fn validate_block(
 			&prev.total_kernel_offset,
 			&prev.total_kernel_sum,
 			verifier_cache,
-		).map_err(|e| ErrorKind::InvalidBlockProof(e))?;
+		)
+		.map_err(|e| ErrorKind::InvalidBlockProof(e))?;
 	Ok(())
 }
 
@@ -567,8 +568,7 @@ fn verify_block_sums(b: &Block, ext: &mut txhashset::Extension) -> Result<(), Er
 	let offset = b.header.total_kernel_offset();
 
 	// Verify the kernel sums for the block_sums with the new block applied.
-	let (utxo_sum, kernel_sum) =
-		(block_sums, b as &Committed).verify_kernel_sums(overage, offset)?;
+	let (utxo_sum, kernel_sum) = (block_sums, b as &Committed).verify_kernel_sums(overage, offset)?;
 
 	// Save the new block_sums for the new block to the db via the batch.
 	ext.batch.save_block_sums(

core/Cargo.toml

@@ -14,6 +14,7 @@ failure = "0.1"
 failure_derive = "0.1"
 lazy_static = "1"
 lru-cache = "0.1"
+num = "0.2"
 num-bigint = "0.2"
 rand = "0.5"
 serde = "1"

core/src/core/block.rs

@@ -25,7 +25,7 @@ use std::sync::{Arc, RwLock};
 use consensus::{self, reward, REWARD};
 use core::committed::{self, Committed};
 use core::compact_block::{CompactBlock, CompactBlockBody};
-use core::hash::{Hash, HashWriter, Hashed, ZERO_HASH};
+use core::hash::{Hash, Hashed, ZERO_HASH};
 use core::verifier_cache::VerifierCache;
 use core::{
 	transaction, Commitment, Input, KernelFeatures, Output, OutputFeatures, Transaction,
@@ -278,16 +278,19 @@ impl BlockHeader {
 		Ok(())
 	}
 
-	/// Returns the pre-pow hash, as the post-pow hash
-	/// should just be the hash of the POW
-	pub fn pre_pow_hash(&self) -> Hash {
-		let mut hasher = HashWriter::default();
-		self.write_pre_pow(&mut hasher).unwrap();
-		self.pow.write_pre_pow(self.version, &mut hasher).unwrap();
-		hasher.write_u64(self.pow.nonce).unwrap();
-		let mut ret = [0; 32];
-		hasher.finalize(&mut ret);
-		Hash(ret)
+	/// Return the pre-pow, unhashed
+	/// Let the cuck(at)oo miner/verifier handle the hashing
+	/// for consistency with how this call is performed everywhere
+	/// else
+	pub fn pre_pow(&self) -> Vec<u8> {
+		let mut header_buf = vec![];
+		{
+			let mut writer = ser::BinWriter::new(&mut header_buf);
+			self.write_pre_pow(&mut writer).unwrap();
+			self.pow.write_pre_pow(self.version, &mut writer).unwrap();
+			writer.write_u64(self.pow.nonce).unwrap();
+		}
+		header_buf
 	}
 
 	/// Total difficulty accumulated by the proof of work on this header

core/src/global.rs

@@ -22,7 +22,7 @@ use consensus::{
 	DIFFICULTY_ADJUST_WINDOW, INITIAL_DIFFICULTY, MEDIAN_TIME_WINDOW, PROOFSIZE,
 	REFERENCE_SIZESHIFT,
 };
-use pow::Difficulty;
+use pow::{self, CuckooContext, Difficulty, EdgeType, PoWContext};
 /// An enum collecting sets of parameters used throughout the
 /// code wherever mining is needed. This should allow for
 /// different sets of parameters for different purposes,
@@ -93,10 +93,23 @@ impl Default for ChainTypes {
 	}
 }
 
+/// PoW test mining and verifier context
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
+pub enum PoWContextTypes {
+	/// Classic Cuckoo
+	Cuckoo,
+	/// Bleeding edge Cuckatoo
+	Cuckatoo,
+}
+
 lazy_static!{
 	/// The mining parameter mode
 	pub static ref CHAIN_TYPE: RwLock<ChainTypes> =
 			RwLock::new(ChainTypes::Mainnet);
+
+	/// PoW context type to instantiate
+	pub static ref POW_CONTEXT_TYPE: RwLock<PoWContextTypes> =
+			RwLock::new(PoWContextTypes::Cuckoo);
 }
 
 /// Set the mining mode
@@ -105,6 +118,25 @@ pub fn set_mining_mode(mode: ChainTypes) {
 	*param_ref = mode;
 }
 
+/// Return either a cuckoo context or a cuckatoo context
+/// Single change point
+pub fn create_pow_context<T>(
+	edge_bits: u8,
+	proof_size: usize,
+	easiness_pct: u32,
+	max_sols: u32,
+) -> Result<Box<impl PoWContext<T>>, pow::Error>
+where
+	T: EdgeType,
+{
+	// Perform whatever tests, configuration etc are needed to determine desired context + edge size
+	// + params
+	// Hardcode to regular cuckoo for now
+	CuckooContext::<T>::new(edge_bits, proof_size, easiness_pct, max_sols)
+	// Or switch to cuckatoo as follows:
+	// CuckatooContext::<T>::new(edge_bits, proof_size, easiness_pct, max_sols)
+}
+
 /// The minimum acceptable sizeshift
 pub fn min_sizeshift() -> u8 {
 	let param_ref = CHAIN_TYPE.read().unwrap();

core/src/pow/common.rs

@@ -0,0 +1,153 @@
+// Copyright 2018 The Grin Developers
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Common types and traits for cuckoo/cuckatoo family of solvers
+
+use blake2::blake2b::blake2b;
+use pow::error::{Error, ErrorKind};
+use pow::num::{PrimInt, ToPrimitive};
+use pow::siphash::siphash24;
+use std::hash::Hash;
+use std::io::Cursor;
+use std::ops::{BitOrAssign, Mul};
+use std::{fmt, mem};
+
+use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt};
+
+/// Operations needed for edge type (going to be u32 or u64)
+pub trait EdgeType: PrimInt + ToPrimitive + Mul + BitOrAssign + Hash {}
+impl EdgeType for u32 {}
+impl EdgeType for u64 {}
+
+/// An edge in the Cuckoo graph, simply references two u64 nodes.
+#[derive(Debug, Copy, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
+pub struct Edge<T>
+where
+	T: EdgeType,
+{
+	pub u: T,
+	pub v: T,
+}
+
+impl<T> fmt::Display for Edge<T>
+where
+	T: EdgeType,
+{
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		write!(
+			f,
+			"(u: {}, v: {})",
+			self.u.to_u64().unwrap_or(0),
+			self.v.to_u64().unwrap_or(0)
+		)
+	}
+}
+
+/// An element of an adjencency list
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct Link<T>
+where
+	T: EdgeType,
+{
+	pub next: T,
+	pub to: T,
+}
+
+impl<T> fmt::Display for Link<T>
+where
+	T: EdgeType,
+{
+	fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+		write!(
+			f,
+			"(next: {}, to: {})",
+			self.next.to_u64().unwrap_or(0),
+			self.to.to_u64().unwrap_or(0)
+		)
+	}
+}
+
+pub fn set_header_nonce(header: Vec<u8>, nonce: Option<u32>) -> Result<[u64; 4], Error> {
+	let len = header.len();
+	let mut header = header.clone();
+	if let Some(n) = nonce {
+		header.truncate(len - mem::size_of::<u32>());
+		header.write_u32::<LittleEndian>(n)?;
+	}
+	create_siphash_keys(header)
+}
+
+pub fn create_siphash_keys(header: Vec<u8>) -> Result<[u64; 4], Error> {
+	let h = blake2b(32, &[], &header);
+	let hb = h.as_bytes();
+	let mut rdr = Cursor::new(hb);
+	Ok([
+		rdr.read_u64::<LittleEndian>()?,
+		rdr.read_u64::<LittleEndian>()?,
+		rdr.read_u64::<LittleEndian>()?,
+		rdr.read_u64::<LittleEndian>()?,
+	])
+}
+
+/// Return siphash masked for type
+pub fn sipnode<T>(
+	keys: &[u64; 4],
+	edge: T,
+	edge_mask: &T,
+	uorv: u64,
+	shift: bool,
+) -> Result<T, Error>
+where
+	T: EdgeType,
+{
+	let hash_u64 = siphash24(
+		keys,
+		2 * edge.to_u64().ok_or(ErrorKind::IntegerCast)? + uorv,
+	);
+	let mut masked = hash_u64 & edge_mask.to_u64().ok_or(ErrorKind::IntegerCast)?;
+	if shift {
+		masked = masked << 1;
+		masked |= uorv;
+	}
+	Ok(T::from(masked).ok_or(ErrorKind::IntegerCast)?)
+}
+
+/// Macros to clean up integer unwrapping
+#[macro_export]
+macro_rules! to_u64 {
+	($n:expr) => {
+		$n.to_u64().ok_or(ErrorKind::IntegerCast)?
+	};
+}
+
+#[macro_export]
+macro_rules! to_u32 {
+	($n:expr) => {
+		$n.to_u64().ok_or(ErrorKind::IntegerCast)? as u32
+	};
+}
+
+#[macro_export]
+macro_rules! to_usize {
+	($n:expr) => {
+		$n.to_u64().ok_or(ErrorKind::IntegerCast)? as usize
+	};
+}
+
+#[macro_export]
+macro_rules! to_edge {
+	($n:expr) => {
+		T::from($n).ok_or(ErrorKind::IntegerCast)?
+	};
+}