[2.x.x] Writeable protocol version aware (#2856)

* introduce protocol version to deserialize and read

* thread protocol version through our reader

* cleanup

* cleanup

* streaming_reader cleanup

* Pass protocol version into BinWriter to allow for version specific serialization rules.

* rustfmt

* read and write now protocol version specific

api/src/handlers/pool_api.rs

@@ -76,7 +76,7 @@ impl PoolPushHandler {
 				})
 				.and_then(move |tx_bin| {
 					// TODO - pass protocol version in via the api call?
-					let version = ProtocolVersion::default();
+					let version = ProtocolVersion::local();
 
 					ser::deserialize(&mut &tx_bin[..], version)
 						.map_err(|e| ErrorKind::RequestError(format!("Bad request: {}", e)).into())

api/src/types.rs

@@ -83,7 +83,7 @@ pub struct Status {
 impl Status {
 	pub fn from_tip_and_peers(current_tip: chain::Tip, connections: u32) -> Status {
 		Status {
-			protocol_version: ser::ProtocolVersion::default().into(),
+			protocol_version: ser::ProtocolVersion::local().into(),
 			user_agent: p2p::msg::USER_AGENT.to_string(),
 			connections: connections,
 			tip: Tip::from_tip(current_tip),

chain/src/chain.rs

@@ -647,7 +647,8 @@ impl Chain {
 	/// TODO - Write this data to disk and validate the rebuilt kernel MMR.
 	pub fn kernel_data_write(&self, reader: &mut Read) -> Result<(), Error> {
 		let mut count = 0;
-		let mut stream = StreamingReader::new(reader, ProtocolVersion::default(), Duration::from_secs(1));
+		let mut stream =
+			StreamingReader::new(reader, ProtocolVersion::local(), Duration::from_secs(1));
 		while let Ok(_kernel) = TxKernelEntry::read(&mut stream) {
 			count += 1;
 		}

core/src/core/block.rs

@@ -359,7 +359,7 @@ impl BlockHeader {
 	pub fn pre_pow(&self) -> Vec<u8> {
 		let mut header_buf = vec![];
 		{
-			let mut writer = ser::BinWriter::new(&mut header_buf);
+			let mut writer = ser::BinWriter::default(&mut header_buf);
 			self.write_pre_pow(&mut writer).unwrap();
 			self.pow.write_pre_pow(&mut writer).unwrap();
 			writer.write_u64(self.pow.nonce).unwrap();

core/src/core/hash.rs

@@ -25,7 +25,9 @@ use std::{fmt, ops};
 
 use crate::blake2::blake2b::Blake2b;
 
-use crate::ser::{self, AsFixedBytes, Error, FixedLength, Readable, Reader, Writeable, Writer};
+use crate::ser::{
+	self, AsFixedBytes, Error, FixedLength, ProtocolVersion, Readable, Reader, Writeable, Writer,
+};
 use crate::util;
 
 /// A hash consisting of all zeroes, used as a sentinel. No known preimage.
@@ -219,6 +221,10 @@ impl ser::Writer for HashWriter {
 		self.state.update(b32.as_ref());
 		Ok(())
 	}
+
+	fn protocol_version(&self) -> ProtocolVersion {
+		ProtocolVersion::local()
+	}
 }
 
 /// A trait for types that have a canonical hash

core/src/core/merkle_proof.rs

@@ -17,7 +17,7 @@
 use crate::core::hash::Hash;
 use crate::core::pmmr;
 use crate::ser;
-use crate::ser::{PMMRIndexHashable, ProtocolVersion, Readable, Reader, Writeable, Writer};
+use crate::ser::{PMMRIndexHashable, Readable, Reader, Writeable, Writer};
 use crate::util;
 
 /// Merkle proof errors.
@@ -78,14 +78,14 @@ impl MerkleProof {
 	/// Serialize the Merkle proof as a hex string (for api json endpoints)
 	pub fn to_hex(&self) -> String {
 		let mut vec = Vec::new();
-		ser::serialize(&mut vec, &self).expect("serialization failed");
+		ser::serialize_default(&mut vec, &self).expect("serialization failed");
 		util::to_hex(vec)
 	}
 
 	/// Convert hex string representation back to a Merkle proof instance
 	pub fn from_hex(hex: &str) -> Result<MerkleProof, String> {
 		let bytes = util::from_hex(hex.to_string()).unwrap();
-		let res = ser::deserialize(&mut &bytes[..], ProtocolVersion::default())
+		let res = ser::deserialize_default(&mut &bytes[..])
 			.map_err(|_| "failed to deserialize a Merkle Proof".to_string())?;
 		Ok(res)
 	}

core/src/core/transaction.rs

@@ -185,13 +185,19 @@ hashable_ord!(TxKernel);
 impl ::std::hash::Hash for TxKernel {
 	fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
 		let mut vec = Vec::new();
-		ser::serialize(&mut vec, &self).expect("serialization failed");
+		ser::serialize_default(&mut vec, &self).expect("serialization failed");
 		::std::hash::Hash::hash(&vec, state);
 	}
 }
 
 impl Writeable for TxKernel {
 	fn write<W: Writer>(&self, writer: &mut W) -> Result<(), ser::Error> {
+		// We have access to the protocol version here.
+		// This may be a protocol version based on a peer connection
+		// or the version used locally for db storage.
+		// We can handle version specific serialization here.
+		let _version = writer.protocol_version();
+
 		self.features.write(writer)?;
 		ser_multiwrite!(writer, [write_u64, self.fee], [write_u64, self.lock_height]);
 		self.excess.write(writer)?;
@@ -1165,7 +1171,7 @@ hashable_ord!(Input);
 impl ::std::hash::Hash for Input {
 	fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
 		let mut vec = Vec::new();
-		ser::serialize(&mut vec, &self).expect("serialization failed");
+		ser::serialize_default(&mut vec, &self).expect("serialization failed");
 		::std::hash::Hash::hash(&vec, state);
 	}
 }
@@ -1276,7 +1282,7 @@ hashable_ord!(Output);
 impl ::std::hash::Hash for Output {
 	fn hash<H: ::std::hash::Hasher>(&self, state: &mut H) {
 		let mut vec = Vec::new();
-		ser::serialize(&mut vec, &self).expect("serialization failed");
+		ser::serialize_default(&mut vec, &self).expect("serialization failed");
 		::std::hash::Hash::hash(&vec, state);
 	}
 }
@@ -1528,7 +1534,7 @@ mod test {
 		};
 
 		let mut vec = vec![];
-		ser::serialize(&mut vec, &kernel).expect("serialized failed");
+		ser::serialize_default(&mut vec, &kernel).expect("serialized failed");
 		let kernel2: TxKernel = ser::deserialize_default(&mut &vec[..]).unwrap();
 		assert_eq!(kernel2.features, KernelFeatures::Plain);
 		assert_eq!(kernel2.lock_height, 0);
@@ -1546,7 +1552,7 @@ mod test {
 		};
 
 		let mut vec = vec![];
-		ser::serialize(&mut vec, &kernel).expect("serialized failed");
+		ser::serialize_default(&mut vec, &kernel).expect("serialized failed");
 		let kernel2: TxKernel = ser::deserialize_default(&mut &vec[..]).unwrap();
 		assert_eq!(kernel2.features, KernelFeatures::HeightLocked);
 		assert_eq!(kernel2.lock_height, 100);

core/src/genesis.rs

@@ -288,13 +288,13 @@ pub fn genesis_main() -> core::Block {
 mod test {
 	use super::*;
 	use crate::core::hash::Hashed;
-	use crate::ser;
+	use crate::ser::{self, ProtocolVersion};
 
 	#[test]
 	fn floonet_genesis_hash() {
 		let gen_hash = genesis_floo().hash();
 		println!("floonet genesis hash: {}", gen_hash.to_hex());
-		let gen_bin = ser::ser_vec(&genesis_floo()).unwrap();
+		let gen_bin = ser::ser_vec(&genesis_floo(), ProtocolVersion(1)).unwrap();
 		println!("floonet genesis full hash: {}\n", gen_bin.hash().to_hex());
 		assert_eq!(
 			gen_hash.to_hex(),
@@ -310,7 +310,7 @@ mod test {
 	fn mainnet_genesis_hash() {
 		let gen_hash = genesis_main().hash();
 		println!("mainnet genesis hash: {}", gen_hash.to_hex());
-		let gen_bin = ser::ser_vec(&genesis_main()).unwrap();
+		let gen_bin = ser::ser_vec(&genesis_main(), ProtocolVersion(1)).unwrap();
 		println!("mainnet genesis full hash: {}\n", gen_bin.hash().to_hex());
 		assert_eq!(
 			gen_hash.to_hex(),

core/src/global.rs

@@ -33,6 +33,13 @@ use crate::util::RwLock;
 /// Define these here, as they should be developer-set, not really tweakable
 /// by users
 
+/// The default "local" protocol version for this node.
+/// We negotiate compatible versions with each peer via Hand/Shake.
+/// Note: We also use a specific (possible different) protocol version
+/// for both the backend database and MMR data files.
+/// This one is p2p layer specific.
+pub const PROTOCOL_VERSION: u32 = 1;
+
 /// Automated testing edge_bits
 pub const AUTOMATED_TESTING_MIN_EDGE_BITS: u8 = 9;
 

core/src/ser.rs

@@ -20,6 +20,7 @@
 //! `serialize` or `deserialize` functions on them as appropriate.
 
 use crate::core::hash::{DefaultHashable, Hash, Hashed};
+use crate::global::PROTOCOL_VERSION;
 use crate::keychain::{BlindingFactor, Identifier, IDENTIFIER_SIZE};
 use crate::util::read_write::read_exact;
 use crate::util::secp::constants::{
@@ -135,6 +136,9 @@ pub trait Writer {
 	/// The mode this serializer is writing in
 	fn serialization_mode(&self) -> SerializationMode;
 
+	/// Protocol version for version specific serialization rules.
+	fn protocol_version(&self) -> ProtocolVersion;
+
 	/// Writes a u8 as bytes
 	fn write_u8(&mut self, n: u8) -> Result<(), Error> {
 		self.write_fixed_bytes(&[n])
@@ -286,9 +290,10 @@ where
 #[derive(Clone, Copy, Debug, Deserialize, Eq, Ord, PartialOrd, PartialEq, Serialize)]
 pub struct ProtocolVersion(pub u32);
 
-impl Default for ProtocolVersion {
-	fn default() -> ProtocolVersion {
-		ProtocolVersion(1)
+impl ProtocolVersion {
+	/// Our default "local" protocol version.
+	pub fn local() -> ProtocolVersion {
+		ProtocolVersion(PROTOCOL_VERSION)
 	}
 }
 
@@ -346,27 +351,31 @@ pub fn deserialize<T: Readable>(
 	T::read(&mut reader)
 }
 
-/// Deserialize a Readable based on our local db version protocol.
-pub fn deserialize_db<T: Readable>(source: &mut dyn Read) -> Result<T, Error> {
-	deserialize(source, ProtocolVersion::local_db())
-}
-
-/// Deserialize a Readable based on our local "default" version protocol.
+/// Deserialize a Readable based on our default "local" protocol version.
 pub fn deserialize_default<T: Readable>(source: &mut dyn Read) -> Result<T, Error> {
-	deserialize(source, ProtocolVersion::default())
+	deserialize(source, ProtocolVersion::local())
 }
 
 /// Serializes a Writeable into any std::io::Write implementation.
-pub fn serialize<W: Writeable>(sink: &mut dyn Write, thing: &W) -> Result<(), Error> {
-	let mut writer = BinWriter { sink };
+pub fn serialize<W: Writeable>(
+	sink: &mut dyn Write,
+	version: ProtocolVersion,
+	thing: &W,
+) -> Result<(), Error> {
+	let mut writer = BinWriter::new(sink, version);
 	thing.write(&mut writer)
 }
 
+/// Serialize a Writeable according to our default "local" protocol version.
+pub fn serialize_default<W: Writeable>(sink: &mut dyn Write, thing: &W) -> Result<(), Error> {
+	serialize(sink, ProtocolVersion::local(), thing)
+}
+
 /// Utility function to serialize a writeable directly in memory using a
 /// Vec<u8>.
-pub fn ser_vec<W: Writeable>(thing: &W) -> Result<Vec<u8>, Error> {
+pub fn ser_vec<W: Writeable>(thing: &W, version: ProtocolVersion) -> Result<Vec<u8>, Error> {
 	let mut vec = vec![];
-	serialize(&mut vec, thing)?;
+	serialize(&mut vec, version, thing)?;
 	Ok(vec)
 }
 
@@ -475,32 +484,28 @@ impl<'a> StreamingReader<'a> {
 	}
 }
 
+/// Note: We use read_fixed_bytes() here to ensure our "async" I/O behaves as expected.
 impl<'a> Reader for StreamingReader<'a> {
 	fn read_u8(&mut self) -> Result<u8, Error> {
 		let buf = self.read_fixed_bytes(1)?;
 		Ok(buf[0])
 	}
-
 	fn read_u16(&mut self) -> Result<u16, Error> {
 		let buf = self.read_fixed_bytes(2)?;
 		Ok(BigEndian::read_u16(&buf[..]))
 	}
-
 	fn read_u32(&mut self) -> Result<u32, Error> {
 		let buf = self.read_fixed_bytes(4)?;
 		Ok(BigEndian::read_u32(&buf[..]))
 	}
-
 	fn read_i32(&mut self) -> Result<i32, Error> {
 		let buf = self.read_fixed_bytes(4)?;
 		Ok(BigEndian::read_i32(&buf[..]))
 	}
-
 	fn read_u64(&mut self) -> Result<u64, Error> {
 		let buf = self.read_fixed_bytes(8)?;
 		Ok(BigEndian::read_u64(&buf[..]))
 	}
-
 	fn read_i64(&mut self) -> Result<i64, Error> {
 		let buf = self.read_fixed_bytes(8)?;
 		Ok(BigEndian::read_i64(&buf[..]))
@@ -683,12 +688,18 @@ impl<T: Hashed> VerifySortedAndUnique<T> for Vec<T> {
 /// to write numbers, byte vectors, hashes, etc.
 pub struct BinWriter<'a> {
 	sink: &'a mut dyn Write,
+	version: ProtocolVersion,
 }
 
 impl<'a> BinWriter<'a> {
 	/// Wraps a standard Write in a new BinWriter
-	pub fn new(write: &'a mut dyn Write) -> BinWriter<'a> {
-		BinWriter { sink: write }
+	pub fn new(sink: &'a mut dyn Write, version: ProtocolVersion) -> BinWriter<'a> {
+		BinWriter { sink, version }
+	}
+
+	/// Constructor for BinWriter with default "local" protocol version.
+	pub fn default(sink: &'a mut dyn Write) -> BinWriter<'a> {
+		BinWriter::new(sink, ProtocolVersion::local())
 	}
 }
 
@@ -702,6 +713,10 @@ impl<'a> Writer for BinWriter<'a> {
 		self.sink.write_all(bs)?;
 		Ok(())
 	}
+
+	fn protocol_version(&self) -> ProtocolVersion {
+		self.version
+	}
 }
 
 macro_rules! impl_int {