From 661df38cf24b8e97aefce9f9fdea5c1e1a359d2d Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 13 Jan 2020 11:58:31 +0100 Subject: [PATCH 1/6] Use blake3 instead of blake2_simd --- Cargo.lock | 14 ++++++++------ lib/runtime-core/Cargo.toml | 10 +++++----- lib/runtime-core/build.rs | 8 ++++---- lib/runtime-core/src/cache.rs | 10 +++++----- 4 files changed, 22 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0718cf6f673..d6d4ed440be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -63,13 +63,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" [[package]] -name = "blake2b_simd" -version = "0.5.10" +name = "blake3" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +checksum = "b8c90bf4a07dd0b816948d0915ba1b110e7ce54169797feb5b39d6139a0a81ca" dependencies = [ "arrayref", "arrayvec", + "cc", + "cfg-if", "constant_time_eq", ] @@ -534,9 +536,9 @@ dependencies = [ [[package]] name = "hex" -version = "0.3.2" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "805026a5d0141ffc30abb3be3173848ad46a1b1664fe632428479619a3644d77" +checksum = "023b39be39e3a2da62a94feb433e91e8bcd37676fbc8bea371daf52b7a769a3e" [[package]] name = "indexmap" @@ -1788,7 +1790,7 @@ name = "wasmer-runtime-core" version = "0.12.0" dependencies = [ "bincode", - "blake2b_simd", + "blake3", "cc", "digest", "errno", diff --git a/lib/runtime-core/Cargo.toml b/lib/runtime-core/Cargo.toml index c4cd63baa99..c8b908894d7 100644 --- a/lib/runtime-core/Cargo.toml +++ b/lib/runtime-core/Cargo.toml @@ -13,11 +13,11 @@ edition = "2018" nix = "0.15" page_size = "0.4" wasmparser = "0.45.0" -parking_lot = "0.9" +parking_lot = "0.10.0" lazy_static = "1.4" errno = "0.2" libc = "0.2.60" -hex = "0.3" +hex = "0.4" smallvec = "0.6" bincode = "1.1" @@ -36,8 +36,8 @@ version = "1.0" version = "0.11" [dependencies.serde-bench] version = "0.0.7" -[dependencies.blake2b_simd] -version = "0.5" +[dependencies.blake3] +version = "0.1.0" [dependencies.digest] version = "0.8" @@ -45,7 +45,7 @@ version = "0.8" winapi = { version = "0.3", features = ["memoryapi"] } [build-dependencies] -blake2b_simd = "0.5" +blake3 = "0.1.0" rustc_version = "0.2" cc = "1.0" diff --git a/lib/runtime-core/build.rs b/lib/runtime-core/build.rs index 3a549e45fd4..9fd1ea457c0 100644 --- a/lib/runtime-core/build.rs +++ b/lib/runtime-core/build.rs @@ -1,13 +1,13 @@ -use blake2b_simd::blake2bp; +use blake3; use std::{env, fs, io::Write, path::PathBuf}; const WASMER_VERSION: &'static str = env!("CARGO_PKG_VERSION"); fn main() { - let mut state = blake2bp::State::new(); - state.update(WASMER_VERSION.as_bytes()); + let mut hasher = blake3::Hasher::new(); + hasher.update(WASMER_VERSION.as_bytes()); - let hasher = state.finalize(); + let hasher = hasher.finalize(); let hash_string = hasher.to_hex().as_str().to_owned(); let crate_dir = env::var("OUT_DIR").unwrap(); diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index e924cd9f5dd..dc88e99779e 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -7,7 +7,7 @@ use crate::{ module::{Module, ModuleInfo}, sys::Memory, }; -use blake2b_simd::blake2bp; +use blake3; use std::{fmt, io, mem, slice}; /// Indicates the invalid type of invalid cache file @@ -65,11 +65,11 @@ impl WasmHash { let mut first_part = [0u8; 32]; let mut second_part = [0u8; 32]; - let mut state = blake2bp::State::new(); - state.update(wasm); + let mut hasher = blake3::Hasher::new(); + hasher.update(wasm); - let hasher = state.finalize(); - let generic_array = hasher.as_bytes(); + let hash = hasher.finalize(); + let generic_array = hash.as_bytes(); first_part.copy_from_slice(&generic_array[0..32]); second_part.copy_from_slice(&generic_array[32..64]); From 6c0568798fb65b0fa78bb5e5cfc5d318bc9d5db5 Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 13 Jan 2020 12:02:07 +0100 Subject: [PATCH 2/6] Added Blake3 changes into changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fe82692b441..2f7515991a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## **[Unreleased]** +- [#1140](https://github.com/wasmerio/wasmer/pull/1140) Use [`blake3`](https://github.com/BLAKE3-team/BLAKE3) as default hashing algorithm for caching. - [#1128](https://github.com/wasmerio/wasmer/pull/1128) Fix a crash when a host function is missing and the `allow_missing_functions` flag is enabled - [#1097](https://github.com/wasmerio/wasmer/pull/1097) Move inline breakpoint outside of runtime backend - [#1095](https://github.com/wasmerio/wasmer/pull/1095) Update to cranelift 0.52. From c45819de324fe824fa66aed69e877f2f6a907068 Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 13 Jan 2020 14:59:50 +0100 Subject: [PATCH 3/6] Remove unnecessary import --- lib/runtime-core/build.rs | 1 - lib/runtime-core/src/cache.rs | 1 - 2 files changed, 2 deletions(-) diff --git a/lib/runtime-core/build.rs b/lib/runtime-core/build.rs index 9fd1ea457c0..6187b4fd15a 100644 --- a/lib/runtime-core/build.rs +++ b/lib/runtime-core/build.rs @@ -1,4 +1,3 @@ -use blake3; use std::{env, fs, io::Write, path::PathBuf}; const WASMER_VERSION: &'static str = env!("CARGO_PKG_VERSION"); diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index c3a5f50a4bd..f95bba614ec 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -3,7 +3,6 @@ //! and loaded to allow skipping compilation and fast startup. use crate::{module::ModuleInfo, sys::Memory}; -use blake3; use std::{io, mem, slice}; /// Indicates the invalid type of invalid cache file From fcb158b24355c76b80c3e7237d8e2848598cafaf Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 13 Jan 2020 15:11:02 +0100 Subject: [PATCH 4/6] Fixed cache --- lib/runtime-core/src/cache.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index f95bba614ec..127d110332b 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -58,7 +58,8 @@ impl WasmHash { /// is, in fact, a wasm module. pub fn generate(wasm: &[u8]) -> Self { let mut first_part = [0u8; 32]; - let mut second_part = [0u8; 32]; + // We don't use the second part for now, since the blake3 hash is 32 byte array + let second_part = [0u8; 32]; let mut hasher = blake3::Hasher::new(); hasher.update(wasm); @@ -67,7 +68,6 @@ impl WasmHash { let generic_array = hash.as_bytes(); first_part.copy_from_slice(&generic_array[0..32]); - second_part.copy_from_slice(&generic_array[32..64]); WasmHash(first_part, second_part) } From 1cd198a5a5e7b06875f280a44f5d6c37be9d423f Mon Sep 17 00:00:00 2001 From: Syrus Date: Tue, 14 Jan 2020 10:32:32 +0100 Subject: [PATCH 5/6] Use a 32 byte array instead of 64 one --- lib/runtime-core/src/cache.rs | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index 127d110332b..b5aa8ccfc27 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -45,10 +45,8 @@ impl From for Error { /// /// [`Cache`]: trait.Cache.html #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] -// WasmHash is made up of two 32 byte arrays instead of a 64 byte array -// because derive only works on fixed sized arrays size 32 or below -// TODO: fix this when this gets fixed by improved const generics -pub struct WasmHash([u8; 32], [u8; 32]); +// WasmHash is made up of a 32 byte array +pub struct WasmHash([u8; 32]); impl WasmHash { /// Hash a wasm module. @@ -57,18 +55,8 @@ impl WasmHash { /// This does no verification that the supplied data /// is, in fact, a wasm module. pub fn generate(wasm: &[u8]) -> Self { - let mut first_part = [0u8; 32]; - // We don't use the second part for now, since the blake3 hash is 32 byte array - let second_part = [0u8; 32]; - - let mut hasher = blake3::Hasher::new(); - hasher.update(wasm); - - let hash = hasher.finalize(); - let generic_array = hash.as_bytes(); - - first_part.copy_from_slice(&generic_array[0..32]); - WasmHash(first_part, second_part) + let hash = blake3::hash(wasm); + WasmHash(hash.into()) } /// Create the hexadecimal representation of the @@ -85,26 +73,22 @@ impl WasmHash { e )) })?; - if bytes.len() != 64 { + if bytes.len() != 32 { return Err(Error::DeserializeError( - "Prehashed keys must deserialze into exactly 64 bytes".to_string(), + "Prehashed keys must deserialze into exactly 32 bytes".to_string(), )); } use std::convert::TryInto; Ok(WasmHash( bytes[0..32].try_into().map_err(|e| { Error::DeserializeError(format!("Could not get first 32 bytes: {}", e)) - })?, - bytes[32..64].try_into().map_err(|e| { - Error::DeserializeError(format!("Could not get last 32 bytes: {}", e)) - })?, + })? )) } - pub(crate) fn into_array(self) -> [u8; 64] { - let mut total = [0u8; 64]; + pub(crate) fn into_array(self) -> [u8; 32] { + let mut total = [0u8; 32]; total[0..32].copy_from_slice(&self.0); - total[32..64].copy_from_slice(&self.1); total } } From 0d2d4369aa2692ae06f50bf6a4fb2119d6c1d672 Mon Sep 17 00:00:00 2001 From: Syrus Date: Mon, 20 Jan 2020 14:15:02 -0800 Subject: [PATCH 6/6] Fixed lint issue --- lib/runtime-core/src/cache.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index b5aa8ccfc27..76a058f8bbb 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -79,11 +79,9 @@ impl WasmHash { )); } use std::convert::TryInto; - Ok(WasmHash( - bytes[0..32].try_into().map_err(|e| { - Error::DeserializeError(format!("Could not get first 32 bytes: {}", e)) - })? - )) + Ok(WasmHash(bytes[0..32].try_into().map_err(|e| { + Error::DeserializeError(format!("Could not get first 32 bytes: {}", e)) + })?)) } pub(crate) fn into_array(self) -> [u8; 32] {