Skip to content

Commit

Permalink
Merge pull request #4372 from wasmerio/use-faster-hash
Browse files Browse the repository at this point in the history
use faster hash algorithm when loading and saving a module
  • Loading branch information
syrusakbary authored Dec 22, 2023

Unverified

No user is associated with the committer email.
2 parents 521b5d9 + f5e4871 commit 1b803c9
Showing 9 changed files with 40 additions and 43 deletions.
7 changes: 7 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions lib/wasix/Cargo.toml
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@ repository.workspace = true
rust-version.workspace = true

[dependencies]
xxhash-rust = { version = "0.8.8", features = ["xxh64"] }
rusty_pool = { version = "0.7.0", optional = true }
cfg-if = "1.0"
thiserror = "1"
6 changes: 3 additions & 3 deletions lib/wasix/src/bin_factory/binary_package.rs
Original file line number Diff line number Diff line change
@@ -52,7 +52,7 @@ impl BinaryPackageCommand {
}

pub fn hash(&self) -> &ModuleHash {
self.hash.get_or_init(|| ModuleHash::sha256(self.atom()))
self.hash.get_or_init(|| ModuleHash::hash(self.atom()))
}
}

@@ -144,9 +144,9 @@ impl BinaryPackage {
pub fn hash(&self) -> ModuleHash {
*self.hash.get_or_init(|| {
if let Some(entry) = self.entrypoint_bytes() {
ModuleHash::sha256(entry)
ModuleHash::hash(entry)
} else {
ModuleHash::sha256(self.package_name.as_bytes())
ModuleHash::hash(self.package_name.as_bytes())
}
})
}
2 changes: 1 addition & 1 deletion lib/wasix/src/runtime/mod.rs
Original file line number Diff line number Diff line change
@@ -116,7 +116,7 @@ pub async fn load_module(
module_cache: &(dyn ModuleCache + Send + Sync),
wasm: &[u8],
) -> Result<Module, anyhow::Error> {
let hash = ModuleHash::sha256(wasm);
let hash = ModuleHash::hash(wasm);
let result = module_cache.load(hash, engine).await;

match result {
6 changes: 3 additions & 3 deletions lib/wasix/src/runtime/module_cache/fallback.rs
Original file line number Diff line number Diff line change
@@ -179,7 +179,7 @@ mod tests {
async fn load_from_primary() {
let engine = Engine::default();
let module = Module::new(&engine, ADD_WAT).unwrap();
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);
let primary = SharedCache::default();
let fallback = SharedCache::default();
primary.save(key, &engine, &module).await.unwrap();
@@ -204,7 +204,7 @@ mod tests {
async fn loading_from_fallback_also_populates_primary() {
let engine = Engine::default();
let module = Module::new(&engine, ADD_WAT).unwrap();
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);
let primary = SharedCache::default();
let fallback = SharedCache::default();
fallback.save(key, &engine, &module).await.unwrap();
@@ -230,7 +230,7 @@ mod tests {
async fn saving_will_update_both() {
let engine = Engine::default();
let module = Module::new(&engine, ADD_WAT).unwrap();
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);
let primary = SharedCache::default();
let fallback = SharedCache::default();
let cache = FallbackCache::new(&primary, &fallback);
10 changes: 5 additions & 5 deletions lib/wasix/src/runtime/module_cache/filesystem.rs
Original file line number Diff line number Diff line change
@@ -182,7 +182,7 @@ mod tests {
let engine = Engine::default();
let module = Module::new(&engine, ADD_WAT).unwrap();
let cache = FileSystemCache::new(temp.path());
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);
let expected_path = cache.path(key, engine.deterministic_id());

cache.save(key, &engine, &module).await.unwrap();
@@ -198,7 +198,7 @@ mod tests {
let cache_dir = temp.path().join("this").join("doesn't").join("exist");
assert!(!cache_dir.exists());
let cache = FileSystemCache::new(&cache_dir);
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);

cache.save(key, &engine, &module).await.unwrap();

@@ -209,7 +209,7 @@ mod tests {
async fn missing_file() {
let temp = TempDir::new().unwrap();
let engine = Engine::default();
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);
let cache = FileSystemCache::new(temp.path());

let err = cache.load(key, &engine).await.unwrap_err();
@@ -222,7 +222,7 @@ mod tests {
let temp = TempDir::new().unwrap();
let engine = Engine::default();
let module = Module::new(&engine, ADD_WAT).unwrap();
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);
let cache = FileSystemCache::new(temp.path());
let expected_path = cache.path(key, engine.deterministic_id());
std::fs::create_dir_all(expected_path.parent().unwrap()).unwrap();
@@ -245,7 +245,7 @@ mod tests {
let temp = TempDir::new().unwrap();
let engine = Engine::default();
let module = Module::new(&engine, ADD_WAT).unwrap();
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);
let cache = FileSystemCache::new(temp.path());
let expected_path = cache.path(key, engine.deterministic_id());
std::fs::create_dir_all(expected_path.parent().unwrap()).unwrap();
2 changes: 1 addition & 1 deletion lib/wasix/src/runtime/module_cache/shared.rs
Original file line number Diff line number Diff line change
@@ -64,7 +64,7 @@ mod tests {
let engine = Engine::default();
let module = Module::new(&engine, ADD_WAT).unwrap();
let cache = SharedCache::default();
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);

cache.save(key, &engine, &module).await.unwrap();
let round_tripped = cache.load(key, &engine).await.unwrap();
2 changes: 1 addition & 1 deletion lib/wasix/src/runtime/module_cache/thread_local.rs
Original file line number Diff line number Diff line change
@@ -70,7 +70,7 @@ mod tests {
let engine = Engine::default();
let module = Module::new(&engine, ADD_WAT).unwrap();
let cache = ThreadLocalCache::default();
let key = ModuleHash::from_bytes([0; 32]);
let key = ModuleHash::from_bytes([0; 8]);

cache.save(key, &engine, &module).await.unwrap();
let round_tripped = cache.load(key, &engine).await.unwrap();
47 changes: 18 additions & 29 deletions lib/wasix/src/runtime/module_cache/types.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
use std::hash::Hash;
use std::{
fmt::{self, Debug, Display, Formatter},
ops::Deref,
path::PathBuf,
};

use sha2::{Digest, Sha256};
use wasmer::{Engine, Module};

use crate::runtime::module_cache::FallbackCache;
@@ -121,34 +121,34 @@ impl CacheError {
}
}

/// The SHA-256 hash of a WebAssembly module.
/// The XXHash hash of a WebAssembly module.
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct ModuleHash([u8; 32]);
pub struct ModuleHash([u8; 8]);

impl ModuleHash {
/// Create a new [`ModuleHash`] from the raw SHA-256 hash.
pub fn from_bytes(key: [u8; 32]) -> Self {
/// Create a new [`ModuleHash`] from the raw XXHash hash.
pub fn from_bytes(key: [u8; 8]) -> Self {
ModuleHash(key)
}

/// Parse a sha256 hash from a hex-encoded string.
/// Parse a XXHash hash from a hex-encoded string.
pub fn parse_hex(hex_str: &str) -> Result<Self, hex::FromHexError> {
let mut hash = [0_u8; 32];
let mut hash = [0_u8; 8];
hex::decode_to_slice(hex_str, &mut hash)?;
Ok(Self(hash))
}

/// Generate a new [`ModuleCache`] based on the SHA-256 hash of some bytes.
pub fn sha256(wasm: impl AsRef<[u8]>) -> Self {
/// Generate a new [`ModuleCache`] based on the XXHash hash of some bytes.
pub fn hash(wasm: impl AsRef<[u8]>) -> Self {
let wasm = wasm.as_ref();

let mut hasher = Sha256::default();
hasher.update(wasm);
ModuleHash::from_bytes(hasher.finalize().into())
let hash = xxhash_rust::xxh64::xxh64(wasm, 0);

Self(hash.to_ne_bytes())
}

/// Get the raw SHA-256 hash.
pub fn as_bytes(self) -> [u8; 32] {
/// Get the raw XXHash hash.
pub fn as_bytes(self) -> [u8; 8] {
self.0
}
}
@@ -174,30 +174,19 @@ mod tests {

#[test]
fn key_is_displayed_as_hex() {
let key = ModuleHash::from_bytes([
0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d,
0x0e, 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16, 0x17, 0x18, 0x19, 0x1a, 0x1b,
0x1c, 0x1d, 0x1e, 0x1f,
]);
let key = ModuleHash::from_bytes([0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07]);

let repr = key.to_string();

assert_eq!(
repr,
"000102030405060708090A0B0C0D0E0F101112131415161718191A1B1C1D1E1F"
);
assert_eq!(repr, "0001020304050607");
}

#[test]
fn module_hash_is_just_sha_256() {
let wasm = b"\0asm...";
let raw = [
0x5a, 0x39, 0xfe, 0xef, 0x52, 0xe5, 0x3b, 0x8f, 0xfe, 0xdf, 0xd7, 0x05, 0x15, 0x56,
0xec, 0x10, 0x5e, 0xd8, 0x69, 0x82, 0xf1, 0x22, 0xa0, 0x5d, 0x27, 0x28, 0xd9, 0x67,
0x78, 0xe4, 0xeb, 0x96,
];
let raw = [0x0c, 0xc7, 0x88, 0x60, 0xd4, 0x14, 0x71, 0x4c];

let hash = ModuleHash::sha256(wasm);
let hash = ModuleHash::hash(wasm);

assert_eq!(hash.as_bytes(), raw);
}

0 comments on commit 1b803c9

Please sign in to comment.