From b746a881382b3b84bf9467583e6ffe2d5939757e Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Mon, 8 Jul 2019 10:50:28 -0700 Subject: [PATCH 1/4] Update cache key generation to use backend compiler name too --- CHANGELOG.md | 1 + lib/runtime-core/src/backend.rs | 50 +++++++++++++++++++++++++++++++++ lib/runtime-core/src/cache.rs | 4 ++- src/bin/wasmer.rs | 40 ++------------------------ 4 files changed, 56 insertions(+), 39 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 529bd97e55a..514bc58b14d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ Blocks of changes will separated by version increments. ## **[Unreleased]** ## 0.5.4 +- [#536](https://github.com/wasmerio/wasmer/pull/536) Update cache to use compiler backend name in cache key - [#529](https://github.com/wasmerio/wasmer/pull/529) Updates the Wasm Interface library, which is used by wapm, with bug fixes and error message improvements ## 0.5.3 diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index ed8733bdef9..5af052b5a7e 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -29,6 +29,56 @@ pub enum Backend { LLVM, } +impl Backend { + pub fn variants() -> &'static [&'static str] { + &[ + "cranelift", + #[cfg(feature = "backend:singlepass")] + "singlepass", + #[cfg(feature = "backend:llvm")] + "llvm", + ] + } + + /// stable string representation of the backend + /// can be used as part of a cache key, for example + pub fn to_string(&self) -> &'static str { + match self { + Backend::Cranelift => "cranelift", + Backend::Singlepass => "singlepass", + Backend::LLVM => "llvm", + } + } +} + +impl std::str::FromStr for Backend { + type Err = String; + fn from_str(s: &str) -> Result { + match s.to_lowercase().as_str() { + "singlepass" => Ok(Backend::Singlepass), + "cranelift" => Ok(Backend::Cranelift), + "llvm" => Ok(Backend::LLVM), + _ => Err(format!("The backend {} doesn't exist", s)), + } + } +} + +#[cfg(test)] +mod backend_test { + use super::*; + use std::str::FromStr; + + #[test] + fn str_repr_matches() { + // if this test breaks, think hard about why it's breaking + // can we avoid having these be different? + + for &backend in &[Backend::Cranelift, Backend::LLVM, Backend::Singlepass] { + assert_eq!(backend, Backend::from_str(backend.to_string()).unwrap()); + } + } +} + /// This type cannot be constructed from /// outside the runtime crate. pub struct Token { diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index 93e14b0f9b3..eaaf7a8daea 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -1,4 +1,5 @@ use crate::{ + backend::Backend, module::{Module, ModuleInfo}, sys::Memory, }; @@ -41,12 +42,13 @@ impl WasmHash { /// # Note: /// This does no verification that the supplied data /// is, in fact, a wasm module. - pub fn generate(wasm: &[u8]) -> Self { + pub fn generate(wasm: &[u8], backend: Backend) -> Self { let mut first_part = [0u8; 32]; let mut second_part = [0u8; 32]; let mut state = blake2bp::State::new(); state.update(wasm); + state.update(backend.to_string().as_bytes()); let hasher = state.finalize(); let generic_array = hasher.as_bytes(); diff --git a/src/bin/wasmer.rs b/src/bin/wasmer.rs index 7d5a453458b..d1dbab75e6f 100644 --- a/src/bin/wasmer.rs +++ b/src/bin/wasmer.rs @@ -23,7 +23,7 @@ use wasmer_runtime::{ }; use wasmer_runtime_core::{ self, - backend::{Compiler, CompilerConfig, MemoryBoundCheckMode}, + backend::{Backend, Compiler, CompilerConfig, MemoryBoundCheckMode}, loader::{Instance as LoadedInstance, LocalLoader}, }; #[cfg(feature = "backend:singlepass")] @@ -153,42 +153,6 @@ impl FromStr for LoaderName { } } -#[allow(dead_code)] -#[derive(Debug, Eq, PartialEq)] -enum Backend { - Cranelift, - Singlepass, - LLVM, -} - -impl Backend { - pub fn variants() -> &'static [&'static str] { - &[ - "cranelift", - #[cfg(feature = "backend:singlepass")] - "singlepass", - #[cfg(feature = "backend:llvm")] - "llvm", - ] - } -} - -impl FromStr for Backend { - type Err = String; - fn from_str(s: &str) -> Result { - match s.to_lowercase().as_str() { - "singlepass" => Ok(Backend::Singlepass), - "cranelift" => Ok(Backend::Cranelift), - "llvm" => Ok(Backend::LLVM), - // "llvm" => Err( - // "The LLVM backend option is not enabled by default due to binary size constraints" - // .to_string(), - // ), - _ => Err(format!("The backend {} doesn't exist", s)), - } - } -} - #[derive(Debug, StructOpt)] enum Cache { /// Clear the cache @@ -388,7 +352,7 @@ fn execute_wasm(options: &Run) -> Result<(), String> { // We generate a hash for the given binary, so we can use it as key // for the Filesystem cache - let hash = WasmHash::generate(&wasm_binary); + let hash = WasmHash::generate(&wasm_binary, options.backend); let wasmer_cache_dir = get_cache_dir(); From f45d52301227de7a969422bf1e495e3b4bdc029a Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Mon, 8 Jul 2019 11:11:27 -0700 Subject: [PATCH 2/4] update tests to use backend as in cache key gen --- lib/runtime/benches/nginx.rs | 7 +++++-- lib/runtime/src/cache.rs | 6 ++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/runtime/benches/nginx.rs b/lib/runtime/benches/nginx.rs index 1466addb1aa..9c80ae7683b 100644 --- a/lib/runtime/benches/nginx.rs +++ b/lib/runtime/benches/nginx.rs @@ -6,6 +6,7 @@ use wasmer_runtime::{ cache::{Cache, FileSystemCache, WasmHash}, compile, validate, }; +use wasmer_runtime_core::backend::Backend; static NGINX_WASM: &'static [u8] = include_bytes!("../../../examples/nginx/nginx.wasm"); @@ -18,7 +19,9 @@ fn load_module(hash: WasmHash, cache: &impl Cache) { } fn hashing_benchmark(c: &mut Criterion) { - c.bench_function("nginx HASH", |b| b.iter(|| WasmHash::generate(NGINX_WASM))); + c.bench_function("nginx HASH", |b| { + b.iter(|| WasmHash::generate(NGINX_WASM, Backend::Cranelift)) + }); } fn validate_benchmark(c: &mut Criterion) { @@ -36,7 +39,7 @@ fn load_benchmark(c: &mut Criterion) { FileSystemCache::new(tempdir.path()).expect("unable to create file system cache") }; let module = compile(NGINX_WASM).unwrap(); - let wasm_hash = WasmHash::generate(NGINX_WASM); + let wasm_hash = WasmHash::generate(NGINX_WASM, Backend::Cranelift); cache .store(wasm_hash, module) .expect("unable to store into cache"); diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index b6d1d779bbb..9b452c70a29 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -20,6 +20,7 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash, WASMER_VERSION_H /// /// ```rust /// use wasmer_runtime::cache::{Cache, FileSystemCache, WasmHash}; +/// use wasmer_runtime_core::backend::Backend; /// /// # use wasmer_runtime::{Module, error::CacheError}; /// fn store_module(module: Module) -> Result { @@ -28,7 +29,7 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash, WASMER_VERSION_H /// // corrupted or tampered with. /// let mut fs_cache = unsafe { FileSystemCache::new("some/directory/goes/here")? }; /// // Compute a key for a given WebAssembly binary -/// let key = WasmHash::generate(&[]); +/// let key = WasmHash::generate(&[], Backend::Cranelift); /// // Store a module into the cache given a key /// fs_cache.store(key, module.clone())?; /// Ok(module) @@ -119,6 +120,7 @@ mod tests { use super::*; use std::env; + use wasmer_runtime_core::backend::Backend; #[test] fn test_file_system_cache_run() { @@ -147,7 +149,7 @@ mod tests { .unwrap() }; // store module - let key = WasmHash::generate(&wasm); + let key = WasmHash::generate(&wasm, Backend::Cranelift); fs_cache.store(key, module.clone()).unwrap(); // load module From 9a3fd82a4b879dce5000c4c9419288f21a65cc86 Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Mon, 8 Jul 2019 12:22:17 -0700 Subject: [PATCH 3/4] Keep WashHash::generate function the same --- lib/runtime-core/src/backend.rs | 6 ++++++ lib/runtime-core/src/cache.rs | 17 +++++++++++++++-- lib/runtime/benches/nginx.rs | 7 ++----- src/bin/wasmer.rs | 2 +- 4 files changed, 24 insertions(+), 8 deletions(-) diff --git a/lib/runtime-core/src/backend.rs b/lib/runtime-core/src/backend.rs index 5af052b5a7e..2c51cb5761a 100644 --- a/lib/runtime-core/src/backend.rs +++ b/lib/runtime-core/src/backend.rs @@ -51,6 +51,12 @@ impl Backend { } } +impl Default for Backend { + fn default() -> Self { + Backend::Cranelift + } +} + impl std::str::FromStr for Backend { type Err = String; fn from_str(s: &str) -> Result { diff --git a/lib/runtime-core/src/cache.rs b/lib/runtime-core/src/cache.rs index eaaf7a8daea..8f1c82db2b4 100644 --- a/lib/runtime-core/src/cache.rs +++ b/lib/runtime-core/src/cache.rs @@ -37,12 +37,25 @@ impl From for Error { pub struct WasmHash([u8; 32], [u8; 32]); impl WasmHash { - /// Hash a wasm module. + /// Hash a wasm module for the default compiler backend. + /// + /// See also: `WasmHash::generate_for_backend`. /// /// # Note: /// This does no verification that the supplied data /// is, in fact, a wasm module. - pub fn generate(wasm: &[u8], backend: Backend) -> Self { + pub fn generate(wasm: &[u8]) -> Self { + WasmHash::generate_for_backend(wasm, Backend::default()) + } + + /// Hash a wasm module for a specific compiler backend. + /// This allows multiple cache entries containing the same compiled + /// module with different compiler backends. + /// + /// # Note: + /// This function also does no verification that the supplied data + /// is a wasm module + pub fn generate_for_backend(wasm: &[u8], backend: Backend) -> Self { let mut first_part = [0u8; 32]; let mut second_part = [0u8; 32]; diff --git a/lib/runtime/benches/nginx.rs b/lib/runtime/benches/nginx.rs index 9c80ae7683b..1466addb1aa 100644 --- a/lib/runtime/benches/nginx.rs +++ b/lib/runtime/benches/nginx.rs @@ -6,7 +6,6 @@ use wasmer_runtime::{ cache::{Cache, FileSystemCache, WasmHash}, compile, validate, }; -use wasmer_runtime_core::backend::Backend; static NGINX_WASM: &'static [u8] = include_bytes!("../../../examples/nginx/nginx.wasm"); @@ -19,9 +18,7 @@ fn load_module(hash: WasmHash, cache: &impl Cache) { } fn hashing_benchmark(c: &mut Criterion) { - c.bench_function("nginx HASH", |b| { - b.iter(|| WasmHash::generate(NGINX_WASM, Backend::Cranelift)) - }); + c.bench_function("nginx HASH", |b| b.iter(|| WasmHash::generate(NGINX_WASM))); } fn validate_benchmark(c: &mut Criterion) { @@ -39,7 +36,7 @@ fn load_benchmark(c: &mut Criterion) { FileSystemCache::new(tempdir.path()).expect("unable to create file system cache") }; let module = compile(NGINX_WASM).unwrap(); - let wasm_hash = WasmHash::generate(NGINX_WASM, Backend::Cranelift); + let wasm_hash = WasmHash::generate(NGINX_WASM); cache .store(wasm_hash, module) .expect("unable to store into cache"); diff --git a/src/bin/wasmer.rs b/src/bin/wasmer.rs index d1dbab75e6f..297d0199b78 100644 --- a/src/bin/wasmer.rs +++ b/src/bin/wasmer.rs @@ -352,7 +352,7 @@ fn execute_wasm(options: &Run) -> Result<(), String> { // We generate a hash for the given binary, so we can use it as key // for the Filesystem cache - let hash = WasmHash::generate(&wasm_binary, options.backend); + let hash = WasmHash::generate_for_backend(&wasm_binary, options.backend); let wasmer_cache_dir = get_cache_dir(); From 62798fd96650fde260697df8205bcc88e0e64e7c Mon Sep 17 00:00:00 2001 From: Mark McCaskey Date: Mon, 8 Jul 2019 13:05:25 -0700 Subject: [PATCH 4/4] fix test fix doc test --- lib/runtime/src/cache.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/runtime/src/cache.rs b/lib/runtime/src/cache.rs index 9b452c70a29..5b2c2669e16 100644 --- a/lib/runtime/src/cache.rs +++ b/lib/runtime/src/cache.rs @@ -29,7 +29,7 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash, WASMER_VERSION_H /// // corrupted or tampered with. /// let mut fs_cache = unsafe { FileSystemCache::new("some/directory/goes/here")? }; /// // Compute a key for a given WebAssembly binary -/// let key = WasmHash::generate(&[], Backend::Cranelift); +/// let key = WasmHash::generate_for_backend(&[], Backend::Cranelift); /// // Store a module into the cache given a key /// fs_cache.store(key, module.clone())?; /// Ok(module) @@ -120,7 +120,6 @@ mod tests { use super::*; use std::env; - use wasmer_runtime_core::backend::Backend; #[test] fn test_file_system_cache_run() { @@ -149,7 +148,7 @@ mod tests { .unwrap() }; // store module - let key = WasmHash::generate(&wasm, Backend::Cranelift); + let key = WasmHash::generate(&wasm); fs_cache.store(key, module.clone()).unwrap(); // load module