do backend caching with a directory

CHANGELOG.md

@@ -5,8 +5,8 @@ All PRs to the Wasmer repository must add to this file.
 Blocks of changes will separated by version increments.
 
 ## **[Unreleased]**
-- [#537](https://github.com/wasmerio/wasmer/pull/537) Add hidden flag (`--cache-key`) to use prehashed key into the compiled wasm cache
-- [#536](https://github.com/wasmerio/wasmer/pull/536) Update cache to use compiler backend name in cache key
+- [#537](https://github.com/wasmerio/wasmer/pull/537) Add hidden flag (`--cache-key`) to use prehashed key into the compiled wasm cache and change compiler backend-specific caching to use directories
+- [#536](https://github.com/wasmerio/wasmer/pull/536) ~Update cache to use compiler backend name in cache key~
 
 ## 0.5.4
 - [#529](https://github.com/wasmerio/wasmer/pull/529) Updates the Wasm Interface library, which is used by wapm, with bug fixes and error message improvements

Cargo.toml

@@ -69,8 +69,8 @@ trace = ["wasmer-runtime-core/trace"]
 extra-debug = ["wasmer-clif-backend/debug", "wasmer-runtime-core/debug"]
 # This feature will allow cargo test to run much faster
 fast-tests = []
-"backend:llvm" = ["wasmer-llvm-backend"]
-"backend:singlepass" = ["wasmer-singlepass-backend"]
+"backend:llvm" = ["wasmer-llvm-backend", "wasmer-runtime-core/backend:llvm"]
+"backend:singlepass" = ["wasmer-singlepass-backend", "wasmer-runtime-core/backend:singlepass"]
 wasi = ["wasmer-wasi"]
 # vfs = ["wasmer-runtime-abi"]
 

lib/runtime-core/Cargo.toml

@@ -54,3 +54,6 @@ cc = "1.0"
 [features]
 debug = []
 trace = ["debug"]
+# backend flags used in conditional compilation of Backend::variants
+"backend:singlepass" = []
+"backend:llvm" = []

lib/runtime-core/src/cache.rs

@@ -37,31 +37,17 @@ impl From<io::Error> for Error {
 pub struct WasmHash([u8; 32], [u8; 32]);
 
 impl WasmHash {
-    /// Hash a wasm module for the default compiler backend.
-    ///
-    /// See also: `WasmHash::generate_for_backend`.
+    /// Hash a wasm module.
     ///
     /// # Note:
     /// This does no verification that the supplied data
     /// is, in fact, a wasm module.
     pub fn generate(wasm: &[u8]) -> Self {
-        WasmHash::generate_for_backend(wasm, Backend::default())
-    }
-
-    /// Hash a wasm module for a specific compiler backend.
-    /// This allows multiple cache entries containing the same compiled
-    /// module with different compiler backends.
-    ///
-    /// # Note:
-    /// This function also does no verification that the supplied data
-    /// is a wasm module
-    pub fn generate_for_backend(wasm: &[u8], backend: Backend) -> Self {
         let mut first_part = [0u8; 32];
         let mut second_part = [0u8; 32];
 
         let mut state = blake2bp::State::new();
         state.update(wasm);
-        state.update(backend.to_string().as_bytes());
 
         let hasher = state.finalize();
         let generic_array = hasher.as_bytes();
@@ -243,7 +229,11 @@ pub trait Cache {
     type LoadError: fmt::Debug;
     type StoreError: fmt::Debug;
 
+    /// loads a module using the default `Backend`
     fn load(&self, key: WasmHash) -> Result<Module, Self::LoadError>;
+    /// loads a cached module using a specific `Backend`
+    fn load_with_backend(&self, key: WasmHash, backend: Backend)
+        -> Result<Module, Self::LoadError>;
     fn store(&mut self, key: WasmHash, module: Module) -> Result<(), Self::StoreError>;
 }
 

lib/runtime/src/cache.rs

@@ -7,7 +7,10 @@ use std::{
 };
 
 use wasmer_runtime_core::cache::Error as CacheError;
-pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash, WASMER_VERSION_HASH};
+pub use wasmer_runtime_core::{
+    backend::Backend,
+    cache::{Artifact, Cache, WasmHash, WASMER_VERSION_HASH},
+};
 
 /// Representation of a directory that contains compiled wasm artifacts.
 ///
@@ -20,7 +23,6 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash, WASMER_VERSION_H
 ///
 /// ```rust
 /// use wasmer_runtime::cache::{Cache, FileSystemCache, WasmHash};
-/// use wasmer_runtime_core::backend::Backend;
 ///
 /// # use wasmer_runtime::{Module, error::CacheError};
 /// fn store_module(module: Module) -> Result<Module, CacheError> {
@@ -29,7 +31,7 @@ pub use wasmer_runtime_core::cache::{Artifact, Cache, WasmHash, WASMER_VERSION_H
 ///     // corrupted or tampered with.
 ///     let mut fs_cache = unsafe { FileSystemCache::new("some/directory/goes/here")? };
 ///     // Compute a key for a given WebAssembly binary
-///     let key = WasmHash::generate_for_backend(&[], Backend::Cranelift);
+///     let key = WasmHash::generate(&[]);
 ///     // Store a module into the cache given a key
 ///     fs_cache.store(key, module.clone())?;
 ///     Ok(module)
@@ -88,8 +90,13 @@ impl Cache for FileSystemCache {
     type StoreError = CacheError;
 
     fn load(&self, key: WasmHash) -> Result<Module, CacheError> {
+        self.load_with_backend(key, Backend::default())
+    }
+
+    fn load_with_backend(&self, key: WasmHash, backend: Backend) -> Result<Module, CacheError> {
         let filename = key.encode();
         let mut new_path_buf = self.path.clone();
+        new_path_buf.push(backend.to_string());
         new_path_buf.push(filename);
         let file = File::open(new_path_buf)?;
         let mmap = unsafe { Mmap::map(&file)? };
@@ -102,12 +109,15 @@ impl Cache for FileSystemCache {
 
     fn store(&mut self, key: WasmHash, module: Module) -> Result<(), CacheError> {
         let filename = key.encode();
+        let backend_str = module.info().backend.to_string();
         let mut new_path_buf = self.path.clone();
-        new_path_buf.push(filename);
+        new_path_buf.push(backend_str);
 
         let serialized_cache = module.cache()?;
         let buffer = serialized_cache.serialize()?;
 
+        std::fs::create_dir_all(&new_path_buf)?;
+        new_path_buf.push(filename);
         let mut file = File::create(new_path_buf)?;
         file.write_all(&buffer)?;
 

src/bin/wasmer.rs

@@ -370,8 +370,10 @@ fn execute_wasm(options: &Run) -> Result<(), String> {
         };
         let load_cache_key = || -> Result<_, String> {
             if let Some(ref prehashed_cache_key) = options.cache_key {
-                if let Ok(module) = WasmHash::decode(prehashed_cache_key)
-                    .and_then(|prehashed_key| cache.load(prehashed_key))
+                if let Ok(module) =
+                    WasmHash::decode(prehashed_cache_key).and_then(|prehashed_key| {
+                        cache.load_with_backend(prehashed_key, options.backend)
+                    })
                 {
                     debug!("using prehashed key: {}", prehashed_cache_key);
                     return Ok(module);
@@ -380,12 +382,12 @@ fn execute_wasm(options: &Run) -> Result<(), String> {
 
             // We generate a hash for the given binary, so we can use it as key
             // for the Filesystem cache
-            let hash = WasmHash::generate_for_backend(&wasm_binary, options.backend);
+            let hash = WasmHash::generate(&wasm_binary);
 
             // cache.load will return the Module if it's able to deserialize it properly, and an error if:
             // * The file is not found
             // * The file exists, but it's corrupted or can't be converted to a module
-            match cache.load(hash) {
+            match cache.load_with_backend(hash, options.backend) {
                 Ok(module) => {
                     // We are able to load the module from cache
                     Ok(module)