Store and restore hash from a compiled module

.github/cross-linux-riscv64/Dockerfile

@@ -28,7 +28,7 @@ RUN apt-get update  && \
 # install rust tools
 RUN curl --proto "=https" --tlsv1.2 --retry 3 -sSfL https://sh.rustup.rs | sh -s -- -y
 ENV PATH="/root/.cargo/bin:${PATH}"
-RUN rustup -v toolchain install 1.73
+RUN rustup -v toolchain install 1.74
 # add docker the manual way
 COPY install_docker.sh /
 RUN /install_docker.sh
@@ -61,7 +61,7 @@ ENV CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc
     RUST_TEST_THREADS=1 \
     PKG_CONFIG_PATH="/usr/lib/riscv64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}"
 
-RUN rustup target add riscv64gc-unknown-linux-gnu --toolchain 1.73-x86_64-unknown-linux-gnu
+RUN rustup target add riscv64gc-unknown-linux-gnu --toolchain 1.74-x86_64-unknown-linux-gnu
 
 #compile libssl-dev for riscv64!
 COPY build_openssl.sh /

.github/workflows/benchmark.yaml

@@ -7,7 +7,7 @@ on:
 
 env:
   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git
-  MSRV: "1.73"
+  MSRV: "1.74"
 
 jobs:
   run_benchmark:

.github/workflows/build.yml

@@ -3,7 +3,7 @@ name: Builds
 env:
   RUST_BACKTRACE: 1
   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git
-  MSRV: "1.73"
+  MSRV: "1.74"
 
 on:
   push:

.github/workflows/cloudcompiler.yaml

@@ -2,7 +2,7 @@ name: Release cloudcompiler.wasm
 
 env:
   RUST_BACKTRACE: 1
-  MSRV: "1.73"
+  MSRV: "1.74"
 
 on:
   push:

.github/workflows/documentation.yaml

@@ -8,7 +8,7 @@ on:
       - 'lib/**'
 
 env:
-  MSRV: "1.73"
+  MSRV: "1.74"
 
 jobs:
   documentation:

.github/workflows/test.yaml

@@ -24,7 +24,7 @@ env:
   # Rust, but it's not stable on 1.69 yet. By explicitly setting the protocol we
   # can override that behaviour
   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git
-  MSRV: "1.73"
+  MSRV: "1.74"
   NEXTEST_PROFILE: "ci"
 
 jobs:

Cargo.toml

@@ -83,7 +83,7 @@ edition = "2021"
 homepage = "https://wasmer.io/"
 license = "MIT"
 repository = "https://github.com/wasmerio/wasmer"
-rust-version = "1.73"
+rust-version = "1.74"
 version = "4.3.0-beta.1"
 
 [workspace.dependencies]

lib/api/src/jsc/module.rs

@@ -67,8 +67,9 @@ impl Module {
     ///
     pub(crate) unsafe fn from_js_module(module: JSObject, binary: impl IntoBytes) -> Self {
         let binary = binary.into_bytes();
+
         // The module is now validated, so we can safely parse it's types
-        let info = crate::module_info_polyfill::translate_module(&binary[..])
+        let mut info = crate::module_info_polyfill::translate_module(&binary[..])
             .unwrap()
             .info;
 

lib/cli/src/commands/binfmt.rs

@@ -55,7 +55,7 @@ fn seccheck(path: &Path) -> Result<()> {
 
 impl Binfmt {
     /// The filename used to register the wasmer CLI as a binfmt interpreter.
-    pub const FILENAME: &str = "wasmer-binfmt-interpreter";
+    pub const FILENAME: &'static str = "wasmer-binfmt-interpreter";
 
     /// execute [Binfmt]
     pub fn execute(&self) -> Result<()> {

lib/cli/src/commands/create_exe.rs

@@ -380,7 +380,7 @@ pub(super) fn compile_pirita_into_directory(
     let volume_path = target_dir.join("volumes").join("volume.o");
     write_volume_obj(&volume_bytes, volume_name, &volume_path, target)?;
     let volume_path = volume_path.canonicalize()?;
-    let volume_path = pathdiff::diff_paths(&volume_path, &target_dir).unwrap();
+    let volume_path = pathdiff::diff_paths(volume_path, &target_dir).unwrap();
 
     std::fs::create_dir_all(target_dir.join("atoms")).map_err(|e| {
         anyhow::anyhow!("cannot create /atoms dir in {}: {e}", target_dir.display())

lib/cli/src/commands/run/mod.rs

@@ -29,6 +29,7 @@ use wasmer::{
 use wasmer_compiler::ArtifactBuild;
 use wasmer_config::package::PackageSource as PackageSpecifier;
 use wasmer_registry::{wasmer_env::WasmerEnv, Package};
+use wasmer_types::ModuleHash;
 #[cfg(feature = "journal")]
 use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger};
 use wasmer_wasix::{
@@ -43,9 +44,7 @@ use wasmer_wasix::{
         MappedCommand, MappedDirectory, Runner,
     },
     runtime::{
-        module_cache::{CacheError, ModuleHash},
-        package_loader::PackageLoader,
-        resolver::QueryError,
+        module_cache::CacheError, package_loader::PackageLoader, resolver::QueryError,
         task_manager::VirtualTaskManagerExt,
     },
     Runtime, WasiError,
@@ -712,10 +711,8 @@ impl ExecutableTarget {
                 let engine = runtime.engine();
                 pb.set_message("Deserializing pre-compiled WebAssembly module");
                 let module = unsafe { Module::deserialize_from_file(&engine, path)? };
-                let module_hash = {
-                    let wasm = std::fs::read(path)?;
-                    ModuleHash::xxhash(wasm)
-                };
+
+                let module_hash = module.info().hash.unwrap_or(ModuleHash::XXHash([0u8; 8]));
 
                 Ok(ExecutableTarget::WebAssembly {
                     module,

lib/cli/src/commands/run/wasi.rs

@@ -15,6 +15,7 @@ use virtual_fs::{DeviceFile, FileSystem, PassthruFileSystem, RootFileSystemBuild
 use wasmer::{Engine, Function, Instance, Memory32, Memory64, Module, RuntimeError, Store, Value};
 use wasmer_config::package::PackageSource as PackageSpecifier;
 use wasmer_registry::wasmer_env::WasmerEnv;
+use wasmer_types::ModuleHash;
 #[cfg(feature = "journal")]
 use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger};
 use wasmer_wasix::{
@@ -27,7 +28,7 @@ use wasmer_wasix::{
     rewind_ext,
     runners::{MappedCommand, MappedDirectory},
     runtime::{
-        module_cache::{FileSystemCache, ModuleCache, ModuleHash},
+        module_cache::{FileSystemCache, ModuleCache},
         package_loader::{BuiltinPackageLoader, PackageLoader},
         resolver::{FileSystemSource, InMemorySource, MultiSource, Source, WapmSource, WebSource},
         task_manager::{

lib/compiler/Cargo.toml

@@ -22,6 +22,7 @@ serde = { version = "1.0", features = ["derive"], optional = true }
 thiserror = "1.0"
 serde_bytes = { version = "0.11", optional = true }
 smallvec = "1.6"
+xxhash-rust = { version = "0.8.10", features = ["xxh64"] }
 
 backtrace = "0.3"
 memmap2 = "0.5"

lib/compiler/src/translator/environ.rs

@@ -8,14 +8,14 @@ use crate::wasmparser::{Operator, ValType};
 use std::convert::{TryFrom, TryInto};
 use std::ops::Range;
 use wasmer_types::entity::PrimaryMap;
-use wasmer_types::FunctionType;
 use wasmer_types::WasmResult;
 use wasmer_types::{
     CustomSectionIndex, DataIndex, DataInitializer, DataInitializerLocation, ElemIndex,
     ExportIndex, FunctionIndex, GlobalIndex, GlobalInit, GlobalType, ImportIndex,
     LocalFunctionIndex, MemoryIndex, MemoryType, ModuleInfo, SignatureIndex, TableIndex,
     TableInitializer, TableType,
 };
+use wasmer_types::{FunctionType, ModuleHash};
 
 /// Contains function data: bytecode and its offset in the module.
 #[derive(Hash)]
@@ -89,6 +89,9 @@ impl<'data> ModuleEnvironment<'data> {
         assert!(self.module_translation_state.is_none());
         let module_translation_state = translate_module(data, &mut self)?;
         self.module_translation_state = Some(module_translation_state);
+
+        self.module.hash = Some(ModuleHash::xxhash(data));
+
         Ok(self)
     }
 

lib/types/Cargo.toml

@@ -23,6 +23,17 @@ enum-iterator = "0.7.0"
 target-lexicon = { version = "0.12.2", default-features = false }
 enumset.workspace = true
 bytecheck = "0.6.8"
+rand = "0.8"
+webc = { workspace = true, default-features = false }
+xxhash-rust = { version = "0.8.8", features = ["xxh64"] }
+sha2 = { version = "0.10" }
+hex = { version = "^0.4" }
+
+# `rand` uses `getrandom` transitively, and to be able to
+# compile the project for `js`, we need to enable this feature
+[dependencies.getrandom]
+version = "*"
+features = ["js"]
 
 [dev-dependencies]
 memoffset.workspace = true

lib/types/src/lib.rs

@@ -58,6 +58,7 @@ mod initializers;
 mod libcalls;
 mod memory;
 mod module;
+mod module_hash;
 mod serialize;
 mod stack;
 mod store_id;
@@ -97,6 +98,7 @@ pub use crate::initializers::{
 };
 pub use crate::memory::{Memory32, Memory64, MemorySize};
 pub use crate::module::{ExportsIterator, ImportKey, ImportsIterator, ModuleInfo};
+pub use crate::module_hash::ModuleHash;
 pub use crate::units::{
     Bytes, PageCountOutOfRange, Pages, WASM_MAX_PAGES, WASM_MIN_PAGES, WASM_PAGE_SIZE,
 };

lib/types/src/module.rs

@@ -8,9 +8,10 @@ use crate::entity::{EntityRef, PrimaryMap};
 use crate::{
     CustomSectionIndex, DataIndex, ElemIndex, ExportIndex, ExportType, ExternType, FunctionIndex,
     FunctionType, GlobalIndex, GlobalInit, GlobalType, ImportIndex, ImportType, LocalFunctionIndex,
-    LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryIndex, MemoryType, SignatureIndex,
-    TableIndex, TableInitializer, TableType,
+    LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryIndex, MemoryType, ModuleHash,
+    SignatureIndex, TableIndex, TableInitializer, TableType,
 };
+
 use indexmap::IndexMap;
 use rkyv::{
     de::SharedDeserializeRegistry, ser::ScratchSpace, ser::Serializer,
@@ -99,6 +100,10 @@ mod serde_imports {
 
 /// A translated WebAssembly module, excluding the function bodies and
 /// memory initializers.
+///
+/// IMPORTANT: since this struct will be serialized as part of the compiled module artifact,
+/// if you change this struct, do not forget to update [`MetadataHeader::version`](crate::serialize::MetadataHeader)
+/// to make sure we don't break compatibility between versions.
 #[derive(Debug, Clone, Default)]
 #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))]
 pub struct ModuleInfo {
@@ -111,6 +116,9 @@ pub struct ModuleInfo {
     #[cfg_attr(feature = "enable-serde", serde(skip_serializing, skip_deserializing))]
     pub id: ModuleId,
 
+    /// hash of the module
+    pub hash: Option<ModuleHash>,
+
     /// The name of this wasm module, often found in the wasm file.
     pub name: Option<String>,
 
@@ -182,6 +190,7 @@ pub struct ModuleInfo {
 #[archive_attr(derive(CheckBytes, Debug))]
 pub struct ArchivableModuleInfo {
     name: Option<String>,
+    hash: Option<ModuleHash>,
     imports: IndexMap<ImportKey, ImportIndex>,
     exports: IndexMap<String, ExportIndex>,
     start_function: Option<FunctionIndex>,
@@ -207,6 +216,7 @@ impl From<ModuleInfo> for ArchivableModuleInfo {
     fn from(it: ModuleInfo) -> Self {
         Self {
             name: it.name,
+            hash: it.hash,
             imports: it.imports,
             exports: it.exports,
             start_function: it.start_function,
@@ -235,6 +245,7 @@ impl From<ArchivableModuleInfo> for ModuleInfo {
         Self {
             id: Default::default(),
             name: it.name,
+            hash: it.hash,
             imports: it.imports,
             exports: it.exports,
             start_function: it.start_function,
@@ -325,6 +336,11 @@ impl ModuleInfo {
         Default::default()
     }
 
+    /// Returns the module hash if available
+    pub fn hash(&self) -> Option<ModuleHash> {
+        self.hash
+    }
+
     /// Get the given passive element, if it exists.
     pub fn get_passive_element(&self, index: ElemIndex) -> Option<&[FunctionIndex]> {
         self.passive_elements.get(&index).map(|es| &**es)