diff --git a/.github/cross-linux-riscv64/Dockerfile b/.github/cross-linux-riscv64/Dockerfile index b158afb518c..47eb4b6e5d2 100644 --- a/.github/cross-linux-riscv64/Dockerfile +++ b/.github/cross-linux-riscv64/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && \ # install rust tools RUN curl --proto "=https" --tlsv1.2 --retry 3 -sSfL https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" -RUN rustup -v toolchain install 1.73 +RUN rustup -v toolchain install 1.74 # add docker the manual way COPY install_docker.sh / RUN /install_docker.sh @@ -61,7 +61,7 @@ ENV CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc RUST_TEST_THREADS=1 \ PKG_CONFIG_PATH="/usr/lib/riscv64-linux-gnu/pkgconfig/:${PKG_CONFIG_PATH}" -RUN rustup target add riscv64gc-unknown-linux-gnu --toolchain 1.73-x86_64-unknown-linux-gnu +RUN rustup target add riscv64gc-unknown-linux-gnu --toolchain 1.74-x86_64-unknown-linux-gnu #compile libssl-dev for riscv64! COPY build_openssl.sh / diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 77fd27fad4d..53b75bd202e 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -7,7 +7,7 @@ on: env: CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git - MSRV: "1.73" + MSRV: "1.74" jobs: run_benchmark: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 70457dd92b4..1e06f6aa459 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,7 @@ name: Builds env: RUST_BACKTRACE: 1 CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git - MSRV: "1.73" + MSRV: "1.74" on: push: diff --git a/.github/workflows/cloudcompiler.yaml b/.github/workflows/cloudcompiler.yaml index 2c82c071649..22ac4b1e7da 100644 --- a/.github/workflows/cloudcompiler.yaml +++ b/.github/workflows/cloudcompiler.yaml @@ -2,7 +2,7 @@ name: Release cloudcompiler.wasm env: RUST_BACKTRACE: 1 - MSRV: "1.73" + MSRV: "1.74" on: push: diff --git a/.github/workflows/documentation.yaml b/.github/workflows/documentation.yaml index 59f6e5b5f4b..7a023a4f5cc 100644 --- a/.github/workflows/documentation.yaml +++ b/.github/workflows/documentation.yaml @@ -8,7 +8,7 @@ on: - 'lib/**' env: - MSRV: "1.73" + MSRV: "1.74" jobs: documentation: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3cdf2157c5b..37a022c43df 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -24,7 +24,7 @@ env: # Rust, but it's not stable on 1.69 yet. By explicitly setting the protocol we # can override that behaviour CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git - MSRV: "1.73" + MSRV: "1.74" NEXTEST_PROFILE: "ci" jobs: diff --git a/Cargo.lock b/Cargo.lock index 2ebd78e6325..03420ee6edd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6573,6 +6573,7 @@ dependencies = [ "wasmer-vm", "wasmparser 0.121.2", "winapi 0.3.9", + "xxhash-rust", ] [[package]] @@ -6901,14 +6902,19 @@ dependencies = [ "bytecheck", "enum-iterator", "enumset", + "getrandom", + "hex", "indexmap 1.9.3", "memoffset 0.9.1", "more-asserts", "rkyv", "serde", "serde_bytes", + "sha2", "target-lexicon 0.12.14", "thiserror", + "webc", + "xxhash-rust", ] [[package]] @@ -7068,6 +7074,7 @@ dependencies = [ "tokio 1.37.0", "virtual-fs", "wasmer", + "wasmer-types", "wasmer-wasix", "wast 38.0.1", ] diff --git a/Cargo.toml b/Cargo.toml index eeb2d675873..e190667a6bd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -83,7 +83,7 @@ edition = "2021" homepage = "https://wasmer.io/" license = "MIT" repository = "https://github.com/wasmerio/wasmer" -rust-version = "1.73" +rust-version = "1.74" version = "4.3.0-beta.1" [workspace.dependencies] diff --git a/lib/api/src/jsc/module.rs b/lib/api/src/jsc/module.rs index 1eda38c57c8..08146f4ad77 100644 --- a/lib/api/src/jsc/module.rs +++ b/lib/api/src/jsc/module.rs @@ -67,6 +67,7 @@ impl Module { /// pub(crate) unsafe fn from_js_module(module: JSObject, binary: impl IntoBytes) -> Self { let binary = binary.into_bytes(); + // The module is now validated, so we can safely parse it's types let info = crate::module_info_polyfill::translate_module(&binary[..]) .unwrap() diff --git a/lib/api/src/sys/engine.rs b/lib/api/src/sys/engine.rs index c54f6651e50..076bd40153e 100644 --- a/lib/api/src/sys/engine.rs +++ b/lib/api/src/sys/engine.rs @@ -6,7 +6,7 @@ pub use wasmer_compiler::{ }; #[cfg(feature = "compiler")] use wasmer_types::Features; -use wasmer_types::{DeserializeError, Target}; +use wasmer_types::{DeserializeError, HashAlgorithm, Target}; /// Get the default config for the sys Engine #[allow(unreachable_code)] @@ -57,6 +57,9 @@ pub trait NativeEngineExt { #[cfg(feature = "compiler")] fn new(compiler_config: Box, target: Target, features: Features) -> Self; + /// Sets the hash algorithm + fn set_hash_algorithm(&mut self, hash_algorithm: Option); + /// Create a headless `Engine` /// /// A headless engine is an engine without any compiler attached. @@ -153,4 +156,8 @@ impl NativeEngineExt for crate::engine::Engine { artifact, ))) } + + fn set_hash_algorithm(&mut self, hash_algorithm: Option) { + self.0.set_hash_algorithm(hash_algorithm) + } } diff --git a/lib/cli-compiler/src/commands/compile.rs b/lib/cli-compiler/src/commands/compile.rs index 9ebccc97043..7a5147066b5 100644 --- a/lib/cli-compiler/src/commands/compile.rs +++ b/lib/cli-compiler/src/commands/compile.rs @@ -59,7 +59,9 @@ impl Compile { }) .unwrap_or_default(); let (engine_builder, compiler_type) = self.store.get_engine_for_target(target.clone())?; - let engine = engine_builder.engine(); + let engine = engine_builder + .set_hash_algorithm(Some(wasmer_types::HashAlgorithm::Sha256)) + .engine(); let output_filename = self .output .file_stem() @@ -105,6 +107,7 @@ impl Compile { &target, memory_styles, table_styles, + engine.hash_algorithm(), )?; let serialized = artifact.serialize()?; fs::write(output_filename, serialized)?; diff --git a/lib/cli/src/commands/binfmt.rs b/lib/cli/src/commands/binfmt.rs index 271386e2271..7af57e06759 100644 --- a/lib/cli/src/commands/binfmt.rs +++ b/lib/cli/src/commands/binfmt.rs @@ -55,7 +55,7 @@ fn seccheck(path: &Path) -> Result<()> { impl Binfmt { /// The filename used to register the wasmer CLI as a binfmt interpreter. - pub const FILENAME: &str = "wasmer-binfmt-interpreter"; + pub const FILENAME: &'static str = "wasmer-binfmt-interpreter"; /// execute [Binfmt] pub fn execute(&self) -> Result<()> { diff --git a/lib/cli/src/commands/compile.rs b/lib/cli/src/commands/compile.rs index 9739b284bd4..9ee969574a6 100644 --- a/lib/cli/src/commands/compile.rs +++ b/lib/cli/src/commands/compile.rs @@ -4,7 +4,7 @@ use anyhow::{Context, Result}; use clap::Parser; use wasmer::*; -use crate::{store::StoreOptions, warning}; +use crate::{common::HashAlgorithm, store::StoreOptions, warning}; #[derive(Debug, Parser)] /// The options for the `wasmer compile` subcommand @@ -26,6 +26,10 @@ pub struct Compile { #[clap(short = 'm')] cpu_features: Vec, + + /// Hashing algorithm to be used for module hash + #[clap(long, value_enum)] + hash_algorithm: Option, } impl Compile { @@ -54,6 +58,11 @@ impl Compile { }) .unwrap_or_default(); let (store, compiler_type) = self.store.get_store_for_target(target.clone())?; + + let mut engine = store.engine().clone(); + let hash_algorithm = self.hash_algorithm.unwrap_or_default().into(); + engine.set_hash_algorithm(Some(hash_algorithm)); + let output_filename = self .output .file_stem() diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 45d004d5600..add738ee754 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -21,7 +21,10 @@ use webc::{ }; use self::utils::normalize_atom_name; -use crate::{common::normalize_path, store::CompilerOptions}; +use crate::{ + common::{normalize_path, HashAlgorithm}, + store::CompilerOptions, +}; const LINK_SYSTEM_LIBRARIES_WINDOWS: &[&str] = &["userenv", "Ws2_32", "advapi32", "bcrypt"]; @@ -91,6 +94,10 @@ pub struct CreateExe { #[clap(flatten)] compiler: CompilerOptions, + + /// Hashing algorithm to be used for module hash + #[clap(long, value_enum)] + hash_algorithm: Option, } /// Url or version to download the release from @@ -216,6 +223,10 @@ impl CreateExe { let (store, compiler_type) = self.compiler.get_store_for_target(target.clone())?; + let mut engine = store.engine().clone(); + let hash_algorithm = self.hash_algorithm.unwrap_or_default().into(); + engine.set_hash_algorithm(Some(hash_algorithm)); + println!("Compiler: {}", compiler_type.to_string()); println!("Target: {}", target.triple()); println!( @@ -380,7 +391,7 @@ pub(super) fn compile_pirita_into_directory( let volume_path = target_dir.join("volumes").join("volume.o"); write_volume_obj(&volume_bytes, volume_name, &volume_path, target)?; let volume_path = volume_path.canonicalize()?; - let volume_path = pathdiff::diff_paths(&volume_path, &target_dir).unwrap(); + let volume_path = pathdiff::diff_paths(volume_path, &target_dir).unwrap(); std::fs::create_dir_all(target_dir.join("atoms")).map_err(|e| { anyhow::anyhow!("cannot create /atoms dir in {}: {e}", target_dir.display()) diff --git a/lib/cli/src/commands/inspect.rs b/lib/cli/src/commands/inspect.rs index 1f5c404f54d..80b4d05abd9 100644 --- a/lib/cli/src/commands/inspect.rs +++ b/lib/cli/src/commands/inspect.rs @@ -27,6 +27,7 @@ impl Inspect { fn inner_execute(&self) -> Result<()> { let (store, _compiler_type) = self.store.get_store()?; + let module_contents = std::fs::read(&self.path)?; let iswasm = is_wasm(&module_contents); let module_len = module_contents.len(); diff --git a/lib/cli/src/commands/run/mod.rs b/lib/cli/src/commands/run/mod.rs index 5551029f175..bfe988ddbea 100644 --- a/lib/cli/src/commands/run/mod.rs +++ b/lib/cli/src/commands/run/mod.rs @@ -21,14 +21,18 @@ use once_cell::sync::Lazy; use sha2::{Digest, Sha256}; use tempfile::NamedTempFile; use url::Url; +#[cfg(feature = "sys")] +use wasmer::NativeEngineExt; use wasmer::{ DeserializeError, Engine, Function, Imports, Instance, Module, Store, Type, TypedFunction, Value, }; + #[cfg(feature = "compiler")] use wasmer_compiler::ArtifactBuild; use wasmer_config::package::PackageSource as PackageSpecifier; use wasmer_registry::{wasmer_env::WasmerEnv, Package}; +use wasmer_types::ModuleHash; #[cfg(feature = "journal")] use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ @@ -43,16 +47,17 @@ use wasmer_wasix::{ MappedCommand, MappedDirectory, Runner, }, runtime::{ - module_cache::{CacheError, ModuleHash}, - package_loader::PackageLoader, - resolver::QueryError, + module_cache::CacheError, package_loader::PackageLoader, resolver::QueryError, task_manager::VirtualTaskManagerExt, }, Runtime, WasiError, }; use webc::{metadata::Manifest, Container}; -use crate::{commands::run::wasi::Wasi, error::PrettyError, logging::Output, store::StoreOptions}; +use crate::{ + commands::run::wasi::Wasi, common::HashAlgorithm, error::PrettyError, logging::Output, + store::StoreOptions, +}; const TICK: Duration = Duration::from_millis(250); @@ -81,6 +86,9 @@ pub struct Run { input: PackageSource, /// Command-line arguments passed to the package args: Vec, + /// Hashing algorithm to be used for module hash + #[clap(long, value_enum)] + hash_algorithm: Option, } impl Run { @@ -115,12 +123,21 @@ impl Run { let _guard = handle.enter(); let (store, _) = self.store.get_store()?; - let runtime = self.wasi.prepare_runtime( - store.engine().clone(), - &self.env, - runtime, - preferred_webc_version, - )?; + + #[cfg(feature = "sys")] + let engine = { + let mut engine = store.engine().clone(); + let hash_algorithm = self.hash_algorithm.unwrap_or_default().into(); + engine.set_hash_algorithm(Some(hash_algorithm)); + + engine + }; + #[cfg(not(feature = "sys"))] + let engine = store.engine().clone(); + + let runtime = + self.wasi + .prepare_runtime(engine, &self.env, runtime, preferred_webc_version)?; // This is a slow operation, so let's temporarily wrap the runtime with // something that displays progress @@ -484,6 +501,7 @@ impl Run { coredump_on_trap: None, input: PackageSource::infer(executable)?, args: args.to_vec(), + hash_algorithm: None, }) } } @@ -721,10 +739,10 @@ impl ExecutableTarget { let engine = runtime.engine(); pb.set_message("Deserializing pre-compiled WebAssembly module"); let module = unsafe { Module::deserialize_from_file(&engine, path)? }; - let module_hash = { - let wasm = std::fs::read(path)?; - ModuleHash::xxhash(wasm) - }; + + let module_hash = module.info().hash.ok_or_else(|| { + anyhow::Error::msg("module hash is not present in the artifact") + })?; Ok(ExecutableTarget::WebAssembly { module, diff --git a/lib/cli/src/commands/run/wasi.rs b/lib/cli/src/commands/run/wasi.rs index 5163d9b805c..f452f46777d 100644 --- a/lib/cli/src/commands/run/wasi.rs +++ b/lib/cli/src/commands/run/wasi.rs @@ -15,6 +15,7 @@ use virtual_fs::{DeviceFile, FileSystem, PassthruFileSystem, RootFileSystemBuild use wasmer::{Engine, Function, Instance, Memory32, Memory64, Module, RuntimeError, Store, Value}; use wasmer_config::package::PackageSource as PackageSpecifier; use wasmer_registry::wasmer_env::WasmerEnv; +use wasmer_types::ModuleHash; #[cfg(feature = "journal")] use wasmer_wasix::journal::{LogFileJournal, SnapshotTrigger}; use wasmer_wasix::{ @@ -27,7 +28,7 @@ use wasmer_wasix::{ rewind_ext, runners::{MappedCommand, MappedDirectory}, runtime::{ - module_cache::{FileSystemCache, ModuleCache, ModuleHash}, + module_cache::{FileSystemCache, ModuleCache}, package_loader::{BuiltinPackageLoader, PackageLoader}, resolver::{FileSystemSource, InMemorySource, MultiSource, Source, WapmSource, WebSource}, task_manager::{ diff --git a/lib/cli/src/commands/validate.rs b/lib/cli/src/commands/validate.rs index f54404877ca..8806ca49f17 100644 --- a/lib/cli/src/commands/validate.rs +++ b/lib/cli/src/commands/validate.rs @@ -25,6 +25,7 @@ impl Validate { } fn inner_execute(&self) -> Result<()> { let (store, _compiler_type) = self.store.get_store()?; + let module_contents = std::fs::read(&self.path)?; if !is_wasm(&module_contents) { bail!("`wasmer validate` only validates WebAssembly files"); diff --git a/lib/cli/src/commands/wast.rs b/lib/cli/src/commands/wast.rs index 293e23896fa..a2ce63b7891 100644 --- a/lib/cli/src/commands/wast.rs +++ b/lib/cli/src/commands/wast.rs @@ -3,9 +3,10 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::Parser; +use wasmer::NativeEngineExt; use wasmer_wast::Wast as WastSpectest; -use crate::store::StoreOptions; +use crate::{common::HashAlgorithm, store::StoreOptions}; #[derive(Debug, Parser)] /// The options for the `wasmer wast` subcommand @@ -20,6 +21,10 @@ pub struct Wast { #[clap(short, long)] /// A flag to indicate wast stop at the first error or continue. fail_fast: bool, + + /// Hashing algorithm to be used for module hash + #[clap(long, value_enum)] + hash_algorithm: Option, } impl Wast { @@ -30,6 +35,11 @@ impl Wast { } fn inner_execute(&self) -> Result<()> { let (store, _compiler_name) = self.store.get_store()?; + + let mut engine = store.engine().clone(); + let hash_algorithm = self.hash_algorithm.unwrap_or_default().into(); + engine.set_hash_algorithm(Some(hash_algorithm)); + let mut wast = WastSpectest::new_with_spectest(store); wast.fail_fast = self.fail_fast; wast.run_file(&self.path).with_context(|| "tests failed")?; diff --git a/lib/cli/src/common.rs b/lib/cli/src/common.rs index 1e5e8641a9b..99fc28b4b6c 100644 --- a/lib/cli/src/common.rs +++ b/lib/cli/src/common.rs @@ -1,7 +1,7 @@ //! Common module with common used structures across different //! commands. -use clap::Parser; +use clap::{Parser, ValueEnum}; #[derive(Debug, Parser, Clone, Default)] /// The WebAssembly features that can be passed through the @@ -39,3 +39,27 @@ pub struct WasmFeatures { pub(crate) fn normalize_path(s: &str) -> String { wasmer_registry::utils::normalize_path(s) } + +/// Hashing algorithm to be used for the module info +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, ValueEnum)] +pub enum HashAlgorithm { + /// Sha256 + Sha256, + /// XXHash + XXHash, +} + +impl Default for HashAlgorithm { + fn default() -> Self { + Self::Sha256 + } +} + +impl From for wasmer_types::HashAlgorithm { + fn from(value: HashAlgorithm) -> Self { + match value { + HashAlgorithm::Sha256 => wasmer_types::HashAlgorithm::Sha256, + HashAlgorithm::XXHash => wasmer_types::HashAlgorithm::XXHash, + } + } +} diff --git a/lib/compiler/Cargo.toml b/lib/compiler/Cargo.toml index ade20dece0d..65b96fe95b6 100644 --- a/lib/compiler/Cargo.toml +++ b/lib/compiler/Cargo.toml @@ -22,6 +22,7 @@ serde = { version = "1.0", features = ["derive"], optional = true } thiserror = "1.0" serde_bytes = { version = "0.11", optional = true } smallvec = "1.6" +xxhash-rust = { version = "0.8.10", features = ["xxh64"] } backtrace = "0.3" memmap2 = "0.5" diff --git a/lib/compiler/src/artifact_builders/artifact_builder.rs b/lib/compiler/src/artifact_builders/artifact_builder.rs index fb0a66df9fc..88a843a79c3 100644 --- a/lib/compiler/src/artifact_builders/artifact_builder.rs +++ b/lib/compiler/src/artifact_builders/artifact_builder.rs @@ -24,11 +24,12 @@ use wasmer_types::CompileModuleInfo; use wasmer_types::DeserializeError; use wasmer_types::{ CompileError, CpuFeature, CustomSection, Dwarf, FunctionIndex, LocalFunctionIndex, MemoryIndex, - MemoryStyle, ModuleInfo, OwnedDataInitializer, Relocation, SectionIndex, SignatureIndex, - TableIndex, TableStyle, Target, + MemoryStyle, ModuleHash, ModuleInfo, OwnedDataInitializer, Relocation, SectionIndex, + SignatureIndex, TableIndex, TableStyle, Target, }; use wasmer_types::{ - CompiledFunctionFrameInfo, FunctionBody, SerializableCompilation, SerializableModule, + CompiledFunctionFrameInfo, FunctionBody, HashAlgorithm, SerializableCompilation, + SerializableModule, }; use wasmer_types::{MetadataHeader, SerializeError}; @@ -54,6 +55,7 @@ impl ArtifactBuild { target: &Target, memory_styles: PrimaryMap, table_styles: PrimaryMap, + hash_algorithm: Option, ) -> Result { let environ = ModuleEnvironment::new(); let features = inner_engine.features().clone(); @@ -67,6 +69,15 @@ impl ArtifactBuild { let middlewares = compiler.get_middlewares(); middlewares.apply_on_module_info(&mut module); + if let Some(hash_algorithm) = hash_algorithm { + let hash = match hash_algorithm { + HashAlgorithm::Sha256 => ModuleHash::sha256(data), + HashAlgorithm::XXHash => ModuleHash::xxhash(data), + }; + + module.hash = Some(hash); + } + let compile_info = CompileModuleInfo { module: Arc::new(module), features, diff --git a/lib/compiler/src/engine/artifact.rs b/lib/compiler/src/engine/artifact.rs index ff38a755c55..16d21dbbb1d 100644 --- a/lib/compiler/src/engine/artifact.rs +++ b/lib/compiler/src/engine/artifact.rs @@ -36,8 +36,9 @@ use wasmer_types::DataInitializerLocation; use wasmer_types::DataInitializerLocationLike; use wasmer_types::MetadataHeader; use wasmer_types::{ - CompileError, CpuFeature, DataInitializer, DeserializeError, FunctionIndex, LocalFunctionIndex, - MemoryIndex, ModuleInfo, OwnedDataInitializer, SignatureIndex, TableIndex, Target, + CompileError, CpuFeature, DataInitializer, DeserializeError, FunctionIndex, HashAlgorithm, + LocalFunctionIndex, MemoryIndex, ModuleInfo, OwnedDataInitializer, SignatureIndex, TableIndex, + Target, }; use wasmer_types::{SerializableModule, SerializeError}; use wasmer_vm::{FunctionBodyPtr, MemoryStyle, TableStyle, VMSharedSignatureIndex, VMTrampoline}; @@ -113,6 +114,7 @@ impl Artifact { engine: &Engine, data: &[u8], tunables: &dyn Tunables, + hash_algorithm: Option, ) -> Result { let mut inner_engine = engine.inner_mut(); let environ = ModuleEnvironment::new(); @@ -135,6 +137,7 @@ impl Artifact { engine.target(), memory_styles, table_styles, + hash_algorithm, )?; Self::from_parts( diff --git a/lib/compiler/src/engine/builder.rs b/lib/compiler/src/engine/builder.rs index 0f07396578d..5cb356526b5 100644 --- a/lib/compiler/src/engine/builder.rs +++ b/lib/compiler/src/engine/builder.rs @@ -1,6 +1,6 @@ use super::Engine; use crate::CompilerConfig; -use wasmer_types::{Features, Target}; +use wasmer_types::{Features, HashAlgorithm, Target}; /// The Builder contents of `Engine` pub struct EngineBuilder { @@ -10,6 +10,8 @@ pub struct EngineBuilder { target: Option, /// The features to compile the Wasm module with features: Option, + /// The hashing algorithm + hash_algorithm: Option, } impl EngineBuilder { @@ -22,6 +24,7 @@ impl EngineBuilder { compiler_config: Some(compiler_config.into()), target: None, features: None, + hash_algorithm: None, } } @@ -31,6 +34,7 @@ impl EngineBuilder { compiler_config: None, target: None, features: None, + hash_algorithm: None, } } @@ -46,6 +50,12 @@ impl EngineBuilder { self } + /// Set the hashing algorithm + pub fn set_hash_algorithm(mut self, hash_algorithm: Option) -> Self { + self.hash_algorithm = hash_algorithm; + self + } + /// Build the `Engine` for this configuration #[cfg(feature = "compiler")] pub fn engine(self) -> Engine { @@ -54,7 +64,11 @@ impl EngineBuilder { let features = self .features .unwrap_or_else(|| compiler_config.default_features_for_target(&target)); - Engine::new(compiler_config, target, features) + let mut engine = Engine::new(compiler_config, target, features); + + engine.set_hash_algorithm(self.hash_algorithm); + + engine } else { Engine::headless() } diff --git a/lib/compiler/src/engine/inner.rs b/lib/compiler/src/engine/inner.rs index cadb5cc5ef1..3957c8fe1d0 100644 --- a/lib/compiler/src/engine/inner.rs +++ b/lib/compiler/src/engine/inner.rs @@ -19,6 +19,7 @@ use shared_buffer::OwnedBuffer; use std::path::Path; use std::sync::atomic::{AtomicUsize, Ordering::SeqCst}; use std::sync::{Arc, Mutex}; +use wasmer_types::HashAlgorithm; #[cfg(not(target_arch = "wasm32"))] use wasmer_types::{ entity::PrimaryMap, DeserializeError, FunctionBodyLike, FunctionIndex, FunctionType, @@ -43,6 +44,7 @@ pub struct Engine { #[cfg(not(target_arch = "wasm32"))] tunables: Arc, name: String, + hash_algorithm: Option, } impl Engine { @@ -71,6 +73,7 @@ impl Engine { #[cfg(not(target_arch = "wasm32"))] tunables: Arc::new(tunables), name, + hash_algorithm: None, } } @@ -88,6 +91,16 @@ impl Engine { self.name.as_str() } + /// Sets the hash algorithm + pub fn set_hash_algorithm(&mut self, hash_algorithm: Option) { + self.hash_algorithm = hash_algorithm; + } + + /// Returns the hash algorithm + pub fn hash_algorithm(&self) -> Option { + self.hash_algorithm + } + /// Returns the deterministic id of this engine pub fn deterministic_id(&self) -> &str { // TODO: add a `deterministic_id` to the Compiler, so two @@ -130,6 +143,7 @@ impl Engine { #[cfg(not(target_arch = "wasm32"))] tunables: Arc::new(tunables), name: "engine-headless".to_string(), + hash_algorithm: None, } } @@ -176,6 +190,7 @@ impl Engine { self, binary, self.tunables.as_ref(), + self.hash_algorithm, )?)) } diff --git a/lib/compiler/src/translator/environ.rs b/lib/compiler/src/translator/environ.rs index bb273b2c3da..88910507d38 100644 --- a/lib/compiler/src/translator/environ.rs +++ b/lib/compiler/src/translator/environ.rs @@ -89,6 +89,7 @@ impl<'data> ModuleEnvironment<'data> { assert!(self.module_translation_state.is_none()); let module_translation_state = translate_module(data, &mut self)?; self.module_translation_state = Some(module_translation_state); + Ok(self) } diff --git a/lib/types/Cargo.toml b/lib/types/Cargo.toml index 2775e02e0cb..ca453521c37 100644 --- a/lib/types/Cargo.toml +++ b/lib/types/Cargo.toml @@ -23,6 +23,16 @@ enum-iterator = "0.7.0" target-lexicon = { version = "0.12.2", default-features = false } enumset.workspace = true bytecheck = "0.6.8" +webc = { workspace = true, default-features = false } +xxhash-rust = { version = "0.8.8", features = ["xxh64"] } +sha2 = { version = "0.10" } +hex = { version = "^0.4" } + +# `rand` uses `getrandom` transitively, and to be able to +# compile the project for `js`, we need to enable this feature +[dependencies.getrandom] +version = "*" +features = ["js"] [dev-dependencies] memoffset.workspace = true diff --git a/lib/types/src/lib.rs b/lib/types/src/lib.rs index db01db3d124..117f88408f2 100644 --- a/lib/types/src/lib.rs +++ b/lib/types/src/lib.rs @@ -58,6 +58,7 @@ mod initializers; mod libcalls; mod memory; mod module; +mod module_hash; mod serialize; mod stack; mod store_id; @@ -97,6 +98,7 @@ pub use crate::initializers::{ }; pub use crate::memory::{Memory32, Memory64, MemorySize}; pub use crate::module::{ExportsIterator, ImportKey, ImportsIterator, ModuleInfo}; +pub use crate::module_hash::{HashAlgorithm, ModuleHash}; pub use crate::units::{ Bytes, PageCountOutOfRange, Pages, WASM_MAX_PAGES, WASM_MIN_PAGES, WASM_PAGE_SIZE, }; diff --git a/lib/types/src/module.rs b/lib/types/src/module.rs index 9176a483e12..2c7f36ed959 100644 --- a/lib/types/src/module.rs +++ b/lib/types/src/module.rs @@ -8,9 +8,10 @@ use crate::entity::{EntityRef, PrimaryMap}; use crate::{ CustomSectionIndex, DataIndex, ElemIndex, ExportIndex, ExportType, ExternType, FunctionIndex, FunctionType, GlobalIndex, GlobalInit, GlobalType, ImportIndex, ImportType, LocalFunctionIndex, - LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryIndex, MemoryType, SignatureIndex, - TableIndex, TableInitializer, TableType, + LocalGlobalIndex, LocalMemoryIndex, LocalTableIndex, MemoryIndex, MemoryType, ModuleHash, + SignatureIndex, TableIndex, TableInitializer, TableType, }; + use indexmap::IndexMap; use rkyv::{ de::SharedDeserializeRegistry, ser::ScratchSpace, ser::Serializer, @@ -99,6 +100,10 @@ mod serde_imports { /// A translated WebAssembly module, excluding the function bodies and /// memory initializers. +/// +/// IMPORTANT: since this struct will be serialized as part of the compiled module artifact, +/// if you change this struct, do not forget to update [`MetadataHeader::version`](crate::serialize::MetadataHeader) +/// to make sure we don't break compatibility between versions. #[derive(Debug, Clone, Default)] #[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct ModuleInfo { @@ -111,6 +116,9 @@ pub struct ModuleInfo { #[cfg_attr(feature = "enable-serde", serde(skip_serializing, skip_deserializing))] pub id: ModuleId, + /// hash of the module + pub hash: Option, + /// The name of this wasm module, often found in the wasm file. pub name: Option, @@ -182,6 +190,7 @@ pub struct ModuleInfo { #[archive_attr(derive(CheckBytes, Debug))] pub struct ArchivableModuleInfo { name: Option, + hash: Option, imports: IndexMap, exports: IndexMap, start_function: Option, @@ -207,6 +216,7 @@ impl From for ArchivableModuleInfo { fn from(it: ModuleInfo) -> Self { Self { name: it.name, + hash: it.hash, imports: it.imports, exports: it.exports, start_function: it.start_function, @@ -235,6 +245,7 @@ impl From for ModuleInfo { Self { id: Default::default(), name: it.name, + hash: it.hash, imports: it.imports, exports: it.exports, start_function: it.start_function, @@ -325,6 +336,11 @@ impl ModuleInfo { Default::default() } + /// Returns the module hash if available + pub fn hash(&self) -> Option { + self.hash + } + /// Get the given passive element, if it exists. pub fn get_passive_element(&self, index: ElemIndex) -> Option<&[FunctionIndex]> { self.passive_elements.get(&index).map(|es| &**es) diff --git a/lib/types/src/module_hash.rs b/lib/types/src/module_hash.rs new file mode 100644 index 00000000000..ea876a8059a --- /dev/null +++ b/lib/types/src/module_hash.rs @@ -0,0 +1,118 @@ +use std::fmt::{self, Display, Formatter}; + +use rkyv::{Archive, CheckBytes, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; +use sha2::Digest; + +/// Hashing algorithm to be used for the module info +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum HashAlgorithm { + /// Sha256 + Sha256, + /// XXHash + XXHash, +} + +/// The hash of a WebAssembly module. +#[derive( + Debug, + Copy, + Clone, + PartialEq, + Eq, + Hash, + PartialOrd, + Ord, + RkyvSerialize, + RkyvDeserialize, + Archive, +)] +#[archive_attr(derive(CheckBytes, Debug))] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub enum ModuleHash { + /// xxhash + XXHash([u8; 8]), + + /// sha256 + Sha256([u8; 32]), +} + +impl ModuleHash { + /// Create a new [`ModuleHash`] from the raw xxhash hash. + pub fn xxhash_from_bytes(key: [u8; 8]) -> Self { + Self::XXHash(key) + } + + /// Create a new [`ModuleHash`] from the raw sha256 hash. + pub fn sha256_from_bytes(key: [u8; 32]) -> Self { + Self::Sha256(key) + } + + /// Parse a XXHash hash from a hex-encoded string. + pub fn xxhash_parse_hex(hex_str: &str) -> Result { + let mut hash = [0_u8; 8]; + hex::decode_to_slice(hex_str, &mut hash)?; + Ok(Self::xxhash_from_bytes(hash)) + } + + /// Parse a Sha256 hash from a hex-encoded string. + pub fn sha256_parse_hex(hex_str: &str) -> Result { + let mut hash = [0_u8; 32]; + hex::decode_to_slice(hex_str, &mut hash)?; + Ok(Self::sha256_from_bytes(hash)) + } + + /// Generate a new [`ModuleCache`] based on the XXHash hash of some bytes. + pub fn xxhash(wasm: impl AsRef<[u8]>) -> Self { + let wasm = wasm.as_ref(); + + let hash = xxhash_rust::xxh64::xxh64(wasm, 0); + + Self::XXHash(hash.to_ne_bytes()) + } + + /// Generate a new [`ModuleCache`] based on the Sha256 hash of some bytes. + pub fn sha256(wasm: impl AsRef<[u8]>) -> Self { + let wasm = wasm.as_ref(); + + let hash: [u8; 32] = sha2::Sha256::digest(wasm).into(); + + Self::Sha256(hash) + } + + /// Get the raw hash. + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::XXHash(bytes) => bytes.as_slice(), + Self::Sha256(bytes) => bytes.as_slice(), + } + } +} + +impl From for ModuleHash { + fn from(value: webc::metadata::AtomSignature) -> Self { + match value { + webc::metadata::AtomSignature::Sha256(bytes) => Self::Sha256(bytes), + } + } +} + +impl Display for ModuleHash { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + fn format(f: &mut Formatter<'_>, bytes: &[u8; N]) -> fmt::Result { + for byte in bytes { + write!(f, "{byte:02X}")?; + } + + Ok(()) + } + + match self { + Self::XXHash(bytes) => format(f, bytes)?, + Self::Sha256(bytes) => format(f, bytes)?, + } + + Ok(()) + } +} diff --git a/lib/types/src/serialize.rs b/lib/types/src/serialize.rs index 0c2044ce4fa..d98af1dac76 100644 --- a/lib/types/src/serialize.rs +++ b/lib/types/src/serialize.rs @@ -211,7 +211,7 @@ pub struct MetadataHeader { impl MetadataHeader { /// Current ABI version. Increment this any time breaking changes are made /// to the format of the serialized data. - pub const CURRENT_VERSION: u32 = 6; + pub const CURRENT_VERSION: u32 = 7; /// Magic number to identify wasmer metadata. const MAGIC: [u8; 8] = *b"WASMER\0\0"; diff --git a/lib/wasix/src/bin_factory/binary_package.rs b/lib/wasix/src/bin_factory/binary_package.rs index d73755f1da5..5a83725462b 100644 --- a/lib/wasix/src/bin_factory/binary_package.rs +++ b/lib/wasix/src/bin_factory/binary_package.rs @@ -9,12 +9,10 @@ use wasmer_config::package::{PackageHash, PackageId, PackageSource}; use webc::{compat::SharedBytes, Container}; use crate::{ - runtime::{ - module_cache::ModuleHash, - resolver::{PackageInfo, ResolveError}, - }, + runtime::resolver::{PackageInfo, ResolveError}, Runtime, }; +use wasmer_types::ModuleHash; #[derive(Derivative, Clone)] #[derivative(Debug)] diff --git a/lib/wasix/src/lib.rs b/lib/wasix/src/lib.rs index 53c61cb5ed4..936fad5054b 100644 --- a/lib/wasix/src/lib.rs +++ b/lib/wasix/src/lib.rs @@ -142,7 +142,7 @@ pub enum SpawnError { /// Failed to compile the Wasmer process #[error("compile error")] CompileError { - module_hash: crate::runtime::module_cache::ModuleHash, + module_hash: wasmer_types::ModuleHash, error: wasmer::CompileError, }, /// Invalid ABI diff --git a/lib/wasix/src/os/console/mod.rs b/lib/wasix/src/os/console/mod.rs index 67b1f167c1a..ca63393c32c 100644 --- a/lib/wasix/src/os/console/mod.rs +++ b/lib/wasix/src/os/console/mod.rs @@ -300,7 +300,7 @@ impl Console { data.insert_str(0, ConsoleConst::TERM_NO_WRAPAROUND); let mut stderr = self.stderr.clone(); - virtual_fs::AsyncWriteExt::write_all(&mut stderr, data.as_str().as_bytes()) + virtual_fs::AsyncWriteExt::write_all(&mut stderr, data.as_bytes()) .await .ok(); } diff --git a/lib/wasix/src/os/task/control_plane.rs b/lib/wasix/src/os/task/control_plane.rs index b7826a058b2..0845b075c08 100644 --- a/lib/wasix/src/os/task/control_plane.rs +++ b/lib/wasix/src/os/task/control_plane.rs @@ -7,7 +7,8 @@ use std::{ time::Duration, }; -use crate::{runtime::module_cache::ModuleHash, WasiProcess, WasiProcessId}; +use crate::{WasiProcess, WasiProcessId}; +use wasmer_types::ModuleHash; #[derive(Debug, Clone)] pub struct WasiControlPlane { @@ -212,7 +213,7 @@ pub enum ControlPlaneError { mod tests { use wasmer_wasix_types::wasix::ThreadStartType; - use crate::os::task::thread::WasiMemoryLayout; + use crate::{os::task::thread::WasiMemoryLayout, utils::xxhash_random}; use super::*; @@ -225,7 +226,7 @@ mod tests { enable_exponential_cpu_backoff: None, }); - let p1 = p.new_process(ModuleHash::xxhash_random()).unwrap(); + let p1 = p.new_process(xxhash_random()).unwrap(); let _t1 = p1 .new_thread(WasiMemoryLayout::default(), ThreadStartType::MainThread) .unwrap(); @@ -234,7 +235,7 @@ mod tests { .unwrap(); assert_eq!( - p.new_process(ModuleHash::xxhash_random()).unwrap_err(), + p.new_process(xxhash_random()).unwrap_err(), ControlPlaneError::TaskLimitReached { max: 2 } ); } @@ -248,7 +249,7 @@ mod tests { enable_exponential_cpu_backoff: None, }); - let p1 = p.new_process(ModuleHash::xxhash_random()).unwrap(); + let p1 = p.new_process(xxhash_random()).unwrap(); for _ in 0..10 { let _thread = p1 @@ -264,7 +265,7 @@ mod tests { .unwrap(); assert_eq!( - p.new_process(ModuleHash::xxhash_random()).unwrap_err(), + p.new_process(xxhash_random()).unwrap_err(), ControlPlaneError::TaskLimitReached { max: 2 } ); } diff --git a/lib/wasix/src/os/task/process.rs b/lib/wasix/src/os/task/process.rs index 5d13e9558ab..844dfd6f4a7 100644 --- a/lib/wasix/src/os/task/process.rs +++ b/lib/wasix/src/os/task/process.rs @@ -1,8 +1,6 @@ #[cfg(feature = "journal")] use crate::{journal::JournalEffector, syscalls::do_checkpoint_from_outside, unwind, WasiResult}; -use crate::{ - journal::SnapshotTrigger, runtime::module_cache::ModuleHash, WasiEnv, WasiRuntimeError, -}; +use crate::{journal::SnapshotTrigger, WasiEnv, WasiRuntimeError}; use serde::{Deserialize, Serialize}; #[cfg(feature = "journal")] use std::collections::HashSet; @@ -19,6 +17,7 @@ use std::{ }; use tracing::trace; use wasmer::FunctionEnvMut; +use wasmer_types::ModuleHash; use wasmer_wasix_types::{ types::Signal, wasi::{Errno, ExitCode, Snapshot0Clockid}, @@ -774,7 +773,7 @@ impl WasiProcess { }) } } - let (child, res) = futures::future::select_all(waits.into_iter().map(|a| Box::pin(a))) + let (child, res) = futures::future::select_all(waits.into_iter().map(Box::pin)) .await .0; diff --git a/lib/wasix/src/runners/wasi.rs b/lib/wasix/src/runners/wasi.rs index 72fabdcf268..42c65826886 100644 --- a/lib/wasix/src/runners/wasi.rs +++ b/lib/wasix/src/runners/wasi.rs @@ -13,9 +13,10 @@ use crate::{ capabilities::Capabilities, journal::{DynJournal, SnapshotTrigger}, runners::{wasi_common::CommonWasiOptions, MappedDirectory, MountedDirectory}, - runtime::{module_cache::ModuleHash, task_manager::VirtualTaskManagerExt}, + runtime::task_manager::VirtualTaskManagerExt, Runtime, WasiEnvBuilder, WasiError, WasiRuntimeError, }; +use wasmer_types::ModuleHash; use super::wasi_common::MappedCommand; diff --git a/lib/wasix/src/runners/wcgi/callbacks.rs b/lib/wasix/src/runners/wcgi/callbacks.rs index 1716691b314..80fb68955b3 100644 --- a/lib/wasix/src/runners/wcgi/callbacks.rs +++ b/lib/wasix/src/runners/wcgi/callbacks.rs @@ -3,9 +3,9 @@ use std::{collections::HashMap, sync::Arc}; use virtual_fs::Pipe; use wasmer::{Memory, Module, Store}; -use crate::{runtime::module_cache::ModuleHash, WasiEnv}; - use super::{create_env::default_recycle_env, handler::SetupBuilder, *}; +use crate::WasiEnv; +use wasmer_types::ModuleHash; /// Configuration used for creating a new environment pub struct CreateEnvConfig { diff --git a/lib/wasix/src/runners/wcgi/handler.rs b/lib/wasix/src/runners/wcgi/handler.rs index 6f0eb012bc8..4c2f0938599 100644 --- a/lib/wasix/src/runners/wcgi/handler.rs +++ b/lib/wasix/src/runners/wcgi/handler.rs @@ -18,12 +18,10 @@ use crate::{ callbacks::{CreateEnvConfig, RecycleEnvConfig}, Callbacks, }, - runtime::{ - module_cache::ModuleHash, - task_manager::{TaskWasm, TaskWasmRecycleProperties}, - }, + runtime::task_manager::{TaskWasm, TaskWasmRecycleProperties}, Runtime, VirtualTaskManager, WasiEnvBuilder, }; +use wasmer_types::ModuleHash; /// The shared object that manages the instantiaion of WASI executables and /// communicating with them via the CGI protocol. diff --git a/lib/wasix/src/runtime/mod.rs b/lib/wasix/src/runtime/mod.rs index 4ec940347fb..00e09360618 100644 --- a/lib/wasix/src/runtime/mod.rs +++ b/lib/wasix/src/runtime/mod.rs @@ -4,10 +4,8 @@ pub mod resolver; pub mod task_manager; pub use self::task_manager::{SpawnMemoryType, VirtualTaskManager}; -use self::{ - module_cache::{CacheError, ModuleHash}, - task_manager::InlineWaker, -}; +use self::{module_cache::CacheError, task_manager::InlineWaker}; +use wasmer_types::ModuleHash; use std::{ fmt, diff --git a/lib/wasix/src/runtime/module_cache/mod.rs b/lib/wasix/src/runtime/module_cache/mod.rs index 88baf4293ac..d786be8c40d 100644 --- a/lib/wasix/src/runtime/module_cache/mod.rs +++ b/lib/wasix/src/runtime/module_cache/mod.rs @@ -42,8 +42,9 @@ pub use self::{ fallback::FallbackCache, shared::SharedCache, thread_local::ThreadLocalCache, - types::{CacheError, ModuleCache, ModuleHash}, + types::{CacheError, ModuleCache}, }; +use wasmer_types::ModuleHash; #[cfg(feature = "sys-thread")] pub use self::filesystem::FileSystemCache; diff --git a/lib/wasix/src/runtime/module_cache/shared.rs b/lib/wasix/src/runtime/module_cache/shared.rs index 522980c6cb6..305e80ff654 100644 --- a/lib/wasix/src/runtime/module_cache/shared.rs +++ b/lib/wasix/src/runtime/module_cache/shared.rs @@ -1,7 +1,8 @@ use dashmap::DashMap; use wasmer::{Engine, Module}; -use crate::runtime::module_cache::{CacheError, ModuleCache, ModuleHash}; +use crate::runtime::module_cache::{CacheError, ModuleCache}; +use wasmer_types::ModuleHash; /// A [`ModuleCache`] based on a [DashMap]<[ModuleHash], [Module]>. #[derive(Debug, Default, Clone)] diff --git a/lib/wasix/src/runtime/module_cache/thread_local.rs b/lib/wasix/src/runtime/module_cache/thread_local.rs index bfccc24fa04..c23744ae19c 100644 --- a/lib/wasix/src/runtime/module_cache/thread_local.rs +++ b/lib/wasix/src/runtime/module_cache/thread_local.rs @@ -2,7 +2,8 @@ use std::{cell::RefCell, collections::HashMap}; use wasmer::{Engine, Module}; -use crate::runtime::module_cache::{CacheError, ModuleCache, ModuleHash}; +use crate::runtime::module_cache::{CacheError, ModuleCache}; +use wasmer_types::ModuleHash; std::thread_local! { static CACHED_MODULES: RefCell> diff --git a/lib/wasix/src/runtime/module_cache/types.rs b/lib/wasix/src/runtime/module_cache/types.rs index ea3dc6c27ff..79494f0e3b0 100644 --- a/lib/wasix/src/runtime/module_cache/types.rs +++ b/lib/wasix/src/runtime/module_cache/types.rs @@ -1,13 +1,7 @@ -use std::hash::Hash; -use std::{ - fmt::{self, Debug, Display, Formatter}, - ops::Deref, - path::PathBuf, -}; +use std::{fmt::Debug, ops::Deref, path::PathBuf}; -use rand::RngCore; -use sha2::Digest; use wasmer::{Engine, Module}; +use wasmer_types::ModuleHash; use crate::runtime::module_cache::FallbackCache; @@ -128,108 +122,6 @@ impl CacheError { } } -/// The hash of a WebAssembly module. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] -pub enum ModuleHash { - XXHash([u8; 8]), - Sha256([u8; 32]), -} - -impl ModuleHash { - /// Create a new [`ModuleHash`] from the raw xxhash hash. - pub fn xxhash_from_bytes(key: [u8; 8]) -> Self { - Self::XXHash(key) - } - - /// Create a new [`ModuleHash`] from the raw sha256 hash. - pub fn sha256_from_bytes(key: [u8; 32]) -> Self { - Self::Sha256(key) - } - - /// Creates a random xxhash for the module - pub fn xxhash_random() -> Self { - let mut rand = rand::thread_rng(); - let mut key = [0u8; 8]; - rand.fill_bytes(&mut key); - Self::xxhash_from_bytes(key) - } - - /// Creates a random sha256 hash for the module - pub fn sha256_random() -> Self { - let mut rand = rand::thread_rng(); - let mut key = [0u8; 32]; - rand.fill_bytes(&mut key); - Self::sha256_from_bytes(key) - } - - /// Parse a XXHash hash from a hex-encoded string. - pub fn xxhash_parse_hex(hex_str: &str) -> Result { - let mut hash = [0_u8; 8]; - hex::decode_to_slice(hex_str, &mut hash)?; - Ok(Self::xxhash_from_bytes(hash)) - } - - /// Parse a Sha256 hash from a hex-encoded string. - pub fn sha256_parse_hex(hex_str: &str) -> Result { - let mut hash = [0_u8; 32]; - hex::decode_to_slice(hex_str, &mut hash)?; - Ok(Self::sha256_from_bytes(hash)) - } - - /// Generate a new [`ModuleCache`] based on the XXHash hash of some bytes. - pub fn xxhash(wasm: impl AsRef<[u8]>) -> Self { - let wasm = wasm.as_ref(); - - let hash = xxhash_rust::xxh64::xxh64(wasm, 0); - - Self::XXHash(hash.to_ne_bytes()) - } - - /// Generate a new [`ModuleCache`] based on the Sha256 hash of some bytes. - pub fn sha256(wasm: impl AsRef<[u8]>) -> Self { - let wasm = wasm.as_ref(); - - let hash: [u8; 32] = sha2::Sha256::digest(wasm).into(); - - Self::Sha256(hash) - } - - /// Get the raw hash. - pub fn as_bytes(&self) -> &[u8] { - match self { - ModuleHash::XXHash(bytes) => bytes.as_slice(), - ModuleHash::Sha256(bytes) => bytes.as_slice(), - } - } -} - -impl From for ModuleHash { - fn from(value: webc::metadata::AtomSignature) -> Self { - match value { - webc::metadata::AtomSignature::Sha256(bytes) => ModuleHash::Sha256(bytes), - } - } -} - -impl Display for ModuleHash { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - fn format(f: &mut Formatter<'_>, bytes: &[u8; N]) -> fmt::Result { - for byte in bytes { - write!(f, "{byte:02X}")?; - } - - Ok(()) - } - - match self { - ModuleHash::XXHash(bytes) => format(f, bytes)?, - ModuleHash::Sha256(bytes) => format(f, bytes)?, - } - - Ok(()) - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/lib/wasix/src/state/builder.rs b/lib/wasix/src/state/builder.rs index e60971d4b6c..2fb85a3b430 100644 --- a/lib/wasix/src/state/builder.rs +++ b/lib/wasix/src/state/builder.rs @@ -18,14 +18,15 @@ use crate::{ capabilities::Capabilities, fs::{WasiFs, WasiFsRoot, WasiInodes}, os::task::control_plane::{ControlPlaneConfig, ControlPlaneError, WasiControlPlane}, - runtime::module_cache::ModuleHash, state::WasiState, syscalls::{ rewind_ext2, types::{__WASI_STDERR_FILENO, __WASI_STDIN_FILENO, __WASI_STDOUT_FILENO}, }, + utils::xxhash_random, Runtime, WasiEnv, WasiError, WasiFunctionEnv, WasiRuntimeError, }; +use wasmer_types::ModuleHash; use super::env::WasiEnvInit; @@ -910,7 +911,7 @@ impl WasiEnvBuilder { #[allow(clippy::result_large_err)] pub fn build(self) -> Result { - let module_hash = self.module_hash.unwrap_or_else(ModuleHash::xxhash_random); + let module_hash = self.module_hash.unwrap_or_else(xxhash_random); let init = self.build_init()?; WasiEnv::from_init(init, module_hash) } @@ -925,7 +926,7 @@ impl WasiEnvBuilder { self, store: &mut impl AsStoreMut, ) -> Result { - let module_hash = self.module_hash.unwrap_or_else(ModuleHash::xxhash_random); + let module_hash = self.module_hash.unwrap_or_else(xxhash_random); let init = self.build_init()?; let env = WasiEnv::from_init(init, module_hash)?; let func_env = WasiFunctionEnv::new(store, env); @@ -943,7 +944,7 @@ impl WasiEnvBuilder { module: Module, store: &mut impl AsStoreMut, ) -> Result<(Instance, WasiFunctionEnv), WasiRuntimeError> { - self.instantiate_ext(module, ModuleHash::xxhash_random(), store) + self.instantiate_ext(module, xxhash_random(), store) } #[allow(clippy::result_large_err)] @@ -959,7 +960,7 @@ impl WasiEnvBuilder { #[allow(clippy::result_large_err)] pub fn run(self, module: Module) -> Result<(), WasiRuntimeError> { - self.run_ext(module, ModuleHash::xxhash_random()) + self.run_ext(module, xxhash_random()) } #[allow(clippy::result_large_err)] @@ -971,7 +972,7 @@ impl WasiEnvBuilder { #[allow(clippy::result_large_err)] #[tracing::instrument(level = "debug", skip_all)] pub fn run_with_store(self, module: Module, store: &mut Store) -> Result<(), WasiRuntimeError> { - self.run_with_store_ext(module, ModuleHash::xxhash_random(), store) + self.run_with_store_ext(module, xxhash_random(), store) } #[allow(clippy::result_large_err)] diff --git a/lib/wasix/src/state/env.rs b/lib/wasix/src/state/env.rs index 57d539c13f0..a43c9d29155 100644 --- a/lib/wasix/src/state/env.rs +++ b/lib/wasix/src/state/env.rs @@ -34,11 +34,12 @@ use crate::{ process::{WasiProcess, WasiProcessId}, thread::{WasiMemoryLayout, WasiThread, WasiThreadHandle, WasiThreadId}, }, - runtime::{module_cache::ModuleHash, task_manager::InlineWaker, SpawnMemoryType}, + runtime::{task_manager::InlineWaker, SpawnMemoryType}, syscalls::platform_clock_time_get, Runtime, VirtualTaskManager, WasiControlPlane, WasiEnvBuilder, WasiError, WasiFunctionEnv, WasiResult, WasiRuntimeError, WasiStateCreationError, WasiVFork, }; +use wasmer_types::ModuleHash; pub(crate) use super::handles::*; use super::WasiState; diff --git a/lib/wasix/src/syscalls/wasi/fd_read.rs b/lib/wasix/src/syscalls/wasi/fd_read.rs index 78b95691d18..5558a29e52c 100644 --- a/lib/wasix/src/syscalls/wasi/fd_read.rs +++ b/lib/wasix/src/syscalls/wasi/fd_read.rs @@ -117,6 +117,7 @@ pub(crate) fn fd_read_internal_handler( Ok(ret) } +#[allow(clippy::await_holding_lock)] pub(crate) fn fd_read_internal( ctx: &mut FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasi/fd_write.rs b/lib/wasix/src/syscalls/wasi/fd_write.rs index e586d897163..079527e5deb 100644 --- a/lib/wasix/src/syscalls/wasi/fd_write.rs +++ b/lib/wasix/src/syscalls/wasi/fd_write.rs @@ -118,6 +118,7 @@ pub(crate) enum FdWriteSource<'a, M: MemorySize> { Buffer(Cow<'a, [u8]>), } +#[allow(clippy::await_holding_lock)] pub(crate) fn fd_write_internal( ctx: &FunctionEnvMut<'_, WasiEnv>, fd: WasiFd, diff --git a/lib/wasix/src/syscalls/wasix/sock_send_file.rs b/lib/wasix/src/syscalls/wasix/sock_send_file.rs index 70419d74cd5..75500152e66 100644 --- a/lib/wasix/src/syscalls/wasix/sock_send_file.rs +++ b/lib/wasix/src/syscalls/wasix/sock_send_file.rs @@ -48,6 +48,7 @@ pub fn sock_send_file( Ok(Errno::Success) } +#[allow(clippy::await_holding_lock)] pub(crate) fn sock_send_file_internal( ctx: &mut FunctionEnvMut<'_, WasiEnv>, sock: WasiFd, diff --git a/lib/wasix/src/utils/mod.rs b/lib/wasix/src/utils/mod.rs index cd930052860..798c9482d35 100644 --- a/lib/wasix/src/utils/mod.rs +++ b/lib/wasix/src/utils/mod.rs @@ -11,12 +11,22 @@ pub use self::{dummy_waker::WasiDummyWaker, thread_parker::WasiParkingLot}; pub(crate) use owned_mutex_guard::{ read_owned, write_owned, OwnedRwLockReadGuard, OwnedRwLockWriteGuard, }; +use rand::RngCore; +use wasmer_types::ModuleHash; use std::collections::BTreeSet; use wasmer::Module; use wasmer_wasix_types::wasi::Errno; +/// Creates a random xxhash for the module +pub fn xxhash_random() -> ModuleHash { + let mut rand = rand::thread_rng(); + let mut key = [0u8; 8]; + rand.fill_bytes(&mut key); + ModuleHash::xxhash_from_bytes(key) +} + /// Check if a provided module is compiled for some version of WASI. /// Use [`get_wasi_version`] to find out which version of WASI the module is. pub fn is_wasi_module(module: &Module) -> bool { diff --git a/rust-toolchain b/rust-toolchain index e4654bb6830..bc8a6589c98 100644 --- a/rust-toolchain +++ b/rust-toolchain @@ -1 +1 @@ -1.73 +1.74 diff --git a/tests/compilers/artifact.rs b/tests/compilers/artifact.rs index d619730d940..334ec9d2af3 100644 --- a/tests/compilers/artifact.rs +++ b/tests/compilers/artifact.rs @@ -42,7 +42,9 @@ fn artifact_serialization_build() { let path = PathBuf::from("tests/integration/cli/tests/wasm").join(file_name); let wasm_module = fs::read(path).unwrap(); let config = get_default_compiler_config().unwrap(); - let engine = Engine::new(config, target.clone(), Features::default()); + let mut engine = Engine::new(config, target.clone(), Features::default()); + + engine.set_hash_algorithm(Some(wasmer_types::HashAlgorithm::Sha256)); let module = Module::new(&engine, wasm_module).unwrap(); let serialized_bytes = module.serialize().unwrap(); diff --git a/tests/compilers/wasmu/linux/bash.wasmu b/tests/compilers/wasmu/linux/bash.wasmu index f42497bb426..250829379cd 100644 Binary files a/tests/compilers/wasmu/linux/bash.wasmu and b/tests/compilers/wasmu/linux/bash.wasmu differ diff --git a/tests/compilers/wasmu/linux/cowsay.wasmu b/tests/compilers/wasmu/linux/cowsay.wasmu index 261063223be..edad4ec6020 100644 Binary files a/tests/compilers/wasmu/linux/cowsay.wasmu and b/tests/compilers/wasmu/linux/cowsay.wasmu differ diff --git a/tests/compilers/wasmu/linux/python-3.11.3.wasmu b/tests/compilers/wasmu/linux/python-3.11.3.wasmu index 970ea31bb5b..3a00242a73d 100644 Binary files a/tests/compilers/wasmu/linux/python-3.11.3.wasmu and b/tests/compilers/wasmu/linux/python-3.11.3.wasmu differ diff --git a/tests/compilers/wasmu/windows/bash.wasmu b/tests/compilers/wasmu/windows/bash.wasmu index 9005b875235..129d6fdcda6 100644 Binary files a/tests/compilers/wasmu/windows/bash.wasmu and b/tests/compilers/wasmu/windows/bash.wasmu differ diff --git a/tests/compilers/wasmu/windows/cowsay.wasmu b/tests/compilers/wasmu/windows/cowsay.wasmu index dc444bc0e2c..a3b5c42aad2 100644 Binary files a/tests/compilers/wasmu/windows/cowsay.wasmu and b/tests/compilers/wasmu/windows/cowsay.wasmu differ diff --git a/tests/compilers/wasmu/windows/python-3.11.3.wasmu b/tests/compilers/wasmu/windows/python-3.11.3.wasmu index 191322d29b0..c48ae7bb06e 100644 Binary files a/tests/compilers/wasmu/windows/python-3.11.3.wasmu and b/tests/compilers/wasmu/windows/python-3.11.3.wasmu differ diff --git a/tests/lib/wast/Cargo.toml b/tests/lib/wast/Cargo.toml index 505d7f14848..f8306724d80 100644 --- a/tests/lib/wast/Cargo.toml +++ b/tests/lib/wast/Cargo.toml @@ -11,6 +11,7 @@ readme = "README.md" edition = "2018" [dependencies] +wasmer-types = { path = "../../../lib/types" } wasmer-wasix = { path = "../../../lib/wasix", version="=0.19.0" } wasmer = { path = "../../../lib/api", version = "=4.3.0-beta.1", default-features = false } virtual-fs = { path = "../../../lib/virtual-fs", version = "0.11.4" } @@ -21,7 +22,10 @@ serde = "1" tempfile = "3.6.0" thiserror = "1.0" futures = "0.3" -tokio = { version = "1", features = [ "io-util", "rt" ], default_features = false } +tokio = { version = "1", features = [ + "io-util", + "rt", +], default_features = false } [features] default = ["wat"] diff --git a/tests/lib/wast/src/wasi_wast.rs b/tests/lib/wast/src/wasi_wast.rs index 9ceecd10127..632b3e86185 100644 --- a/tests/lib/wast/src/wasi_wast.rs +++ b/tests/lib/wast/src/wasi_wast.rs @@ -13,10 +13,8 @@ use virtual_fs::{ AsyncWriteExt, FileSystem, Pipe, ReadBuf, RootFileSystemBuilder, }; use wasmer::{FunctionEnv, Imports, Module, Store}; -use wasmer_wasix::runtime::{ - module_cache::ModuleHash, - task_manager::{tokio::TokioTaskManager, InlineWaker}, -}; +use wasmer_types::ModuleHash; +use wasmer_wasix::runtime::task_manager::{tokio::TokioTaskManager, InlineWaker}; use wasmer_wasix::types::wasi::{Filesize, Timestamp}; use wasmer_wasix::{ generate_import_object_from_env, get_wasi_version, FsError, PluggableRuntime, VirtualFile,