Introduce a WebcVolumeFileSystem that works for any WEBC version

lib/c-api/Cargo.toml

@@ -23,16 +23,16 @@ crate-type = ["staticlib", "cdylib"] #"cdylib", "rlib", "staticlib"]
 # We rename `wasmer` to `wasmer-api` to avoid the conflict with this
 # library name (see `[lib]`).
 wasmer-api = { version = "=3.2.0-beta.2", path = "../api", default-features = false, package = "wasmer" }
+wasmer-compiler = { version = "=3.2.0-beta.2", path = "../compiler", optional = true }
 wasmer-compiler-cranelift = { version = "=3.2.0-beta.2", path = "../compiler-cranelift", optional = true }
-wasmer-compiler-singlepass = { version = "=3.2.0-beta.2", path = "../compiler-singlepass", optional = true }
 wasmer-compiler-llvm = { version = "=3.2.0-beta.2", path = "../compiler-llvm", optional = true }
+wasmer-compiler-singlepass = { version = "=3.2.0-beta.2", path = "../compiler-singlepass", optional = true }
 wasmer-emscripten = { version = "=3.2.0-beta.2", path = "../emscripten", optional = true }
-wasmer-compiler = { version = "=3.2.0-beta.2", path = "../compiler", optional = true }
 wasmer-middlewares = { version = "=3.2.0-beta.2", path = "../middlewares", optional = true }
-wasmer-wasix = { version = "0.2.0", path = "../wasi", features = ["host-fs", "host-vnet"], optional = true }
 wasmer-types = { version = "=3.2.0-beta.2", path = "../types" }
+wasmer-wasix = { version = "0.2.0", path = "../wasi", features = ["host-fs", "host-vnet"], optional = true }
+webc = { version = "5.0.0-rc.7", optional = true }
 virtual-fs = { version = "0.1.1", path = "../vfs", optional = true, default-features = false, features = ["static-fs"] }
-webc = { version = "=5.0.0-rc.6", optional = true }
 enumset = "1.0.2"
 cfg-if = "1.0"
 lazy_static = "1.4"

lib/cli/Cargo.toml

@@ -64,10 +64,7 @@ regex = "1.6.0"
 toml = "0.5.9"
 url = "2.3.1"
 libc = { version = "^0.2", default-features = false }
-webc = { version = "=5.0.0-rc.6" }
-# HACK(Michael-F-Bryan): Remove this once a new version of wapm-targz-to-pirita
-# is published that doesn't have a public dependency on webc
-webc_v4 = { version = "4", package = "webc" }
+webc = { version = "5.0.0-rc.7" }
 isatty = "0.1.9"
 dialoguer = "0.10.2"
 tldextract = "0.6.0"
@@ -85,7 +82,7 @@ wasm-coredump-builder = { version = "0.1.11" }
 tracing = { version = "0.1" }
 tracing-subscriber = { version = "0.3", features = [ "env-filter", "fmt" ] }
 clap-verbosity-flag = "1"
-wapm-targz-to-pirita = "0.1.10"
+wapm-targz-to-pirita = "0.2.0"
 
 [target.'cfg(not(target_arch = "riscv64"))'.dependencies]
 http_req  = { version="^0.8", default-features = false, features = ["rust-tls"] }

lib/cli/src/commands/run.rs

@@ -15,7 +15,7 @@ use wasmer::FunctionEnv;
 use wasmer::*;
 use wasmer_cache::{Cache, FileSystemCache, Hash};
 use wasmer_types::Type as ValueType;
-use wasmer_wasix::runners::{Runner, WapmContainer};
+use wasmer_wasix::runners::Runner;
 
 mod wasi;
 
@@ -214,7 +214,7 @@ impl RunWithPathBuf {
     fn inner_execute(&self) -> Result<()> {
         #[cfg(feature = "webc_runner")]
         {
-            if let Ok(pf) = WapmContainer::from_path(self.path.clone()) {
+            if let Ok(pf) = webc::Container::from_disk(self.path.clone()) {
                 return self.run_container(pf, self.command_name.as_deref(), &self.args);
             }
         }
@@ -371,7 +371,7 @@ impl RunWithPathBuf {
     #[cfg(feature = "webc_runner")]
     fn run_container(
         &self,
-        container: WapmContainer,
+        container: webc::Container,
         id: Option<&str>,
         args: &[String],
     ) -> Result<(), anyhow::Error> {

lib/cli/src/commands/run_unstable.rs

@@ -26,9 +26,9 @@ use wasmer::{
 use wasmer_cache::Cache;
 use wasmer_compiler::ArtifactBuild;
 use wasmer_registry::Package;
-use wasmer_wasix::runners::{wcgi::AbortHandle, MappedDirectory, Runner, WapmContainer};
-use webc::metadata::Manifest;
-use webc_v4::DirOrFile;
+use wasmer_wasix::runners::wcgi::AbortHandle;
+use wasmer_wasix::runners::{MappedDirectory, Runner};
+use webc::{metadata::Manifest, v1::DirOrFile, Container};
 
 use crate::{
     store::StoreOptions,
@@ -72,10 +72,10 @@ impl RunUnstable {
 
         let (mut store, _) = self.store.get_store()?;
 
-        let cache = self.wasmer_home.module_cache();
-        let result = match target.load(cache, &store)? {
+        let mut cache = self.wasmer_home.module_cache();
+        let result = match target.load(&mut cache, &store)? {
             ExecutableTarget::WebAssembly(wasm) => self.execute_wasm(&target, &wasm, &mut store),
-            ExecutableTarget::Webc(container, cache) => {
+            ExecutableTarget::Webc(container) => {
                 self.execute_webc(&target, container, cache, &mut store)
             }
         };
@@ -114,7 +114,7 @@ impl RunUnstable {
     fn execute_webc(
         &self,
         target: &TargetOnDisk,
-        container: WapmContainer,
+        container: Container,
         mut cache: ModuleCache,
         store: &mut Store,
     ) -> Result<(), Error> {
@@ -136,6 +136,8 @@ impl RunUnstable {
             .map(|(base, version)| base)
             .unwrap_or_else(|| command.runner.as_str());
 
+        let cache = Mutex::new(cache);
+
         match runner_base {
             webc::metadata::annotations::EMSCRIPTEN_RUNNER_URI => {
                 let mut runner = wasmer_wasix::runners::emscripten::EmscriptenRunner::new(store);
@@ -149,6 +151,7 @@ impl RunUnstable {
             webc::metadata::annotations::WCGI_RUNNER_URI => {
                 let mut runner = wasmer_wasix::runners::wcgi::WcgiRunner::new(id).with_compile(
                     move |engine, bytes| {
+                        let mut cache = cache.lock().unwrap();
                         compile_wasm_cached("".to_string(), bytes, &mut cache, engine)
                     },
                 );
@@ -173,6 +176,7 @@ impl RunUnstable {
             | webc::metadata::annotations::WASI_RUNNER_URI => {
                 let mut runner = wasmer_wasix::runners::wasi::WasiRunner::new(store)
                     .with_compile(move |engine, bytes| {
+                        let mut cache = cache.lock().unwrap();
                         compile_wasm_cached("".to_string(), bytes, &mut cache, engine)
                     })
                     .with_args(self.args.clone())
@@ -313,7 +317,7 @@ fn compile_directory_to_webc(dir: &Path) -> Result<Vec<u8>, Error> {
     let mut files = BTreeMap::new();
     load_files_from_disk(&mut files, dir, dir)?;
 
-    let wasmer_toml = webc_v4::DirOrFile::File("wasmer.toml".into());
+    let wasmer_toml = DirOrFile::File("wasmer.toml".into());
     if let Some(toml_data) = files.remove(&wasmer_toml) {
         // HACK(Michael-F-Bryan): The version of wapm-targz-to-pirita we are
         // using doesn't know we renamed "wapm.toml" to "wasmer.toml", so we
@@ -325,7 +329,7 @@ fn compile_directory_to_webc(dir: &Path) -> Result<Vec<u8>, Error> {
     }
 
     let functions = wapm_targz_to_pirita::TransformManifestFunctions::default();
-    wapm_targz_to_pirita::generate_webc_file(files, &dir.to_path_buf(), None, &functions)
+    wapm_targz_to_pirita::generate_webc_file(files, dir, None, &functions)
 }
 
 fn load_files_from_disk(files: &mut FileMap, dir: &Path, base: &Path) -> Result<(), Error> {
@@ -339,11 +343,11 @@ fn load_files_from_disk(files: &mut FileMap, dir: &Path, base: &Path) -> Result<
 
         if path.is_dir() {
             load_files_from_disk(files, &path, base)?;
-            files.insert(webc_v4::DirOrFile::Dir(relative_path), Vec::new());
+            files.insert(DirOrFile::Dir(relative_path), Vec::new());
         } else if path.is_file() {
             let data = std::fs::read(&path)
                 .with_context(|| format!("Unable to read \"{}\"", path.display()))?;
-            files.insert(webc_v4::DirOrFile::File(relative_path), data);
+            files.insert(DirOrFile::File(relative_path), data);
         }
     }
     Ok(())
@@ -458,42 +462,38 @@ impl TargetOnDisk {
         }
     }
 
-    fn load(&self, mut cache: ModuleCache, store: &Store) -> Result<ExecutableTarget, Error> {
+    fn load(&self, cache: &mut ModuleCache, store: &Store) -> Result<ExecutableTarget, Error> {
         match self {
             TargetOnDisk::Webc(webc) => {
                 // As an optimisation, try to use the mmapped version first.
-                if let Ok(container) = WapmContainer::from_path(webc.clone()) {
-                    return Ok(ExecutableTarget::Webc(container, cache));
+                if let Ok(container) = Container::from_disk(webc.clone()) {
+                    return Ok(ExecutableTarget::Webc(container));
                 }
 
                 // Otherwise, fall back to the version that reads everything
                 // into memory.
                 let bytes = std::fs::read(webc)
                     .with_context(|| format!("Unable to read \"{}\"", webc.display()))?;
-                let container = WapmContainer::from_bytes(bytes.into())?;
+                let container = Container::from_bytes(bytes)?;
 
-                Ok(ExecutableTarget::Webc(container, cache))
+                Ok(ExecutableTarget::Webc(container))
             }
             TargetOnDisk::Directory(dir) => {
                 // FIXME: Runners should be able to load directories directly
                 // instead of needing to compile to a WEBC file.
                 let webc = compile_directory_to_webc(dir).with_context(|| {
                     format!("Unable to bundle \"{}\" as a WEBC package", dir.display())
                 })?;
-                let container = WapmContainer::from_bytes(webc.into())
+                let container = Container::from_bytes(webc)
                     .context("Unable to parse the generated WEBC file")?;
 
-                Ok(ExecutableTarget::Webc(container, cache))
+                Ok(ExecutableTarget::Webc(container))
             }
             TargetOnDisk::WebAssemblyBinary(path) => {
                 let wasm = std::fs::read(path)
                     .with_context(|| format!("Unable to read \"{}\"", path.display()))?;
-                let module = compile_wasm_cached(
-                    path.display().to_string(),
-                    &wasm,
-                    &mut cache,
-                    store.engine(),
-                )?;
+                let module =
+                    compile_wasm_cached(path.display().to_string(), &wasm, cache, store.engine())?;
                 Ok(ExecutableTarget::WebAssembly(module))
             }
             TargetOnDisk::Wat(path) => {
@@ -502,12 +502,8 @@ impl TargetOnDisk {
                 let wasm =
                     wasmer::wat2wasm(&wat).context("Unable to convert the WAT to WebAssembly")?;
 
-                let module = compile_wasm_cached(
-                    path.display().to_string(),
-                    &wasm,
-                    &mut cache,
-                    store.engine(),
-                )?;
+                let module =
+                    compile_wasm_cached(path.display().to_string(), &wasm, cache, store.engine())?;
                 Ok(ExecutableTarget::WebAssembly(module))
             }
             TargetOnDisk::Artifact(artifact) => {
@@ -568,7 +564,7 @@ fn compile_wasm_cached(
 #[derive(Debug, Clone)]
 enum ExecutableTarget {
     WebAssembly(Module),
-    Webc(WapmContainer, ModuleCache),
+    Webc(Container),
 }
 
 fn generate_coredump(err: &Error, source: &Path, coredump_path: &Path) -> Result<(), Error> {

lib/registry/Cargo.toml

@@ -27,7 +27,7 @@ tar = "0.4.38"
 flate2 = "1.0.24"
 semver = "1.0.14"
 lzma-rs = "0.2.0"
-webc = { version = "=5.0.0-rc.6", features = ["mmap"] }
+webc = { version = "5.0.0-rc.7", features = ["mmap"] }
 hex = "0.4.3"
 tokio = "1.24.0"
 log = "0.4.17"

lib/vfs/Cargo.toml

@@ -11,7 +11,7 @@ libc = { version = "^0.2", default-features = false, optional = true }
 thiserror = "1"
 tracing = { version = "0.1" }
 typetag = { version = "0.1", optional = true }
-webc = { version = "=5.0.0-rc.6", optional = true }
+webc = { version = "5.0.0-rc.7", optional = true }
 slab = { version = "0.4" }
 derivative = "2.2.0"
 anyhow = { version = "1.0.66", optional = true }

lib/vfs/src/lib.rs

@@ -40,6 +40,8 @@ pub mod static_fs;
 mod trace_fs;
 #[cfg(feature = "webc-fs")]
 pub mod webc_fs;
+#[cfg(feature = "webc-fs")]
+mod webc_volume_fs;
 
 pub use arc_box_file::*;
 pub use arc_file::*;
@@ -57,6 +59,8 @@ pub use special_file::*;
 pub use tmp_fs::*;
 pub use trace_fs::TraceFileSystem;
 pub use union_fs::*;
+#[cfg(feature = "webc-fs")]
+pub use webc_volume_fs::WebcVolumeFileSystem;
 pub use zero_file::*;
 
 pub type Result<T> = std::result::Result<T, FsError>;

lib/vfs/src/overlay_fs.rs

@@ -735,7 +735,7 @@ mod tests {
 
     fn load_webc(bytes: &'static [u8]) -> WebcFileSystem<WebCOwned> {
         let options = ParseOptions::default();
-        let webc = WebCOwned::parse(Bytes::from_static(bytes), &options).unwrap();
+        let webc = WebCOwned::parse(bytes, &options).unwrap();
         WebcFileSystem::init_all(Arc::new(webc))
     }