From a75522e999c241a8978506e7ade34b763dab0c1a Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 28 Mar 2023 00:41:08 +0800 Subject: [PATCH] Update the runner trait to accept a webc::Container --- lib/wasi/src/runners/container.rs | 202 ---------------------------- lib/wasi/src/runners/emscripten.rs | 13 +- lib/wasi/src/runners/mod.rs | 6 +- lib/wasi/src/runners/runner.rs | 10 +- lib/wasi/src/runners/wasi.rs | 24 ++-- lib/wasi/src/runners/wasi_common.rs | 18 ++- lib/wasi/src/runners/wcgi/runner.rs | 26 ++-- lib/wasi/tests/runners.rs | 14 +- 8 files changed, 63 insertions(+), 250 deletions(-) delete mode 100644 lib/wasi/src/runners/container.rs diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs deleted file mode 100644 index 5ac8d62f883..00000000000 --- a/lib/wasi/src/runners/container.rs +++ /dev/null @@ -1,202 +0,0 @@ -use std::{path::PathBuf, sync::Arc}; - -use bytes::Bytes; -use virtual_fs::{webc_fs::WebcFileSystem, FileSystem}; -use webc::{ - metadata::Manifest, - v1::{ParseOptions, WebC, WebCMmap, WebCOwned}, - Version, -}; - -/// A parsed WAPM package. -#[derive(Debug, Clone)] -pub struct WapmContainer { - repr: Repr, -} - -#[allow(dead_code)] // Some pub(crate) items are only used behind #[cfg] code -impl WapmContainer { - /// Parses a .webc container file. Since .webc files - /// can be very large, only file paths are allowed. - pub fn from_path(path: PathBuf) -> std::result::Result { - let webc = webc::v1::WebCMmap::parse(path, &ParseOptions::default())?; - Ok(Self { - repr: Repr::V1Mmap(Arc::new(webc)), - }) - } - - pub fn from_bytes(bytes: Bytes) -> std::result::Result { - match webc::detect(bytes.as_ref())? { - Version::V1 => { - let webc = WebCOwned::parse(bytes, &ParseOptions::default())?; - Ok(WapmContainer { - repr: Repr::V1Owned(Arc::new(webc)), - }) - } - Version::V2 => todo!(), - other => Err(WebcParseError::UnsupportedVersion(other)), - } - } - - /// Returns the bytes of a file or a stringified error - pub fn get_file<'b>(&'b self, path: &str) -> Result<&'b [u8], String> { - match &self.repr { - Repr::V1Mmap(mapped) => mapped - .get_file(&mapped.get_package_name(), path) - .map_err(|e| e.0), - Repr::V1Owned(owned) => owned - .get_file(&owned.get_package_name(), path) - .map_err(|e| e.0), - } - } - - /// Returns a list of volumes in this container - pub fn get_volumes(&self) -> Vec { - match &self.repr { - Repr::V1Mmap(mapped) => mapped.volumes.keys().cloned().collect(), - Repr::V1Owned(owned) => owned.volumes.keys().cloned().collect(), - } - } - - pub fn get_atom(&self, name: &str) -> Option<&[u8]> { - match &self.repr { - Repr::V1Mmap(mapped) => mapped.get_atom(&mapped.get_package_name(), name).ok(), - Repr::V1Owned(owned) => owned.get_atom(&owned.get_package_name(), name).ok(), - } - } - - /// Lookup .wit bindings by name and parse them - pub fn get_bindings( - &self, - bindings: &str, - ) -> std::result::Result { - let bindings = self - .manifest() - .bindings - .iter() - .find(|b| b.name == bindings) - .ok_or_else(|| ParseBindingsError::NoBindings(bindings.to_string()))?; - - T::parse_bindings(self, &bindings.annotations).map_err(ParseBindingsError::ParseBindings) - } - - pub fn manifest(&self) -> &Manifest { - match &self.repr { - Repr::V1Mmap(mapped) => &mapped.manifest, - Repr::V1Owned(owned) => &owned.manifest, - } - } - - // HACK(Michael-F-Bryan): WapmContainer originally exposed its Arc - // field, so every man and his dog accessed it directly instead of going - // through the WapmContainer abstraction. This is an escape hatch to make - // that code keep working for the time being. - // #[deprecated] - pub(crate) fn v1(&self) -> &WebC<'_> { - match &self.repr { - Repr::V1Mmap(mapped) => mapped, - Repr::V1Owned(owned) => owned, - } - } - - /// Load a volume as a [`FileSystem`] node. - pub(crate) fn volume_fs(&self, package_name: &str) -> Box { - match &self.repr { - Repr::V1Mmap(mapped) => { - Box::new(WebcFileSystem::init(Arc::clone(mapped), package_name)) - } - Repr::V1Owned(owned) => Box::new(WebcFileSystem::init(Arc::clone(owned), package_name)), - } - } - - /// Get the entire container as a single filesystem and a list of suggested - /// directories to preopen. - pub(crate) fn container_fs(&self) -> Arc { - match &self.repr { - Repr::V1Mmap(mapped) => { - let fs = WebcFileSystem::init_all(Arc::clone(mapped)); - Arc::new(fs) - } - Repr::V1Owned(owned) => { - let fs = WebcFileSystem::init_all(Arc::clone(owned)); - Arc::new(fs) - } - } - } -} - -#[derive(Debug, Clone)] -enum Repr { - V1Mmap(Arc), - V1Owned(Arc), -} - -/// Error that happened while parsing .wit bindings -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)] -pub enum ParseBindingsError { - /// No bindings are available for the given lookup key - NoBindings(String), - /// Error happened during parsing - ParseBindings(String), -} - -/// Trait to parse bindings (any kind of bindings) for container .wasm files (usually .wit format) -pub trait Bindings { - /// Function that takes annotations in a free-form `Value` struct and returns the parsed bindings or an error - fn parse_bindings(_: &WapmContainer, _: &serde_cbor::Value) -> Result - where - Self: Sized; -} - -/// WIT bindings -#[derive(Default, Debug, Copy, Clone)] -pub struct WitBindings {} - -impl WitBindings { - /// Unused: creates default wit bindings - pub fn parse(_s: &str) -> Result { - Ok(Self::default()) - } -} - -impl Bindings for WitBindings { - fn parse_bindings( - container: &WapmContainer, - value: &serde_cbor::Value, - ) -> Result { - let value: webc::metadata::BindingsExtended = - serde_cbor::from_slice(&serde_cbor::to_vec(value).unwrap()) - .map_err(|e| format!("could not parse WitBindings annotations: {e}"))?; - - let mut wit_bindgen_filepath = value.exports().unwrap_or_default().to_string(); - - for v in container.get_volumes() { - let schema = format!("{v}://"); - if wit_bindgen_filepath.starts_with(&schema) { - wit_bindgen_filepath = wit_bindgen_filepath.replacen(&schema, "", 1); - break; - } - } - - let wit_bindings = container - .get_file(&wit_bindgen_filepath) - .map_err(|e| format!("could not get WitBindings file {wit_bindgen_filepath:?}: {e}"))?; - - let wit_bindings_str = std::str::from_utf8(wit_bindings) - .map_err(|e| format!("could not get WitBindings file {wit_bindgen_filepath:?}: {e}"))?; - - Self::parse(wit_bindings_str) - } -} - -/// Error that ocurred while parsing the .webc file -#[derive(Debug, thiserror::Error)] -pub enum WebcParseError { - /// Parse error - #[error("Unable to parse the WEBC file")] - Parse(#[from] webc::v1::Error), - #[error("Unable to determine the WEBC version")] - Detect(#[from] webc::DetectError), - #[error("Unsupported WEBC version, {_0}")] - UnsupportedVersion(Version), -} diff --git a/lib/wasi/src/runners/emscripten.rs b/lib/wasi/src/runners/emscripten.rs index 81b66a7714e..4a4b0f8cd9e 100644 --- a/lib/wasi/src/runners/emscripten.rs +++ b/lib/wasi/src/runners/emscripten.rs @@ -1,6 +1,5 @@ //! WebC container support for running Emscripten modules -use crate::runners::WapmContainer; use anyhow::{anyhow, Context, Error}; use serde::{Deserialize, Serialize}; use wasmer::{FunctionEnv, Instance, Module, Store}; @@ -8,7 +7,10 @@ use wasmer_emscripten::{ generate_emscripten_env, is_emscripten_module, run_emscripten_instance, EmEnv, EmscriptenGlobals, }; -use webc::metadata::{annotations::Emscripten, Command}; +use webc::{ + metadata::{annotations::Emscripten, Command}, + Container, +}; #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct EmscriptenRunner { @@ -57,7 +59,7 @@ impl crate::runners::Runner for EmscriptenRunner { &mut self, command_name: &str, command: &Command, - container: &WapmContainer, + container: &Container, ) -> Result { let Emscripten { atom: atom_name, @@ -65,8 +67,9 @@ impl crate::runners::Runner for EmscriptenRunner { .. } = command.annotation("emscripten")?.unwrap_or_default(); let atom_name = atom_name.context("The atom name is required")?; - let atom_bytes = container - .get_atom(&atom_name) + let atoms = container.atoms(); + let atom_bytes = atoms + .get(&atom_name) .with_context(|| format!("Unable to read the \"{atom_name}\" atom"))?; let mut module = Module::new(&self.store, atom_bytes)?; diff --git a/lib/wasi/src/runners/mod.rs b/lib/wasi/src/runners/mod.rs index d08667ba8df..5818bc48fba 100644 --- a/lib/wasi/src/runners/mod.rs +++ b/lib/wasi/src/runners/mod.rs @@ -1,4 +1,3 @@ -mod container; mod runner; #[cfg(feature = "webc_runner_rt_emscripten")] @@ -10,10 +9,7 @@ mod wasi_common; #[cfg(feature = "webc_runner_rt_wcgi")] pub mod wcgi; -pub use self::{ - container::{Bindings, WapmContainer, WebcParseError, WitBindings}, - runner::Runner, -}; +pub use self::runner::Runner; #[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct MappedDirectory { diff --git a/lib/wasi/src/runners/runner.rs b/lib/wasi/src/runners/runner.rs index ef330b05801..44248a8fdf3 100644 --- a/lib/wasi/src/runners/runner.rs +++ b/lib/wasi/src/runners/runner.rs @@ -1,7 +1,5 @@ use anyhow::Error; -use webc::metadata::Command; - -use crate::runners::WapmContainer; +use webc::{metadata::Command, Container}; /// Trait that all runners have to implement pub trait Runner { @@ -19,11 +17,11 @@ pub trait Runner { &mut self, command_name: &str, cmd: &Command, - container: &WapmContainer, + container: &Container, ) -> Result; /// Runs the container if the container has an `entrypoint` in the manifest - fn run(&mut self, container: &WapmContainer) -> Result { + fn run(&mut self, container: &Container) -> Result { let cmd = match container.manifest().entrypoint.as_ref() { Some(s) => s, None => { @@ -35,7 +33,7 @@ pub trait Runner { } /// Runs the given `cmd` on the container - fn run_cmd(&mut self, container: &WapmContainer, cmd: &str) -> Result { + fn run_cmd(&mut self, container: &Container, cmd: &str) -> Result { let command_to_exec = container .manifest() .commands diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 479ae00a12a..9ba8f519a61 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -4,11 +4,15 @@ use std::sync::Arc; use anyhow::{Context, Error}; use serde::{Deserialize, Serialize}; +use virtual_fs::WebcVolumeFileSystem; use wasmer::{Module, Store}; -use webc::metadata::{annotations::Wasi, Command}; +use webc::{ + metadata::{annotations::Wasi, Command}, + Container, +}; use crate::{ - runners::{wasi_common::CommonWasiOptions, MappedDirectory, WapmContainer}, + runners::{wasi_common::CommonWasiOptions, MappedDirectory}, PluggableRuntime, VirtualTaskManager, WasiEnvBuilder, }; @@ -118,13 +122,14 @@ impl WasiRunner { fn prepare_webc_env( &self, - container: &WapmContainer, + container: &Container, program_name: &str, wasi: &Wasi, ) -> Result { - let mut builder = - self.wasi - .prepare_webc_env(container.container_fs(), program_name, wasi)?; + let fs = WebcVolumeFileSystem::mount_all(container); + let mut builder = self + .wasi + .prepare_webc_env(Arc::new(fs), program_name, wasi)?; if let Some(tasks) = &self.tasks { let rt = PluggableRuntime::new(Arc::clone(tasks)); @@ -148,14 +153,15 @@ impl crate::runners::Runner for WasiRunner { &mut self, command_name: &str, command: &Command, - container: &WapmContainer, + container: &Container, ) -> Result { let wasi = command .annotation("wasi")? .unwrap_or_else(|| Wasi::new(command_name)); let atom_name = &wasi.atom; - let atom = container - .get_atom(atom_name) + let atoms = container.atoms(); + let atom = atoms + .get(atom_name) .with_context(|| format!("Unable to get the \"{atom_name}\" atom"))?; let mut module = Module::new(&self.store, atom)?; diff --git a/lib/wasi/src/runners/wasi_common.rs b/lib/wasi/src/runners/wasi_common.rs index 55ef2b74ff3..facc3175e1e 100644 --- a/lib/wasi/src/runners/wasi_common.rs +++ b/lib/wasi/src/runners/wasi_common.rs @@ -30,9 +30,14 @@ impl CommonWasiOptions { let fs = prepare_filesystem(&self.mapped_dirs, container_fs, |path| { builder.add_preopen_dir(path).map_err(Error::from) })?; - builder.set_fs(fs); + builder.add_preopen_dir("/")?; - builder.add_preopen_dir(".")?; + if fs.read_dir(".".as_ref()).is_ok() { + // Sometimes "." won't be mounted so preopening will fail. + builder.add_preopen_dir(".")?; + } + + builder.set_fs(fs); self.populate_env(wasi, &mut builder); self.populate_args(wasi, &mut builder); @@ -80,7 +85,6 @@ fn prepare_filesystem( if !mapped_dirs.is_empty() { let host_fs: Arc = Arc::new(crate::default_fs_backing()); - dbg!(mapped_dirs); for dir in mapped_dirs { let MappedDirectory { host, guest } = dir; @@ -133,7 +137,8 @@ fn create_dir_all(fs: &dyn FileSystem, path: &Path) -> Result<(), Error> { mod tests { use tempfile::TempDir; - use crate::runners::WapmContainer; + use virtual_fs::WebcVolumeFileSystem; + use webc::Container; use super::*; @@ -149,9 +154,10 @@ mod tests { guest: "/home".to_string(), host: sub_dir, }]; - let container = WapmContainer::from_bytes(PYTHON.into()).unwrap(); + let container = Container::from_bytes(PYTHON).unwrap(); + let webc_fs = WebcVolumeFileSystem::mount_all(&container); - let fs = prepare_filesystem(&mapping, container.container_fs(), |_| Ok(())).unwrap(); + let fs = prepare_filesystem(&mapping, Arc::new(webc_fs), |_| Ok(())).unwrap(); assert!(fs.metadata("/home/file.txt".as_ref()).unwrap().is_file()); assert!(fs.metadata("lib".as_ref()).unwrap().is_dir()); diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index bf91c75d4bb..3ce19dc2ed2 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -7,19 +7,23 @@ use hyper::Body; use tower::{make::Shared, ServiceBuilder}; use tower_http::{catch_panic::CatchPanicLayer, cors::CorsLayer, trace::TraceLayer}; use tracing::Span; -use virtual_fs::FileSystem; +use virtual_fs::{FileSystem, WebcVolumeFileSystem}; use wasmer::{Engine, Module, Store}; use wcgi_host::CgiDialect; -use webc::metadata::{ - annotations::{Wasi, Wcgi}, - Command, Manifest, +use webc::{ + compat::SharedBytes, + metadata::{ + annotations::{Wasi, Wcgi}, + Command, Manifest, + }, + Container, }; use crate::{ runners::{ wasi_common::CommonWasiOptions, wcgi::handler::{Handler, SharedState}, - MappedDirectory, WapmContainer, + MappedDirectory, }, runtime::task_manager::tokio::TokioTaskManager, PluggableRuntime, VirtualTaskManager, WasiEnvBuilder, @@ -117,7 +121,7 @@ impl WcgiRunner { .get_atom(atom_name) .with_context(|| format!("Unable to retrieve the \"{atom_name}\" atom"))?; - let module = ctx.compile(atom).context("Unable to compile the atom")?; + let module = ctx.compile(&atom).context("Unable to compile the atom")?; Ok(module) } @@ -178,7 +182,7 @@ impl WcgiRunner { // TODO(Michael-F-Bryan): Pass this to Runner::run() as a "&dyn RunnerContext" // when we rewrite the "Runner" trait. struct RunnerContext<'a> { - container: &'a WapmContainer, + container: &'a Container, command: &'a Command, engine: Engine, store: Arc, @@ -202,8 +206,8 @@ impl RunnerContext<'_> { &self.store } - fn get_atom(&self, name: &str) -> Option<&[u8]> { - self.container.get_atom(name) + fn get_atom(&self, name: &str) -> Option { + self.container.atoms().remove(name) } fn compile(&self, wasm: &[u8]) -> Result { @@ -212,7 +216,7 @@ impl RunnerContext<'_> { } fn container_fs(&self) -> Arc { - self.container.container_fs() + Arc::new(WebcVolumeFileSystem::mount_all(self.container)) } } @@ -227,7 +231,7 @@ impl crate::runners::Runner for WcgiRunner { &mut self, command_name: &str, command: &Command, - container: &WapmContainer, + container: &Container, ) -> Result { let store = self.config.store.clone().unwrap_or_default(); diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs index 729bc0aa95f..e55f9c483b2 100644 --- a/lib/wasi/tests/runners.rs +++ b/lib/wasi/tests/runners.rs @@ -4,7 +4,8 @@ use std::{path::Path, time::Duration}; use once_cell::sync::Lazy; use reqwest::Client; -use wasmer_wasix::runners::{Runner, WapmContainer}; +use wasmer_wasix::runners::Runner; +use webc::Container; #[cfg(feature = "webc_runner_rt_wasi")] mod wasi { @@ -20,7 +21,7 @@ mod wasi { async fn can_run_wat2wasm() { let webc = download_cached("https://wapm.io/wasmer/wabt").await; let store = Store::default(); - let container = WapmContainer::from_bytes(webc).unwrap(); + let container = Container::from_bytes(webc).unwrap(); let runner = WasiRunner::new(store); let command = &container.manifest().commands["wat2wasm"]; @@ -32,7 +33,7 @@ mod wasi { let webc = download_cached("https://wapm.io/wasmer/wabt").await; let store = Store::default(); let tasks = TokioTaskManager::new(Handle::current()); - let container = WapmContainer::from_bytes(webc).unwrap(); + let container = Container::from_bytes(webc).unwrap(); // Note: we don't have any way to intercept stdin or stdout, so blindly // assume that everything is fine if it runs successfully. @@ -43,6 +44,7 @@ mod wasi { .run_cmd(&container, "wat2wasm") }); let err = handle.join().unwrap().unwrap_err(); + dbg!(&err); let runtime_error = err .chain() @@ -60,7 +62,7 @@ mod wasi { let webc = download_cached("https://wapm.io/python/python").await; let store = Store::default(); let tasks = TokioTaskManager::new(Handle::current()); - let container = WapmContainer::from_bytes(webc).unwrap(); + let container = Container::from_bytes(webc).unwrap(); let handle = std::thread::spawn(move || { WasiRunner::new(store) @@ -96,7 +98,7 @@ mod wcgi { #[tokio::test] async fn can_run_staticserver() { let webc = download_cached("https://wapm.io/Michael-F-Bryan/staticserver").await; - let container = WapmContainer::from_bytes(webc).unwrap(); + let container = Container::from_bytes(webc).unwrap(); let runner = WcgiRunner::new("staticserver"); let entrypoint = container.manifest().entrypoint.as_ref().unwrap(); @@ -109,7 +111,7 @@ mod wcgi { async fn staticserver() { let webc = download_cached("https://wapm.io/Michael-F-Bryan/staticserver").await; let tasks = TokioTaskManager::new(Handle::current()); - let container = WapmContainer::from_bytes(webc).unwrap(); + let container = Container::from_bytes(webc).unwrap(); let mut runner = WcgiRunner::new("staticserver"); let port = rand::thread_rng().gen_range(10000_u16..65535_u16); let (cb, started) = callbacks(Handle::current());