diff --git a/Cargo.lock b/Cargo.lock index 12574b12ef9..0161fe47cc0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5706,6 +5706,7 @@ dependencies = [ "serde_yaml 0.8.26", "sha2", "shellexpand", + "tempfile", "term_size", "termios", "thiserror", diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index 425e885ca88..7bf1e91c151 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -82,6 +82,7 @@ wasm-bindgen = "0.2.74" [dev-dependencies] wasmer = { path = "../api", version = "=3.2.0-alpha.1", default-features = false, features = ["wat", "js-serializable-module"] } tokio = { version = "1", features = [ "sync", "macros", "rt" ], default_features = false } +tempfile = "3.4.0" [target.'cfg(target_arch = "wasm32")'.dev-dependencies] wasm-bindgen-test = "0.3.0" diff --git a/lib/wasi/src/runners/common.rs b/lib/wasi/src/runners/common.rs new file mode 100644 index 00000000000..2a07a3896e5 --- /dev/null +++ b/lib/wasi/src/runners/common.rs @@ -0,0 +1,173 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; + +use anyhow::Error; +use wasmer_vfs::{FileSystem, PassthruFileSystem, RootFileSystemBuilder, TmpFileSystem}; +use webc::metadata::annotations::Wasi as WasiAnnotations; + +use crate::WasiEnvBuilder; + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub(crate) struct MappedDirectory { + pub host: PathBuf, + pub guest: String, +} + +impl MappedDirectory { + pub fn new(host: impl Into, guest: impl Into) -> Self { + MappedDirectory { + host: host.into(), + guest: guest.into(), + } + } +} + +#[derive(Debug, Default, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub(crate) struct CommonWasiOptions { + pub args: Vec, + pub env: HashMap, + pub forward_host_env: bool, + pub mapped_dirs: Vec, +} + +impl CommonWasiOptions { + pub(crate) fn update( + &self, + builder: &mut WasiEnvBuilder, + annotations: &WasiAnnotations, + webc_fs: Arc, + ) -> Result<(), Error> { + self.populate_args(annotations, builder); + self.populate_env(annotations, builder); + + let fs = self.fs(webc_fs)?; + builder.set_fs(Box::new(fs)); + builder.add_preopen_dir("/")?; + + Ok(()) + } + + /// Set up a filesystem which merges the standard directories (from + /// [`RootFileSystemBuilder`]), the WEBC file's volumes, and any host + /// directories that were mounted. + fn fs(&self, webc_fs: Arc) -> Result { + let root_fs = RootFileSystemBuilder::new().build(); + + root_fs.union(&webc_fs); + + if !self.mapped_dirs.is_empty() { + let fs_backing: Arc = + Arc::new(PassthruFileSystem::new(crate::default_fs_backing())); + + for MappedDirectory { host, guest } in self.mapped_dirs.iter() { + let guest = match guest.starts_with('/') { + true => PathBuf::from(guest), + false => Path::new("/").join(guest), + }; + tracing::trace!( + host=%host.display(), + guest=%guest.display(), + "mounting directory to instance fs", + ); + + root_fs + .mount(host.clone(), &fs_backing, guest.clone()) + .map_err(|error| { + anyhow::anyhow!( + "Unable to mount \"{}\" to \"{}\": {error}", + host.display(), + guest.display() + ) + })?; + } + } + + Ok(root_fs) + } + + fn populate_args(&self, wasi: &WasiAnnotations, builder: &mut WasiEnvBuilder) { + let args = builder.get_args_mut(); + + if let Some(main_args) = &wasi.main_args { + args.extend(main_args.iter().cloned()); + } + + args.extend(self.args.iter().cloned()); + } + + fn populate_env(&self, wasi: &WasiAnnotations, builder: &mut WasiEnvBuilder) { + for item in wasi.env.as_deref().unwrap_or_default() { + // TODO(Michael-F-Bryan): Convert "wasi.env" in the webc crate from an + // Option> to a HashMap so we avoid this + // string.split() business + match item.split_once('=') { + Some((k, v)) => { + builder.add_env(k, v); + } + None => { + builder.add_env(item, ""); + } + } + } + + if self.forward_host_env { + for (key, value) in std::env::vars() { + builder.add_env(key, value); + } + } + + for (key, value) in &self.env { + builder.add_env(key, value); + } + } +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + use wasmer_vfs::{webc_fs::WebcFileSystem, AsyncReadExt}; + use webc::v1::{ParseOptions, WebCOwned}; + + use super::*; + + const PYTHON: &[u8] = include_bytes!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/../c-api/examples/assets/python-0.1.0.wasmer" + )); + + #[tokio::test] + async fn union_of_host_and_webc_filesystem() { + let webc = WebCOwned::parse(PYTHON.to_vec(), &ParseOptions::default()).unwrap(); + let webc = WebcFileSystem::init_all(Arc::new(webc)); + let temp = TempDir::new().unwrap(); + std::fs::write(temp.path().join("host.txt"), b"Host").unwrap(); + let options = CommonWasiOptions { + mapped_dirs: vec![MappedDirectory::new(temp.path(), "/path/to/")], + ..Default::default() + }; + + let fs = options.fs(Arc::new(webc)).unwrap(); + + // Make sure we can read from the host file system + let mut host = fs + .new_open_options() + .read(true) + .open("/path/to/host.txt") + .unwrap(); + let mut buffer = String::new(); + host.read_to_string(&mut buffer).await.unwrap(); + assert_eq!(buffer, "Mounted"); + // We should still have access to the WEBC file's volumes + let mut guest = fs + .new_open_options() + .read(true) + .open("/lib/guest.txt") + .unwrap(); + let mut buffer = String::new(); + guest.read_to_string(&mut buffer).await.unwrap(); + assert_eq!(buffer, "Mounted"); + } +} diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index c05be2663c1..1e5dbdc2b48 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -8,7 +8,7 @@ use webc::{ Version, }; -/// A parsed WAPM package. +/// A cheaply copyable, parsed WAPM package. #[derive(Debug, Clone)] pub struct WapmContainer { repr: Repr, @@ -99,29 +99,17 @@ impl WapmContainer { } } - /// Load a volume as a [`FileSystem`] node. - pub(crate) fn volume_fs(&self, package_name: &str) -> Box { - match &self.repr { - Repr::V1Mmap(mapped) => { - Box::new(WebcFileSystem::init(Arc::clone(mapped), package_name)) - } - Repr::V1Owned(owned) => Box::new(WebcFileSystem::init(Arc::clone(owned), package_name)), - } - } - /// Get the entire container as a single filesystem and a list of suggested /// directories to preopen. - pub(crate) fn container_fs(&self) -> (Box, Vec) { + pub(crate) fn container_fs(&self) -> Arc { match &self.repr { Repr::V1Mmap(mapped) => { let fs = WebcFileSystem::init_all(Arc::clone(mapped)); - let top_level_dirs = fs.top_level_dirs().clone(); - (Box::new(fs), top_level_dirs) + Arc::new(fs) } Repr::V1Owned(owned) => { let fs = WebcFileSystem::init_all(Arc::clone(owned)); - let top_level_dirs = fs.top_level_dirs().clone(); - (Box::new(fs), top_level_dirs) + Arc::new(fs) } } } diff --git a/lib/wasi/src/runners/mod.rs b/lib/wasi/src/runners/mod.rs index d156753fe93..aba5df1c16f 100644 --- a/lib/wasi/src/runners/mod.rs +++ b/lib/wasi/src/runners/mod.rs @@ -7,6 +7,7 @@ pub mod emscripten; pub mod wasi; #[cfg(feature = "webc_runner_rt_wcgi")] pub mod wcgi; +mod common; pub use self::{ container::{Bindings, WapmContainer, WebcParseError, WitBindings}, diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index eac529242b1..e62c5010936 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -1,9 +1,11 @@ //! WebC container support for running WASI modules -use std::{collections::HashMap, sync::Arc}; +use std::sync::Arc; -use crate::{runners::WapmContainer, PluggableRuntimeImplementation, VirtualTaskManager}; -use crate::{WasiEnv, WasiEnvBuilder}; +use crate::{ + runners::{common::CommonWasiOptions, WapmContainer}, + PluggableRuntimeImplementation, VirtualTaskManager, WasiEnvBuilder, +}; use anyhow::{Context, Error}; use serde::{Deserialize, Serialize}; use wasmer::{Module, Store}; @@ -11,9 +13,7 @@ use webc::metadata::{annotations::Wasi, Command}; #[derive(Debug, Serialize, Deserialize)] pub struct WasiRunner { - args: Vec, - env: HashMap, - forward_host_env: bool, + options: CommonWasiOptions, #[serde(skip, default)] store: Store, #[serde(skip, default)] @@ -24,17 +24,15 @@ impl WasiRunner { /// Constructs a new `WasiRunner` given an `Store` pub fn new(store: Store) -> Self { Self { - args: Vec::new(), - env: HashMap::new(), store, - forward_host_env: false, tasks: None, + options: CommonWasiOptions::default(), } } /// Returns the current arguments for this `WasiRunner` pub fn get_args(&self) -> Vec { - self.args.clone() + self.options.args.clone() } /// Builder method to provide CLI args to the runner @@ -53,7 +51,7 @@ impl WasiRunner { A: IntoIterator, S: Into, { - self.args = args.into_iter().map(|s| s.into()).collect(); + self.options.args = args.into_iter().map(|s| s.into()).collect(); } /// Builder method to provide environment variables to the runner. @@ -64,7 +62,7 @@ impl WasiRunner { /// Provide environment variables to the runner. pub fn set_env(&mut self, key: impl Into, value: impl Into) { - self.env.insert(key.into(), value.into()); + self.options.env.insert(key.into(), value.into()); } pub fn with_envs(mut self, envs: I) -> Self @@ -84,7 +82,7 @@ impl WasiRunner { V: Into, { for (key, value) in envs { - self.env.insert(key.into(), value.into()); + self.options.env.insert(key.into(), value.into()); } } @@ -94,7 +92,7 @@ impl WasiRunner { } pub fn set_forward_host_env(&mut self) { - self.forward_host_env = true; + self.options.forward_host_env = true; } pub fn with_task_manager(mut self, tasks: impl VirtualTaskManager) -> Self { @@ -122,28 +120,20 @@ impl crate::runners::Runner for WasiRunner { command: &Command, container: &WapmContainer, ) -> Result { - let atom_name = match command.get_annotation("wasi")? { - Some(Wasi { atom, .. }) => atom, - None => command_name.to_string(), - }; + let annotations: Wasi = command + .get_annotation("wasi")? + .unwrap_or_else(|| Wasi::new(command_name)); + let atom_name = &annotations.atom; let atom = container - .get_atom(&atom_name) + .get_atom(atom_name) .with_context(|| format!("Unable to get the \"{atom_name}\" atom"))?; let mut module = Module::new(&self.store, atom)?; - module.set_name(&atom_name); + module.set_name(atom_name); - let mut builder = prepare_webc_env(container, &atom_name, &self.args)?; - - if self.forward_host_env { - for (k, v) in std::env::vars() { - builder.add_env(k, v); - } - } - - for (k, v) in &self.env { - builder.add_env(k, v); - } + let mut builder = WasiEnvBuilder::new(command_name); + let webc_fs = container.container_fs(); + self.options.update(&mut builder, &annotations, webc_fs)?; if let Some(tasks) = &self.tasks { let rt = PluggableRuntimeImplementation::new(Arc::clone(tasks)); @@ -155,21 +145,3 @@ impl crate::runners::Runner for WasiRunner { Ok(()) } } - -// https://github.com/tokera-com/ate/blob/42c4ce5a0c0aef47aeb4420cc6dc788ef6ee8804/term-lib/src/eval/exec.rs#L444 -fn prepare_webc_env( - container: &WapmContainer, - command: &str, - args: &[String], -) -> Result { - let (filesystem, preopen_dirs) = container.container_fs(); - let mut builder = WasiEnv::builder(command).args(args); - - for entry in preopen_dirs { - builder.add_preopen_build(|p| p.directory(&entry).read(true).write(true).create(true))?; - } - - builder.set_fs(filesystem); - - Ok(builder) -} diff --git a/lib/wasi/src/runners/wcgi/handler.rs b/lib/wasi/src/runners/wcgi/handler.rs index 5ff6f054865..de2af912482 100644 --- a/lib/wasi/src/runners/wcgi/handler.rs +++ b/lib/wasi/src/runners/wcgi/handler.rs @@ -1,7 +1,6 @@ use std::{ collections::HashMap, ops::Deref, - path::{Path, PathBuf}, pin::Pin, sync::Arc, task::Poll, @@ -16,13 +15,13 @@ use tokio::{ runtime::Handle, }; use wasmer::Module; -use wasmer_vfs::{FileSystem, PassthruFileSystem, RootFileSystemBuilder, TmpFileSystem}; use wcgi_host::CgiDialect; use crate::{ http::HttpClientCapabilityV1, - runners::wcgi::{Callbacks, MappedDirectory}, + runners::{wcgi::Callbacks}, Capabilities, Pipe, PluggableRuntimeImplementation, VirtualTaskManager, WasiEnv, + WasiEnvBuilder, }; /// The shared object that manages the instantiaion of WASI executables and @@ -52,10 +51,7 @@ impl Handler { .prepare_environment_variables(parts, &mut request_specific_env); let rt = PluggableRuntimeImplementation::new(Arc::clone(&self.task_manager)); - let builder = builder - .envs(self.env.iter()) - .envs(request_specific_env) - .args(self.args.iter()) + let mut builder = builder .stdin(Box::new(req_body_receiver)) .stdout(Box::new(res_body_sender)) .stderr(Box::new(stderr_sender)) @@ -64,9 +60,9 @@ impl Handler { http_client: HttpClientCapabilityV1::new_allow_all(), threading: Default::default(), }) - .runtime(Arc::new(rt)) - .sandbox_fs(self.fs()?) - .preopen_dir(Path::new("/"))?; + .runtime(Arc::new(rt)); + + (self.update_builder)(&mut builder)?; let module = self.module.clone(); @@ -117,38 +113,6 @@ impl Handler { Ok(response) } - - fn fs(&self) -> Result { - let root_fs = RootFileSystemBuilder::new().build(); - - if !self.mapped_dirs.is_empty() { - let fs_backing: Arc = - Arc::new(PassthruFileSystem::new(crate::default_fs_backing())); - - for MappedDirectory { host, guest } in self.mapped_dirs.iter() { - let guest = match guest.starts_with('/') { - true => PathBuf::from(guest), - false => Path::new("/").join(guest), - }; - tracing::trace!( - host=%host.display(), - guest=%guest.display(), - "mounting directory to instance fs", - ); - - root_fs - .mount(host.clone(), &fs_backing, guest.clone()) - .map_err(|error| { - anyhow::anyhow!( - "Unable to mount \"{}\" to \"{}\": {error}", - host.display(), - guest.display() - ) - })?; - } - } - Ok(root_fs) - } } impl Deref for Handler { @@ -222,17 +186,18 @@ async fn consume_stderr( } } -#[derive(Clone, derivative::Derivative)] +type Updater = dyn Fn(&mut WasiEnvBuilder) -> Result<(), Error> + Send + Sync; + +#[derive(derivative::Derivative)] #[derivative(Debug)] pub(crate) struct SharedState { pub(crate) program: String, - pub(crate) env: HashMap, - pub(crate) args: Vec, - pub(crate) mapped_dirs: Vec, pub(crate) module: Module, pub(crate) dialect: CgiDialect, pub(crate) task_manager: Arc, #[derivative(Debug = "ignore")] + pub(crate) update_builder: Box, + #[derivative(Debug = "ignore")] pub(crate) callbacks: Arc, } diff --git a/lib/wasi/src/runners/wcgi/mod.rs b/lib/wasi/src/runners/wcgi/mod.rs index 1fe6f485c12..ee0ef55c1aa 100644 --- a/lib/wasi/src/runners/wcgi/mod.rs +++ b/lib/wasi/src/runners/wcgi/mod.rs @@ -1,12 +1,6 @@ mod handler; mod runner; -use std::path::PathBuf; -pub use self::runner::{Callbacks, Config, WcgiRunner}; -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct MappedDirectory { - pub host: PathBuf, - pub guest: String, -} +pub use self::runner::{Callbacks, Config, WcgiRunner}; diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index 4d4b3b37012..5ed390b6d8e 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, convert::Infallible, net::SocketAddr, path::PathBuf, sync::Arc}; +use std::{convert::Infallible, net::SocketAddr, path::PathBuf, sync::Arc}; use anyhow::{Context, Error}; use futures::future::AbortHandle; @@ -13,10 +13,8 @@ use webc::metadata::{ use crate::{ runners::{ - wcgi::{ - handler::{Handler, SharedState}, - MappedDirectory, - }, + common::{CommonWasiOptions, MappedDirectory}, + wcgi::handler::{Handler, SharedState}, WapmContainer, }, runtime::task_manager::tokio::TokioTaskManager, @@ -49,7 +47,7 @@ impl WcgiRunner { .load_module(&wasi, ctx) .context("Couldn't load the module")?; - let handler = self.create_handler(module, &wasi, ctx)?; + let handler = self.create_handler(module, wasi, ctx)?; let task_manager = Arc::clone(&handler.task_manager); let make_service = hyper::service::make_service_fn(move |s: &AddrStream| { @@ -119,12 +117,9 @@ impl WcgiRunner { fn create_handler( &self, module: Module, - wasi: &Wasi, + wasi_annotations: Wasi, ctx: &RunnerContext<'_>, ) -> Result { - let env = construct_env(wasi, self.config.forward_host_env, &self.config.env); - let args = construct_args(wasi, &self.config.args); - let Wcgi { dialect, .. } = ctx.command().get_annotation("wcgi")?.unwrap_or_default(); let dialect = match dialect { @@ -132,11 +127,14 @@ impl WcgiRunner { None => CgiDialect::Wcgi, }; + let wasi_options = self.config.wasi.clone(); + let container_fs = ctx.container_fs(); + let shared = SharedState { program: self.program_name.clone(), - env, - args, - mapped_dirs: self.config.mapped_dirs.clone(), + update_builder: Box::new(move |env| { + wasi_options.update(env, &wasi_annotations, Arc::clone(&container_fs)) + }), task_manager: self .config .task_manager @@ -151,48 +149,6 @@ impl WcgiRunner { } } -fn construct_args(wasi: &Wasi, extras: &[String]) -> Vec { - let mut args = Vec::new(); - - if let Some(main_args) = &wasi.main_args { - args.extend(main_args.iter().cloned()); - } - - args.extend(extras.iter().cloned()); - - args -} - -fn construct_env( - wasi: &Wasi, - forward_host_env: bool, - overrides: &HashMap, -) -> HashMap { - let mut env: HashMap = HashMap::new(); - - for item in wasi.env.as_deref().unwrap_or_default() { - // TODO(Michael-F-Bryan): Convert "wasi.env" in the webc crate from an - // Option> to a HashMap so we avoid this - // string.split() business - match item.split_once('=') { - Some((k, v)) => { - env.insert(k.to_string(), v.to_string()); - } - None => { - env.insert(item.to_string(), String::new()); - } - } - } - - if forward_host_env { - env.extend(std::env::vars()); - } - - env.extend(overrides.clone()); - - env -} - // TODO(Michael-F-Bryan): Pass this to Runner::run() as "&dyn RunnerContext" // when we rewrite the "Runner" trait. struct RunnerContext<'a> { @@ -232,6 +188,10 @@ impl RunnerContext<'_> { // TODO(Michael-F-Bryan): wire this up to wasmer-cache Module::new(&self.engine, wasm).map_err(Error::from) } + + fn container_fs(&self) -> Arc { + self.container.container_fs() + } } impl crate::runners::Runner for WcgiRunner { @@ -265,13 +225,10 @@ impl crate::runners::Runner for WcgiRunner { pub struct Config { task_manager: Option>, addr: SocketAddr, - args: Vec, - env: HashMap, - forward_host_env: bool, - mapped_dirs: Vec, #[derivative(Debug = "ignore")] callbacks: Arc, store: Option>, + wasi: CommonWasiOptions, } impl Config { @@ -287,7 +244,7 @@ impl Config { /// Add an argument to the WASI executable's command-line arguments. pub fn arg(&mut self, arg: impl Into) -> &mut Self { - self.args.push(arg.into()); + self.wasi.args.push(arg.into()); self } @@ -297,13 +254,13 @@ impl Config { A: IntoIterator, S: Into, { - self.args.extend(args.into_iter().map(|s| s.into())); + self.wasi.args.extend(args.into_iter().map(|s| s.into())); self } /// Expose an environment variable to the guest. pub fn env(&mut self, name: impl Into, value: impl Into) -> &mut Self { - self.env.insert(name.into(), value.into()); + self.wasi.env.insert(name.into(), value.into()); self } @@ -314,14 +271,15 @@ impl Config { K: Into, V: Into, { - self.env + self.wasi + .env .extend(variables.into_iter().map(|(k, v)| (k.into(), v.into()))); self } /// Forward all of the host's environment variables to the guest. pub fn forward_host_env(&mut self) -> &mut Self { - self.forward_host_env = true; + self.wasi.forward_host_env = true; self } @@ -330,10 +288,9 @@ impl Config { host: impl Into, guest: impl Into, ) -> &mut Self { - self.mapped_dirs.push(MappedDirectory { - host: host.into(), - guest: guest.into(), - }); + self.wasi + .mapped_dirs + .push(MappedDirectory::new(host, guest)); self } @@ -343,11 +300,10 @@ impl Config { H: Into, G: Into, { - let mappings = mappings.into_iter().map(|(h, g)| MappedDirectory { - host: h.into(), - guest: g.into(), - }); - self.mapped_dirs.extend(mappings); + let mappings = mappings + .into_iter() + .map(|(h, g)| MappedDirectory::new(h, g)); + self.wasi.mapped_dirs.extend(mappings); self } @@ -369,10 +325,7 @@ impl Default for Config { Self { task_manager: None, addr: ([127, 0, 0, 1], 8000).into(), - env: HashMap::new(), - forward_host_env: false, - mapped_dirs: Vec::new(), - args: Vec::new(), + wasi: CommonWasiOptions::default(), callbacks: Arc::new(NoopCallbacks), store: None, }