From b2402e356072f3df572d39bade27f7cd63afadb5 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 22 Feb 2023 15:00:19 +0800 Subject: [PATCH 01/26] Move the wcgi-runner crate into the Wasmer repo --- Cargo.lock | 198 +++++++++++++- Cargo.toml | 18 +- lib/wcgi-runner/Cargo.toml | 47 ++++ lib/wcgi-runner/README.md | 9 + lib/wcgi-runner/src/annotations.rs | 24 ++ lib/wcgi-runner/src/bin/wcgi-runner.rs | 111 ++++++++ lib/wcgi-runner/src/builder.rs | 252 ++++++++++++++++++ lib/wcgi-runner/src/context.rs | 226 ++++++++++++++++ lib/wcgi-runner/src/errors.rs | 65 +++++ lib/wcgi-runner/src/lib.rs | 15 ++ lib/wcgi-runner/src/module_loader/cached.rs | 48 ++++ .../src/module_loader/file_loader.rs | 48 ++++ lib/wcgi-runner/src/module_loader/mod.rs | 67 +++++ lib/wcgi-runner/src/module_loader/wasm.rs | 40 +++ lib/wcgi-runner/src/module_loader/webc.rs | 155 +++++++++++ lib/wcgi-runner/src/runner.rs | 85 ++++++ lib/wcgi-runner/tests/integration.rs | 153 +++++++++++ 17 files changed, 1540 insertions(+), 21 deletions(-) create mode 100644 lib/wcgi-runner/Cargo.toml create mode 100644 lib/wcgi-runner/README.md create mode 100644 lib/wcgi-runner/src/annotations.rs create mode 100644 lib/wcgi-runner/src/bin/wcgi-runner.rs create mode 100644 lib/wcgi-runner/src/builder.rs create mode 100644 lib/wcgi-runner/src/context.rs create mode 100644 lib/wcgi-runner/src/errors.rs create mode 100644 lib/wcgi-runner/src/lib.rs create mode 100644 lib/wcgi-runner/src/module_loader/cached.rs create mode 100644 lib/wcgi-runner/src/module_loader/file_loader.rs create mode 100644 lib/wcgi-runner/src/module_loader/mod.rs create mode 100644 lib/wcgi-runner/src/module_loader/wasm.rs create mode 100644 lib/wcgi-runner/src/module_loader/webc.rs create mode 100644 lib/wcgi-runner/src/runner.rs create mode 100644 lib/wcgi-runner/tests/integration.rs diff --git a/Cargo.lock b/Cargo.lock index 030957002db..f670118d1e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,6 +66,9 @@ name = "anyhow" version = "1.0.68" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +dependencies = [ + "backtrace", +] [[package]] name = "arbitrary" @@ -100,6 +103,12 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" +[[package]] +name = "ascii" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" + [[package]] name = "assert_cmd" version = "1.0.8" @@ -116,9 +125,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.63" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eff18d764974428cf3a9328e23fc5c986f5fbed46e6cd4cdf42544df5d297ec1" +checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" dependencies = [ "proc-macro2", "quote", @@ -284,9 +293,12 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfb24e866b15a1af2a1b663f10c6b6b8f397a84aadb828f12e5b289ec23a3a3c" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" +dependencies = [ + "serde", +] [[package]] name = "bytesize" @@ -386,6 +398,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "chunked_transfer" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a" + [[package]] name = "cipher" version = "0.4.3" @@ -415,8 +433,8 @@ checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", "bitflags", - "clap_derive", - "clap_lex", + "clap_derive 3.2.18", + "clap_lex 0.2.4", "indexmap", "once_cell", "strsim", @@ -424,6 +442,21 @@ dependencies = [ "textwrap 0.16.0", ] +[[package]] +name = "clap" +version = "4.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0b0588d44d4d63a87dbd75c136c166bbfd9a86a31cb89e09906521c7d3f5e3" +dependencies = [ + "bitflags", + "clap_derive 4.1.0", + "clap_lex 0.3.1", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + [[package]] name = "clap_derive" version = "3.2.18" @@ -437,6 +470,19 @@ dependencies = [ "syn", ] +[[package]] +name = "clap_derive" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" +dependencies = [ + "heck 0.4.0", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "clap_lex" version = "0.2.4" @@ -446,6 +492,15 @@ dependencies = [ "os_str_bytes", ] +[[package]] +name = "clap_lex" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "cmake" version = "0.1.49" @@ -493,7 +548,7 @@ version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" dependencies = [ - "ascii", + "ascii 0.9.3", "byteorder", "either", "memchr", @@ -1656,6 +1711,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-serde" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e272971f774ba29341db2f686255ff8a979365a26fb9e4277f6b6d9ec0cdd5e" +dependencies = [ + "http", + "serde", +] + [[package]] name = "http_req" version = "0.8.1" @@ -3950,6 +4015,18 @@ dependencies = [ "syn", ] +[[package]] +name = "tiny_http" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" +dependencies = [ + "ascii 1.1.0", + "chunked_transfer", + "httpdate", + "log", +] + [[package]] name = "tinytemplate" version = "1.2.1" @@ -4314,6 +4391,22 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" +[[package]] +name = "ureq" +version = "2.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d" +dependencies = [ + "base64 0.13.1", + "flate2", + "log", + "once_cell", + "rustls 0.20.8", + "url", + "webpki 0.22.0", + "webpki-roots 0.22.6", +] + [[package]] name = "url" version = "2.3.1" @@ -4796,7 +4889,7 @@ dependencies = [ "wasmer-types", "wasmer-vfs", "wasmer-wasi", - "webc", + "webc 4.1.1", ] [[package]] @@ -4901,7 +4994,7 @@ dependencies = [ "wasmer-wasm-interface", "wasmer-wast", "wasmparser 0.51.4", - "webc", + "webc 4.1.1", ] [[package]] @@ -5137,7 +5230,7 @@ dependencies = [ "toml", "url", "wasmer-toml", - "webc", + "webc 4.1.1", "whoami", ] @@ -5207,7 +5300,7 @@ dependencies = [ "tokio", "tracing", "typetag", - "webc", + "webc 4.1.1", ] [[package]] @@ -5298,7 +5391,7 @@ dependencies = [ "wasmer-vnet", "wasmer-wasi-local-networking", "wasmer-wasi-types", - "webc", + "webc 4.1.1", "weezl", "winapi", ] @@ -5554,6 +5647,62 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "wcgi" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "569971bb4ff2a37d0c417821f9cc5e3291aa7fa28e033a405c9da16e86b6fe52" +dependencies = [ + "http", + "http-serde", + "serde", + "serde_json", + "url", +] + +[[package]] +name = "wcgi-host" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d81f49740e252e51e074deb84b931621b6c94f3a9d23764654d230388663c2" +dependencies = [ + "http", + "serde", + "tokio", + "wasmparser 0.98.1", + "wcgi", +] + +[[package]] +name = "wcgi-runner" +version = "3.2.0-alpha.1" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "clap 4.1.6", + "futures", + "http", + "hyper", + "md5", + "reqwest", + "serde", + "serde_cbor", + "thiserror", + "tiny_http", + "tokio", + "tower-service", + "tracing", + "tracing-subscriber 0.3.16", + "ureq", + "wasmer", + "wasmer-vfs", + "wasmer-wasi", + "wcgi", + "wcgi-host", + "webc 5.0.0-rc.1", +] + [[package]] name = "web-sys" version = "0.3.60" @@ -5588,6 +5737,31 @@ dependencies = [ "walkdir", ] +[[package]] +name = "webc" +version = "5.0.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4f6a21de5a426f0ee97316b4f363ca28524489fd065ea437011a5f70f70e34f" +dependencies = [ + "anyhow", + "base64 0.21.0", + "byteorder", + "bytes", + "indexmap", + "leb128", + "lexical-sort", + "once_cell", + "path-clean", + "rand 0.8.5", + "serde", + "serde_cbor", + "serde_json", + "sha2", + "thiserror", + "url", + "walkdir", +] + [[package]] name = "webpki" version = "0.21.4" diff --git a/Cargo.toml b/Cargo.toml index 76420d8dbb6..43e3ec71f6f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,7 +11,9 @@ autoexamples = false [dependencies] wasmer = { version = "=3.2.0-alpha.1", path = "lib/api", default-features = false } -wasmer-compiler = { version = "=3.2.0-alpha.1", path = "lib/compiler", features = ["compiler"] } +wasmer-compiler = { version = "=3.2.0-alpha.1", path = "lib/compiler", features = [ + "compiler", +] } wasmer-compiler-cranelift = { version = "=3.2.0-alpha.1", path = "lib/compiler-cranelift", optional = true } wasmer-compiler-singlepass = { version = "=3.2.0-alpha.1", path = "lib/compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=3.2.0-alpha.1", path = "lib/compiler-llvm", optional = true } @@ -50,6 +52,7 @@ members = [ "lib/wasi-local-networking", "lib/wasix/wasix-http-client", "lib/wasm-interface", + "lib/wcgi-runner", "lib/c-api/tests/wasmer-c-api-test-runner", "lib/c-api/examples/wasmer-capi-examples-runner", "lib/types", @@ -71,7 +74,9 @@ glob = "0.3" rustc_version = "0.4" [dev-dependencies] -wasmer = { version = "=3.2.0-alpha.1", path = "lib/api", default-features = false, features = ["cranelift"] } +wasmer = { version = "=3.2.0-alpha.1", path = "lib/api", default-features = false, features = [ + "cranelift", +] } anyhow = "1.0" criterion = "0.3" lazy_static = "1.4" @@ -105,10 +110,7 @@ wast = ["wasmer-wast"] wasi = ["wasmer-wasi"] emscripten = ["wasmer-emscripten"] wat = ["wasmer/wat"] -compiler = [ - "wasmer/compiler", - "wasmer-compiler/translator", -] +compiler = ["wasmer/compiler", "wasmer-compiler/translator"] singlepass = ["wasmer-compiler-singlepass", "compiler"] cranelift = ["wasmer-compiler-cranelift", "compiler"] llvm = ["wasmer-compiler-llvm", "compiler"] @@ -123,9 +125,7 @@ test-singlepass = ["singlepass"] test-cranelift = ["cranelift"] test-llvm = ["llvm"] -test-universal = [ - "test-generator/test-universal", -] +test-universal = ["test-generator/test-universal"] # Specifies that we're running in coverage testing mode. This disables tests # that raise signals because that interferes with tarpaulin. diff --git a/lib/wcgi-runner/Cargo.toml b/lib/wcgi-runner/Cargo.toml new file mode 100644 index 00000000000..2fa3ed00bfb --- /dev/null +++ b/lib/wcgi-runner/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "wcgi-runner" +version = "3.2.0-alpha.1" +description = "A runner that can serve WCGI programs locally" +authors = ["Wasmer Engineering Team "] +repository = "https://github.com/wasmerio/wasmer" +license = "MIT" +readme = "README.md" +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +anyhow = { version = "1", features = ["backtrace"] } +async-trait = "0.1.64" +bytes = "1.4.0" +clap = { version = "4", features = ["derive", "env"] } +futures = "0.3.25" +http = "0.2.8" +hyper = { version = "0.14.23", features = ["server", "stream"] } +md5 = "0.7.0" +serde = { version = "1.0.152", features = ["derive"] } +serde_cbor = "0.11.2" +thiserror = "1.0.38" +tiny_http = "0.12.0" +tokio = { version = "1", features = ["rt-multi-thread", "macros"] } +tower-service = "0.3" +tracing = "0.1.37" +tracing-subscriber = { version = "0.3.16", features = ["fmt", "env-filter"] } +wasmer = { version = "3.2.0-alpha.1", path = "../api", default-features = false, features = [ + "sys", + "cranelift", + "singlepass", +] } +wasmer-wasi = { version = "3.2.0-alpha.1", path = "../wasi", features = [ + "sys-default", +], default-features = false } +wasmer-vfs = { version = "3.2.0-alpha.1", path = "../vfs", default-features = false } +wcgi = { version = "0.1.1" } +wcgi-host = { version = "0.1.0" } +webc = { version = "5.0.0-rc.1", default-features = false } + +[dev-dependencies] +reqwest = { version = "0.11.0", default-features = false, features = [ + "rustls-tls", +] } +ureq = "2.6.2" diff --git a/lib/wcgi-runner/README.md b/lib/wcgi-runner/README.md new file mode 100644 index 00000000000..18dc8a51abf --- /dev/null +++ b/lib/wcgi-runner/README.md @@ -0,0 +1,9 @@ +# wcgi-runner + +A server that can serve [wcgi][wcgi] web servers. + +## Usage + +`wcgi-runner --watch ./path/to/module.wasm` + +[wcgi]: https://github.com/wasmerio/wcgi diff --git a/lib/wcgi-runner/src/annotations.rs b/lib/wcgi-runner/src/annotations.rs new file mode 100644 index 00000000000..d32cb19d250 --- /dev/null +++ b/lib/wcgi-runner/src/annotations.rs @@ -0,0 +1,24 @@ +use wcgi_host::CgiDialect; + +// FIXME(@Michael-F-Bryan): Make this public in the webc crate +#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct WasiCommandAnnotation { + #[serde(default)] + pub atom: Option, + #[serde(default)] + pub package: Option, + #[serde(default)] + pub env: Option>, + #[serde(default)] + pub main_args: Option>, + #[serde(default, rename = "mountAtomInVolume")] + pub mount_atom_in_volume: Option, +} + +// FIXME(@Michael-F-Bryan): Add this to the webc crate and update +// wapm-targz-to-pirita +#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub struct WcgiAnnotation { + #[serde(default)] + pub dialect: Option, +} diff --git a/lib/wcgi-runner/src/bin/wcgi-runner.rs b/lib/wcgi-runner/src/bin/wcgi-runner.rs new file mode 100644 index 00000000000..f30b12b887d --- /dev/null +++ b/lib/wcgi-runner/src/bin/wcgi-runner.rs @@ -0,0 +1,111 @@ +use std::{convert::Infallible, net::SocketAddr, path::PathBuf}; + +use anyhow::{Context, Error}; +use clap::Parser; +use tracing_subscriber::fmt::format::FmtSpan; +use wcgi_runner::Runner; + +fn main() -> Result<(), Error> { + if std::env::var("RUST_LOG").is_err() { + std::env::set_var("RUST_LOG", "wcgi_runner=trace,info"); + } + tracing_subscriber::fmt() + .with_span_events(FmtSpan::CLOSE) + .init(); + + let Args { + address, + input, + env_all, + mapped_dirs, + } = Args::parse(); + + // Hack to initialize the global shared tokio task manager handle. + // Prevents "cannot drop runtime in async context" errors, because + // the default will be initialized to the current tokio context. + let rt = wasmer_wasi::runtime::task_manager::tokio::TokioTaskManager::default(); + + let mut builder = Runner::builder() + .map_dirs(mapped_dirs) + .tokio_handle(rt.runtime_handle()); + + if env_all { + builder = builder.forward_host_env(); + } + + let runner = builder + .watch(input) + .context("Unable to create the runner")?; + + let make_service = hyper::service::make_service_fn(move |_| { + let runner = runner.clone(); + async move { Ok::<_, Infallible>(runner) } + }); + + tracing::info!(%address, "Started the server"); + rt.runtime_handle() + .block_on(async { hyper::Server::bind(&address).serve(make_service).await }) + .context("Unable to start the server")?; + + Ok(()) +} + +#[derive(Debug, Clone, Parser)] +#[clap(about, version, author)] +struct Args { + /// Server address. + #[clap(long, short, env, default_value_t = ([127, 0, 0, 1], 8000).into())] + address: SocketAddr, + + /// Map a host directory to a different location for the Wasm module + /// + /// Example: + /// + /// --map-dir /www:./my-website + /// => will make the ./my-website directory available for wazsm at /www + #[clap( + long = "mapdir", + name = "GUEST_DIR:HOST_DIR", + value_parser = parse_mapdir, + )] + mapped_dirs: Vec<(String, PathBuf)>, + + /// Forward all host env variables to the wcgi task. + #[clap(long)] + env_all: bool, + + /// A WCGI program. + input: PathBuf, +} + +/// Parses a mapdir from a string +// NOTE: copied from wasmerio/wasmer lib/cli/src/utils.rs. +pub fn parse_mapdir(entry: &str) -> Result<(String, PathBuf), anyhow::Error> { + fn retrieve_alias_pathbuf( + alias: &str, + real_dir: &str, + ) -> Result<(String, PathBuf), anyhow::Error> { + let pb = PathBuf::from(&real_dir); + if let Ok(pb_metadata) = pb.metadata() { + if !pb_metadata.is_dir() { + anyhow::bail!("\"{real_dir}\" exists, but it is not a directory"); + } + } else { + anyhow::bail!("Directory \"{real_dir}\" does not exist"); + } + Ok((alias.to_string(), pb)) + } + + // We try first splitting by `::` + if let Some((alias, real_dir)) = entry.split_once("::") { + retrieve_alias_pathbuf(alias, real_dir) + } + // And then we try splitting by `:` (for compatibility with previous API) + else if let Some((alias, real_dir)) = entry.split_once(':') { + retrieve_alias_pathbuf(alias, real_dir) + } else { + anyhow::bail!( + "Directory mappings must consist of two paths separate by a `::` or `:`. Found {entry}", + ) + } +} diff --git a/lib/wcgi-runner/src/builder.rs b/lib/wcgi-runner/src/builder.rs new file mode 100644 index 00000000000..974ce8b36ab --- /dev/null +++ b/lib/wcgi-runner/src/builder.rs @@ -0,0 +1,252 @@ +use std::{ + collections::HashMap, + path::PathBuf, + sync::{Arc, Mutex}, +}; + +use bytes::Bytes; +use tokio::runtime::Handle; +use wasmer::Engine; +use wcgi_host::CgiDialect; + +use crate::{ + context::Context, + module_loader::{FileLoader, ModuleLoader, WasmLoader, WebcCommand, WebcLoader, WebcOptions}, + Error, Runner, +}; + +/// A builder for initializing a [`Runner`]. +/// +/// # Examples +/// +/// The easiest way to use the builder is by giving it a WEBC file where the +/// default entrypoint is a WCGI command. +/// +/// ```rust,no_run +/// use wcgi_runner::Runner; +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// let webc = std::fs::read("path/to/server.webc")?; +/// let runner = Runner::builder().build_webc(webc)?; +/// # Ok(()) +/// # } +#[derive(Default)] +pub struct Builder { + args: Vec, + program: Option, + dialect: Option, + engine: Option, + env: HashMap, + forward_host_env: bool, + mapped_dirs: Vec<(String, PathBuf)>, + tokio_handle: Option, +} + +impl Builder { + pub fn new() -> Self { + Builder::default() + } + + /// Set the name of the program. + pub fn program(self, program: impl Into) -> Self { + Builder { + program: Some(program.into()), + ..self + } + } + + /// Add an argument to the WASI executable's command-line arguments. + pub fn arg(mut self, arg: impl Into) -> Self { + self.args.push(arg.into()); + self + } + + /// Add multiple arguments to the WASI executable's command-line arguments. + pub fn args(mut self, args: A) -> Self + where + A: IntoIterator, + S: Into, + { + self.args.extend(args.into_iter().map(|s| s.into())); + self + } + + /// Expose an environment variable to the guest. + pub fn env(mut self, name: impl Into, value: impl Into) -> Self { + self.env.insert(name.into(), value.into()); + self + } + + /// Expose multiple environment variables to the guest. + pub fn envs(mut self, variables: I) -> Self + where + I: IntoIterator, + K: Into, + V: Into, + { + self.env + .extend(variables.into_iter().map(|(k, v)| (k.into(), v.into()))); + self + } + + /// Forward all of the host's environment variables to the guest. + pub fn forward_host_env(self) -> Self { + Builder { + forward_host_env: true, + ..self + } + } + + /// Override the CGI dialect. + pub fn cgi_dialect(self, dialect: CgiDialect) -> Self { + Builder { + dialect: Some(dialect), + ..self + } + } + + /// Map `guest_dir` to a directory on the host. + pub fn map_dir(mut self, guest_dir: impl Into, host_dir: impl Into) -> Self { + self.mapped_dirs.push((guest_dir.into(), host_dir.into())); + self + } + + /// Map one or more `guest_dir` to a directory on the host. + pub fn map_dirs(mut self, mapping: I) -> Self + where + I: IntoIterator, + G: Into, + H: Into, + { + for (guest_dir, host_dir) in mapping { + self.mapped_dirs.push((guest_dir.into(), host_dir.into())); + } + + self + } + + /// Pass in a [`Handle`] to a custom tokio runtime. + pub fn tokio_handle(self, handle: Handle) -> Self { + Builder { + tokio_handle: Some(handle), + ..self + } + } + + /// Set the engine used to compile the WebAssembly module. + pub fn engine(self, engine: Engine) -> Self { + Builder { + engine: Some(engine), + ..self + } + } + + /// Create a [`Runner`] that executes a WEBC file. + /// + /// If [`Builder::program`] was set, this will look for the command with + /// that name in the WEBC file. Otherwise, it will fall back to the WEBC + /// file's default entrypoint. + /// + /// This will infer the [`CgiDialect`] from the WEBC file's metadata + pub fn build_webc(self, webc: impl Into) -> Result { + let webc = webc.into(); + + let options = WebcOptions { + command: match &self.program { + Some(program) => WebcCommand::Named(program), + None => WebcCommand::Entrypoint, + }, + dialect: self.dialect, + }; + let loader = WebcLoader::new_with_options(&options, webc)?; + + self.build(loader.load_once()) + } + + /// Create a [`Runner`] that executes a WebAssembly module. + /// + /// This requires the [`Builder::program`] to have been set. + /// + /// Unless otherwise specified (i.e. via [`Builder::cgi_dialect`]), the + /// WebAssembly binary will be assumed to implement the [`CgiDialect::Wcgi`] + /// dialect. + pub fn build_wasm(self, wasm: impl Into) -> Result { + let wasm = wasm.into(); + let program = self.program.clone().ok_or(Error::ProgramNameRequired)?; + + let loader = match self.dialect { + Some(dialect) => WasmLoader::new_with_dialect(program, wasm, dialect), + None => WasmLoader::new(program, wasm), + }; + + self.build(loader.load_once()) + } + + /// Create a new [`Runner`] from a particular file and automatically reload + /// whenever that file changes. + pub fn watch(self, path: impl Into) -> Result { + let path = path.into(); + let loader = FileLoader::new(&path).cached(file_has_changed(path)); + + self.build(loader) + } + + fn build(self, loader: impl ModuleLoader + 'static) -> Result { + let Builder { + args, + engine, + mut env, + forward_host_env, + mapped_dirs, + tokio_handle, + .. + } = self; + + if forward_host_env { + env = std::env::vars().chain(env).collect(); + } + + let tokio_handle = tokio_handle.unwrap_or_else(|| { + Handle::try_current().expect("The builder can only be used inside a Tokio context") + }); + + let ctx = Context { + tokio_handle, + engine: engine.unwrap_or_else(|| { + let store = wasmer::Store::default(); + store.engine().clone() + }), + env: Arc::new(env), + args: args.into(), + loader: Box::new(loader), + mapped_dirs, + }; + + Ok(Runner::new(Arc::new(ctx))) + } +} + +fn file_has_changed(path: PathBuf) -> impl Fn() -> bool + Send + Sync + 'static { + let last_modified: Mutex> = Mutex::new(None); + + move || -> bool { + let modified = match path.metadata().and_then(|m| m.modified()).ok() { + Some(m) => m, + None => { + // we couldn't determine the last modified time so be + // conservative mark the cache as invalidated. + return true; + } + }; + + let mut last_modified = last_modified.lock().expect("Poisoned"); + + let invalidated = match *last_modified { + Some(last) => last != modified, + None => true, + }; + + *last_modified = Some(modified); + invalidated + } +} diff --git a/lib/wcgi-runner/src/context.rs b/lib/wcgi-runner/src/context.rs new file mode 100644 index 00000000000..c79333a4f07 --- /dev/null +++ b/lib/wcgi-runner/src/context.rs @@ -0,0 +1,226 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + sync::Arc, +}; + +use futures::{Future, StreamExt, TryFutureExt}; +use http::{Request, Response}; +use hyper::Body; +use tokio::{ + io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt}, + runtime::Handle, +}; +use wasmer::Engine; +use wasmer_vfs::{FileSystem, PassthruFileSystem, RootFileSystemBuilder, TmpFileSystem}; +use wasmer_wasi::{http::HttpClientCapabilityV1, Capabilities, Pipe, WasiEnv}; + +use crate::{ + module_loader::{LoadedModule, ModuleLoader, ModuleLoaderContext}, + Error, +}; + +/// The shared object that manages the instantiaion of WASI executables and +/// communicating with them via the CGI protocol. +pub(crate) struct Context { + pub(crate) engine: Engine, + pub(crate) env: Arc>, + pub(crate) args: Arc<[String]>, + pub(crate) mapped_dirs: Vec<(String, PathBuf)>, + pub(crate) tokio_handle: Handle, + pub(crate) loader: Box, +} + +impl Context { + pub(crate) async fn handle(&self, req: Request) -> Result, Error> { + let LoadedModule { + program, + module, + dialect, + } = self + .loader + .load(ModuleLoaderContext::new(&self.engine, &self.tokio_handle)) + .await?; + + let (parts, body) = req.into_parts(); + + let (req_body_sender, req_body_receiver) = Pipe::channel(); + let (res_body_sender, res_body_receiver) = Pipe::channel(); + let (stderr_sender, stderr_receiver) = Pipe::channel(); + + let builder = WasiEnv::builder(program.to_string()); + + let mut request_specific_env = HashMap::new(); + dialect.prepare_environment_variables(parts, &mut request_specific_env); + + let builder = builder + .envs(self.env.iter()) + .envs(request_specific_env) + .args(self.args.iter()) + .stdin(Box::new(req_body_receiver)) + .stdout(Box::new(res_body_sender)) + .stderr(Box::new(stderr_sender)) + .capabilities(Capabilities { + insecure_allow_all: true, + http_client: HttpClientCapabilityV1::new_allow_all(), + }) + .sandbox_fs(self.fs()?) + .preopen_dir(Path::new("/"))?; + + let done = self + .tokio_handle + .spawn_blocking(move || builder.run(module)) + .map_err(Error::from) + .and_then(|r| async { r.map_err(Error::from) }); + + let handle = self.tokio_handle.clone(); + self.tokio_handle.spawn(async move { + if let Err(e) = + drive_request_to_completion(&handle, done, body, req_body_sender, stderr_receiver) + .await + { + tracing::error!( + error = &e as &dyn std::error::Error, + "Unable to drive the request to completion" + ); + } + }); + + let mut res_body_receiver = tokio::io::BufReader::new(res_body_receiver); + + let parts = dialect + .extract_response_header(&mut res_body_receiver) + .await?; + let chunks = futures::stream::try_unfold(res_body_receiver, |mut r| async move { + match r.fill_buf().await { + Ok(chunk) if chunk.is_empty() => Ok(None), + Ok(chunk) => { + let chunk = chunk.to_vec(); + r.consume(chunk.len()); + Ok(Some((chunk, r))) + } + Err(e) => Err(e), + } + }); + let body = hyper::Body::wrap_stream(chunks); + + let response = hyper::Response::from_parts(parts, body); + + Ok(response) + } + + fn fs(&self) -> Result { + let root_fs = RootFileSystemBuilder::new().build(); + + if !self.mapped_dirs.is_empty() { + let fs_backing: Arc = + Arc::new(PassthruFileSystem::new(wasmer_wasi::default_fs_backing())); + + for (src, dst) in &self.mapped_dirs { + let src = match src.starts_with('/') { + true => PathBuf::from(src), + false => Path::new("/").join(src), + }; + tracing::trace!( + source=%src.display(), + alias=%dst.display(), + "mounting directory to instance fs", + ); + + root_fs + .mount(PathBuf::from(&src), &fs_backing, dst.clone()) + .map_err(|error| Error::Mount { + error, + src, + dst: dst.to_path_buf(), + })?; + } + } + Ok(root_fs) + } +} + +/// Drive the request to completion by streaming the request body to the +/// instance and waiting for it to exit. +async fn drive_request_to_completion( + handle: &Handle, + done: impl Future>, + mut request_body: hyper::Body, + mut instance_stdin: impl AsyncWrite + Send + Unpin + 'static, + instance_stderr: impl AsyncRead + Send + Unpin + 'static, +) -> Result<(), Error> { + let request_body_send = handle + .spawn(async move { + // Copy the request into our instance, chunk-by-chunk. If the instance + // dies before we finish writing the body, the instance's side of the + // pipe will be automatically closed and we'll error out. + while let Some(res) = request_body.next().await { + // FIXME(theduke): figure out how to propagate a body error to the + // CGI instance. + let chunk = res?; + instance_stdin + .write_all(chunk.as_ref()) + .await + .map_err(Error::Io)?; + } + + instance_stdin.shutdown().await.map_err(Error::Io)?; + + Ok::<(), Error>(()) + }) + .map_err(Error::from) + .and_then(|r| async { r }); + + handle.spawn(async move { + consume_stderr(instance_stderr).await; + }); + + futures::try_join!(done, request_body_send)?; + + Ok(()) +} + +/// Read the instance's stderr, taking care to preserve output even when WASI +/// pipe errors occur so users still have *something* they use for +/// troubleshooting. +async fn consume_stderr(stderr: impl AsyncRead + Send + Unpin + 'static) { + let mut stderr = tokio::io::BufReader::new(stderr); + + // FIXME: this could lead to unbound memory usage + let mut buffer = Vec::new(); + + // Note: we don't want to just read_to_end() because a reading error + // would cause us to lose all of stderr. At least this way we'll be + // able to show users the partial result. + loop { + match stderr.fill_buf().await { + Ok(chunk) if chunk.is_empty() => { + // EOF - the instance's side of the pipe was closed. + break; + } + Ok(chunk) => { + buffer.extend(chunk); + let bytes_read = chunk.len(); + stderr.consume(bytes_read); + } + Err(e) => { + tracing::error!( + error = &e as &dyn std::error::Error, + bytes_read = buffer.len(), + "Unable to read the complete stderr", + ); + break; + } + } + } + + let stderr = String::from_utf8(buffer).unwrap_or_else(|e| { + tracing::warn!( + error = &e as &dyn std::error::Error, + "Stdout wasn't valid UTF-8", + ); + String::from_utf8_lossy(e.as_bytes()).into_owned() + }); + + tracing::info!(%stderr); +} diff --git a/lib/wcgi-runner/src/errors.rs b/lib/wcgi-runner/src/errors.rs new file mode 100644 index 00000000000..bd6655934ba --- /dev/null +++ b/lib/wcgi-runner/src/errors.rs @@ -0,0 +1,65 @@ +use std::path::PathBuf; + +use wasmer_wasi::{FsError, WasiRuntimeError, WasiStateCreationError}; +use webc::Version; + +/// Various errors that can be returned by the WCGI runner. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum Error { + #[error("The provided binary is not in a format known by the runner")] + UnknownFormat, + #[error(transparent)] + Hyper(#[from] hyper::Error), + #[error(transparent)] + Io(std::io::Error), + #[error("Unable to read \"{}\"", path.display())] + File { + #[source] + error: std::io::Error, + path: PathBuf, + }, + #[error(transparent)] + Webc(#[from] WebcLoadError), + #[error("Unable to compile the WebAssembly module")] + Compile(#[from] wasmer::CompileError), + #[error("A spawned task didn't run to completion")] + Join(#[from] tokio::task::JoinError), + #[error("Unable to automatically infer the program name")] + ProgramNameRequired, + #[error("An error occurred while implementing the CGI protocol")] + Cgi(#[from] wcgi_host::CgiError), + #[error("Unable to set up the WASI environment")] + StateCreation(#[from] WasiStateCreationError), + #[error("Could not mount directory mapping: '{src}:{dst}'")] + Mount { + #[source] + error: FsError, + src: PathBuf, + dst: PathBuf, + }, + #[error("Executing the WASI executable failed")] + Exec(#[from] WasiRuntimeError), +} + +/// Errors that may occur when loading a WCGI program from a WEBC file. +#[derive(Debug, thiserror::Error)] +#[non_exhaustive] +pub enum WebcLoadError { + #[error("The WEBC file doesn't contain a \"{name}\" command")] + UnknownCommand { name: String }, + #[error("Unable to find the \"{name}\" atom")] + MissingAtom { name: String }, + #[error("Unable to parse the manifest")] + Manifest(#[from] serde_cbor::Error), + #[error("Unable to detect the WEBC version")] + Detect(#[from] webc::DetectError), + #[error("Unsupported WEBC version, {_0}")] + UnsupportedVersion(Version), + #[error(transparent)] + V1(#[from] webc::v1::Error), + #[error(transparent)] + V2(#[from] webc::v2::read::OwnedReaderError), + #[error("Unable to infer the command to execute")] + UnknownEntrypoint, +} diff --git a/lib/wcgi-runner/src/lib.rs b/lib/wcgi-runner/src/lib.rs new file mode 100644 index 00000000000..c72e74d54a0 --- /dev/null +++ b/lib/wcgi-runner/src/lib.rs @@ -0,0 +1,15 @@ +// For now, let's ignore the fact that some of our Error variants are really big +#![allow(clippy::result_large_err)] + +pub mod annotations; +mod builder; +mod context; +mod errors; +mod module_loader; +mod runner; + +pub use crate::{ + builder::Builder, + errors::{Error, WebcLoadError}, + runner::Runner, +}; diff --git a/lib/wcgi-runner/src/module_loader/cached.rs b/lib/wcgi-runner/src/module_loader/cached.rs new file mode 100644 index 00000000000..f43b8992968 --- /dev/null +++ b/lib/wcgi-runner/src/module_loader/cached.rs @@ -0,0 +1,48 @@ +use tokio::sync::Mutex; + +use crate::{ + module_loader::{ModuleLoader, ModuleLoaderContext}, + Error, +}; + +use super::LoadedModule; + +pub(crate) struct Cached { + loader: L, + invalidated: Box bool + Send + Sync>, + cached: Mutex>, +} + +impl Cached { + pub(crate) fn new(loader: L, invalidated: impl Fn() -> bool + Send + Sync + 'static) -> Self { + Self { + loader, + invalidated: Box::new(invalidated), + cached: Mutex::new(None), + } + } +} + +#[async_trait::async_trait] +impl ModuleLoader for Cached { + async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result { + let mut cached = self.cached.lock().await; + + if (self.invalidated)() { + // Throw away the previous value to make sure we will always load + // the module again, even if invalidated() returns false in the + // future or calling the inner loader fails further down. + let _ = cached.take(); + } + + if let Some(module) = &*cached { + // Cache hit! + return Ok(module.clone()); + } + + let module = self.loader.load(ctx).await?; + *cached = Some(module.clone()); + + Ok(module) + } +} diff --git a/lib/wcgi-runner/src/module_loader/file_loader.rs b/lib/wcgi-runner/src/module_loader/file_loader.rs new file mode 100644 index 00000000000..cfdb528e9f9 --- /dev/null +++ b/lib/wcgi-runner/src/module_loader/file_loader.rs @@ -0,0 +1,48 @@ +use std::path::PathBuf; + +use bytes::Bytes; + +use crate::{ + module_loader::{LoadedModule, ModuleLoader, ModuleLoaderContext, WasmLoader, WebcLoader}, + Error, +}; + +/// A [`ModuleLoader`] that will read a file to disk and try to interpret it as +/// a known module type. +/// +/// This will re-read the file on every [`ModuleLoader::load()`] call. You may +/// want to add a [`ModuleLoader::cached()`] in front of it to only reload when +/// the file has been changed. +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct FileLoader { + path: PathBuf, +} + +impl FileLoader { + pub(crate) fn new(path: impl Into) -> Self { + FileLoader { path: path.into() } + } +} + +#[async_trait::async_trait] +impl ModuleLoader for FileLoader { + async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result { + let bytes: Bytes = tokio::fs::read(&self.path) + .await + .map_err(|error| Error::File { + error, + path: self.path.clone(), + })? + .into(); + + if webc::detect(bytes.as_ref()).is_ok() { + let loader = WebcLoader::new(bytes)?; + loader.load(ctx).await + } else if wasmer::is_wasm(&bytes) { + let loader = WasmLoader::new(self.path.display().to_string(), bytes); + loader.load(ctx).await + } else { + Err(Error::UnknownFormat) + } + } +} diff --git a/lib/wcgi-runner/src/module_loader/mod.rs b/lib/wcgi-runner/src/module_loader/mod.rs new file mode 100644 index 00000000000..2b5135c03df --- /dev/null +++ b/lib/wcgi-runner/src/module_loader/mod.rs @@ -0,0 +1,67 @@ +mod cached; +mod file_loader; +mod wasm; +mod webc; + +use bytes::Bytes; +use tokio::runtime::Handle; +use wasmer::{Engine, Module}; +use wcgi_host::CgiDialect; + +use crate::Error; + +pub(crate) use self::{ + cached::Cached, + file_loader::FileLoader, + wasm::WasmLoader, + webc::{WebcCommand, WebcLoader, WebcOptions}, +}; + +#[async_trait::async_trait] +pub(crate) trait ModuleLoader: Send + Sync { + async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result; + + /// Wrap this [`ModuleLoader`] with a cache that will only reload the + /// module when `invalidated` returns `true`. + fn cached(self, invalidated: impl Fn() -> bool + Send + Sync + 'static) -> Cached + where + Self: Sized, + { + Cached::new(self, invalidated) + } + + fn load_once(self) -> Cached + where + Self: Sized, + { + Cached::new(self, || false) + } +} + +#[derive(Debug, Clone)] +pub(crate) struct LoadedModule { + pub(crate) program: String, + pub(crate) module: Module, + pub(crate) dialect: CgiDialect, +} + +#[derive(Debug, Clone)] +pub(crate) struct ModuleLoaderContext<'a> { + engine: &'a Engine, + handle: &'a Handle, +} + +impl<'a> ModuleLoaderContext<'a> { + pub(crate) fn new(engine: &'a Engine, handle: &'a Handle) -> Self { + ModuleLoaderContext { engine, handle } + } + + pub(crate) async fn compile_wasm(&self, wasm: Bytes) -> Result { + let engine = self.engine.clone(); + let module = self + .handle + .spawn_blocking(move || Module::new(&engine, &wasm)) + .await??; + Ok(module) + } +} diff --git a/lib/wcgi-runner/src/module_loader/wasm.rs b/lib/wcgi-runner/src/module_loader/wasm.rs new file mode 100644 index 00000000000..595bc86b8de --- /dev/null +++ b/lib/wcgi-runner/src/module_loader/wasm.rs @@ -0,0 +1,40 @@ +use bytes::Bytes; + +use wcgi_host::CgiDialect; + +use crate::{ + module_loader::{LoadedModule, ModuleLoader, ModuleLoaderContext}, + Error, +}; + +#[derive(Debug, Clone, PartialEq)] +pub struct WasmLoader { + program: String, + wasm: Bytes, + dialect: CgiDialect, +} + +impl WasmLoader { + pub fn new(program: String, wasm: Bytes) -> Self { + WasmLoader::new_with_dialect(program, wasm, CgiDialect::Wcgi) + } + + pub fn new_with_dialect(program: String, wasm: Bytes, dialect: CgiDialect) -> Self { + WasmLoader { + program, + wasm, + dialect, + } + } +} + +#[async_trait::async_trait] +impl ModuleLoader for WasmLoader { + async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result { + Ok(LoadedModule { + module: ctx.compile_wasm(self.wasm.clone()).await?, + dialect: self.dialect, + program: self.program.clone(), + }) + } +} diff --git a/lib/wcgi-runner/src/module_loader/webc.rs b/lib/wcgi-runner/src/module_loader/webc.rs new file mode 100644 index 00000000000..f10cb845eb0 --- /dev/null +++ b/lib/wcgi-runner/src/module_loader/webc.rs @@ -0,0 +1,155 @@ +use bytes::Bytes; +use wcgi_host::CgiDialect; +use webc::{metadata::Manifest, v1::ParseOptions, v2::read::OwnedReader, Version}; + +use crate::{ + annotations::{WasiCommandAnnotation, WcgiAnnotation}, + errors::WebcLoadError, + module_loader::{ModuleLoader, ModuleLoaderContext}, + Error, +}; + +use super::LoadedModule; + +pub(crate) struct WebcLoader { + command: String, + atom: Bytes, + dialect: CgiDialect, +} + +impl WebcLoader { + /// Create a new [`WebcLoader`] which uses the WEBC file's default + /// entrypoint. + pub fn new(webc: impl Into) -> Result { + WebcLoader::new_with_options(&WebcOptions::default(), webc) + } + + pub fn new_with_options( + options: &WebcOptions<'_>, + webc: impl Into, + ) -> Result { + let webc = webc.into(); + match webc::detect(webc.as_ref())? { + Version::V1 => WebcLoader::v1(options, webc), + Version::V2 => WebcLoader::v2(options, webc), + other => Err(WebcLoadError::UnsupportedVersion(other)), + } + } + + fn v1(options: &WebcOptions<'_>, webc: Bytes) -> Result { + let parse_options = ParseOptions::default(); + let webc = webc::v1::WebC::parse(&webc, &parse_options)?; + + let (command, atom, dialect) = get_atom_and_dialect(&webc.manifest, options, |name| { + let pkg = webc.get_package_name(); + webc.get_atom(&pkg, name).ok().map(|b| b.to_vec().into()) + })?; + + Ok(WebcLoader { + command, + atom, + dialect, + }) + } + + fn v2(options: &WebcOptions<'_>, webc: Bytes) -> Result { + let webc = OwnedReader::parse(webc)?; + let (command, atom, dialect) = get_atom_and_dialect(webc.manifest(), options, |name| { + webc.get_atom(name).cloned() + })?; + + Ok(WebcLoader { + command, + atom, + dialect, + }) + } +} + +#[async_trait::async_trait] +impl ModuleLoader for WebcLoader { + async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result { + Ok(LoadedModule { + module: ctx.compile_wasm(self.atom.clone()).await?, + dialect: self.dialect, + program: self.command.clone(), + }) + } +} + +fn get_atom_and_dialect( + manifest: &Manifest, + options: &WebcOptions<'_>, + get_atom: impl FnOnce(&str) -> Option, +) -> Result<(String, Bytes, CgiDialect), WebcLoadError> { + let command = options + .command + .as_str() + .or(manifest.entrypoint.as_deref()) + .ok_or(WebcLoadError::UnknownEntrypoint)?; + + let cmd = manifest + .commands + .get(command) + .ok_or_else(|| WebcLoadError::UnknownCommand { + name: command.to_string(), + })?; + + let wcgi_annotations: WcgiAnnotation = cmd + .annotations + .get("wcgi") + .cloned() + .and_then(|a| serde_cbor::value::from_value(a).ok()) + .unwrap_or_default(); + + let wasi_annotations: WasiCommandAnnotation = cmd + .annotations + .get("wasi") + .cloned() + .and_then(|a| serde_cbor::value::from_value(a).ok()) + .unwrap_or_default(); + + // Note: Not all WCGI binaries have "wcgi" annotations (e.g. because they + // were published before "wasmer.toml" started using it), so we fall back + // to using the command as our atom name. + let atom_name = wasi_annotations.atom.as_deref().unwrap_or(command); + + let atom = get_atom(atom_name).ok_or_else(|| WebcLoadError::MissingAtom { + name: atom_name.to_string(), + })?; + + // Note: We explicitly use WCGI instead of CgiDialect::default() so we can + // use existing WCGI packages. We also prefer the user-provided CgiDialect + // so they can work around packages with bad metadata. + + Ok(( + command.to_string(), + atom, + options + .dialect + .or(wcgi_annotations.dialect) + .unwrap_or(CgiDialect::Wcgi), + )) +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct WebcOptions<'a> { + pub command: WebcCommand<'a>, + pub dialect: Option, +} + +#[derive(Debug, Default, Copy, Clone)] +pub(crate) enum WebcCommand<'a> { + #[default] + Entrypoint, + Named(&'a str), +} + +impl<'a> WebcCommand<'a> { + fn as_str(self) -> Option<&'a str> { + match self { + WebcCommand::Entrypoint => None, + WebcCommand::Named(name) => Some(name), + } + } +} diff --git a/lib/wcgi-runner/src/runner.rs b/lib/wcgi-runner/src/runner.rs new file mode 100644 index 00000000000..d28fd8f706c --- /dev/null +++ b/lib/wcgi-runner/src/runner.rs @@ -0,0 +1,85 @@ +use std::{future::Future, pin::Pin, sync::Arc, task::Poll}; + +use futures::FutureExt; +use http::{Request, Response}; +use hyper::Body; +use tower_service::Service; + +use crate::{context::Context, Builder, Error}; + +/// A runner for WCGI binaries. +/// +/// # Examples +/// +/// [`Runner`] implements the [`Service`] trait and is cheaply cloneable +/// so it can be easily integrated with a Hyper server and the Tower ecosystem. +/// +/// ```rust,no_run +/// # use std::net::SocketAddr; +/// # #[tokio::main] +/// # async fn main() -> Result<(), Box> { +/// let webc = b"..."; +/// let address: SocketAddr = ([127, 0, 0, 1], 3000).into(); +/// let runner = Runner::builder().build_webc(webc)?; +/// +/// let make_service = hyper::service::make_service_fn(move |_| { +/// let runner = runner.clone(); +/// async move { Ok::<_, std::convert::Infallible>(runner) } +/// }); +/// +/// hyper::Server::bind(&address).serve(make_service).await?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Clone)] +pub struct Runner { + ctx: Arc, +} + +impl Runner { + pub(crate) fn new(ctx: Arc) -> Self { + Runner { ctx } + } + + /// Create a [`Builder`] that can be used to configure a [`Runner`]. + pub fn builder() -> Builder { + Builder::new() + } + + /// Handle a single HTTP request. + pub async fn handle(&self, request: Request) -> Result, Error> { + self.ctx.handle(request).await + } +} + +impl Service> for Runner { + type Response = Response; + type Error = Error; + type Future = Pin, Error>> + Send>>; + + fn poll_ready(&mut self, _cx: &mut std::task::Context<'_>) -> Poll> { + // TODO: We probably should implement some sort of backpressure here... + Poll::Ready(Ok(())) + } + + fn call(&mut self, request: Request) -> Self::Future { + let ctx = Arc::clone(&self.ctx); + let fut = async move { ctx.handle(request).await }; + fut.boxed() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[allow(clippy::extra_unused_type_parameters)] + fn send_and_sync() { + fn assert_send() {} + fn assert_sync() {} + + assert_send::(); + assert_sync::(); + } +} diff --git a/lib/wcgi-runner/tests/integration.rs b/lib/wcgi-runner/tests/integration.rs new file mode 100644 index 00000000000..43761375dc0 --- /dev/null +++ b/lib/wcgi-runner/tests/integration.rs @@ -0,0 +1,153 @@ +use std::{ + path::{Path, PathBuf}, + process::{Command, Output, Stdio}, +}; + +use bytes::Bytes; +use http::{Request, StatusCode}; +use hyper::{body::HttpBody, Body}; +use wcgi_host::CgiDialect; +use wcgi_runner::Builder; + +const FERRIS_SAYS: &str = "https://registry-cdn.wapm.dev/packages/wasmer-examples/ferris-says/ferris-says-0.2.0-2f5dfb76-77a9-11ed-a646-d2c429a5b858.webc"; + +#[tokio::test] +async fn execute_a_webc_server() { + let ferris_says = cached(FERRIS_SAYS); + + let runner = Builder::default() + .cgi_dialect(CgiDialect::Wcgi) + .build_webc(ferris_says) + .unwrap(); + let req = Request::new(Body::default()); + let response = runner.handle(req).await.unwrap(); + + let (parts, mut body) = response.into_parts(); + assert_eq!(parts.status, StatusCode::OK); + let mut buffer = Vec::new(); + while let Some(result) = body.data().await { + let chunk = result.unwrap(); + buffer.extend(chunk); + } + let body = String::from_utf8(buffer).unwrap(); + assert!(body.contains("Wasmer Deploy <3 Rustaceans!")); +} + +#[tokio::test] +async fn execute_a_webassembly_server_with_mounted_directories() { + let static_server = build_wasi("staticserver").join("serve.wasm"); + let wasm = std::fs::read(&static_server).unwrap(); + + let runner = Builder::default() + .program(static_server.display().to_string()) + .map_dir( + "example", + project_root() + .join("examples") + .join("staticserver") + .join("example"), + ) + .build_wasm(wasm) + .unwrap(); + let req = Request::builder() + .uri("/example/index.html") + .body(Body::default()) + .unwrap(); + let response = runner.handle(req).await.unwrap(); + + let (parts, mut body) = response.into_parts(); + assert_eq!(parts.status, StatusCode::OK); + let mut buffer = Vec::new(); + while let Some(result) = body.data().await { + let chunk = result.unwrap(); + buffer.extend(chunk); + } + let body = String::from_utf8(buffer).unwrap(); + assert!(body.contains("

Welcome to WebC

")); +} + +/// Download a file, caching it in $CARGO_TARGET_TMPDIR to avoid unnecessary +/// downloads. +fn cached(url: &str) -> Bytes { + let uri: http::Uri = url.parse().unwrap(); + + let file_name = Path::new(uri.path()).file_name().unwrap(); + let cache_dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(env!("CARGO_PKG_NAME")); + let cached_path = cache_dir.join(file_name); + + if cached_path.exists() { + return std::fs::read(&cached_path).unwrap().into(); + } + + let response = ureq::get(url).call().unwrap(); + assert_eq!( + response.status(), + 200, + "Unable to get \"{url}\": {} {}", + response.status(), + response.status_text() + ); + + let mut body = Vec::new(); + response.into_reader().read_to_end(&mut body).unwrap(); + + std::fs::create_dir_all(&cache_dir).unwrap(); + std::fs::write(&cached_path, &body).unwrap(); + + body.into() +} + +/// Compile a package in this workspace to `wasm32-wasi` and get the directory +/// the final binary was saved to. +fn build_wasi(name: &str) -> PathBuf { + let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); + + let mut cmd = Command::new(cargo); + cmd.arg("build") + .arg("--target=wasm32-wasi") + .args(["--package", name]) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + + // Note: We were seeing build failures in CI because "cargo llvm-cov" + // automatically sets $RUSTFLAGS to include "-Zinstrument-coverage" and + // "profielr_builtins" isn't available for WebAssembly. + // See https://github.com/taiki-e/cargo-llvm-cov/issues/221 + cmd.env("RUSTFLAGS", ""); + + let Output { + status, + stdout, + stderr, + } = cmd.output().expect("Unable to invoke cargo"); + + if !status.success() { + if !stdout.is_empty() { + eprintln!("---- STDOUT ----"); + eprintln!("{}", String::from_utf8_lossy(&stdout)); + } + if !stderr.is_empty() { + eprintln!("---- STDERR ----"); + eprintln!("{}", String::from_utf8_lossy(&stderr)); + } + panic!("{cmd:?} failed with {status}"); + } + + let target_dir = match std::env::var("CARGO_TARGET_DIR") { + Ok(s) => PathBuf::from(s), + Err(_) => project_root().join("target"), + }; + target_dir.join("wasm32-wasi").join("debug") +} + +fn project_root() -> &'static Path { + let path = Path::new(env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(2) + .unwrap(); + + assert!(path.join(".git").exists()); + + path +} From 489108de4e4ad5f869f6eb149082d89de101002e Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 22 Feb 2023 22:54:20 +0800 Subject: [PATCH 02/26] Made lints happy and got tests working again --- Cargo.lock | 34 +------- lib/wcgi-runner/Cargo.toml | 33 +++----- .../{src/bin => examples}/wcgi-runner.rs | 0 lib/wcgi-runner/src/builder.rs | 2 +- lib/wcgi-runner/src/lib.rs | 2 + lib/wcgi-runner/src/module_loader/wasm.rs | 2 +- lib/wcgi-runner/src/runner.rs | 6 +- lib/wcgi-runner/tests/integration.rs | 84 +++---------------- 8 files changed, 38 insertions(+), 125 deletions(-) rename lib/wcgi-runner/{src/bin => examples}/wcgi-runner.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index f670118d1e2..a60d858066b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -103,12 +103,6 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eab1c04a571841102f5345a8fc0f6bb3d31c315dec879b5c6e42e40ce7ffa34e" -[[package]] -name = "ascii" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" - [[package]] name = "assert_cmd" version = "1.0.8" @@ -398,12 +392,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "chunked_transfer" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cca491388666e04d7248af3f60f0c40cfb0991c72205595d7c396e3510207d1a" - [[package]] name = "cipher" version = "0.4.3" @@ -548,7 +536,7 @@ version = "3.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da3da6baa321ec19e1cc41d31bf599f00c783d0517095cdaf0332e3fe8d20680" dependencies = [ - "ascii 0.9.3", + "ascii", "byteorder", "either", "memchr", @@ -4015,18 +4003,6 @@ dependencies = [ "syn", ] -[[package]] -name = "tiny_http" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "389915df6413a2e74fb181895f933386023c71110878cd0825588928e64cdc82" -dependencies = [ - "ascii 1.1.0", - "chunked_transfer", - "httpdate", - "log", -] - [[package]] name = "tinytemplate" version = "1.2.1" @@ -5649,9 +5625,9 @@ dependencies = [ [[package]] name = "wcgi" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "569971bb4ff2a37d0c417821f9cc5e3291aa7fa28e033a405c9da16e86b6fe52" +checksum = "3c55e796fc6fedd026ae32f2ddc3e09df539c456e0089e9b2d59d9eb0acd9d09" dependencies = [ "http", "http-serde", @@ -5684,12 +5660,10 @@ dependencies = [ "futures", "http", "hyper", - "md5", - "reqwest", "serde", "serde_cbor", + "tempfile", "thiserror", - "tiny_http", "tokio", "tower-service", "tracing", diff --git a/lib/wcgi-runner/Cargo.toml b/lib/wcgi-runner/Cargo.toml index 2fa3ed00bfb..ba2042cf861 100644 --- a/lib/wcgi-runner/Cargo.toml +++ b/lib/wcgi-runner/Cargo.toml @@ -11,37 +11,30 @@ edition = "2018" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -anyhow = { version = "1", features = ["backtrace"] } async-trait = "0.1.64" bytes = "1.4.0" -clap = { version = "4", features = ["derive", "env"] } futures = "0.3.25" http = "0.2.8" -hyper = { version = "0.14.23", features = ["server", "stream"] } -md5 = "0.7.0" -serde = { version = "1.0.152", features = ["derive"] } -serde_cbor = "0.11.2" -thiserror = "1.0.38" -tiny_http = "0.12.0" +hyper = { version = "0.14", features = ["server", "stream"] } +serde = { version = "1.0", features = ["derive"] } +serde_cbor = "0.11" +thiserror = "1.0" tokio = { version = "1", features = ["rt-multi-thread", "macros"] } tower-service = "0.3" tracing = "0.1.37" -tracing-subscriber = { version = "0.3.16", features = ["fmt", "env-filter"] } -wasmer = { version = "3.2.0-alpha.1", path = "../api", default-features = false, features = [ - "sys", - "cranelift", - "singlepass", -] } -wasmer-wasi = { version = "3.2.0-alpha.1", path = "../wasi", features = [ - "sys-default", -], default-features = false } +wasmer = { version = "3.2.0-alpha.1", path = "../api", default-features = false } wasmer-vfs = { version = "3.2.0-alpha.1", path = "../vfs", default-features = false } +wasmer-wasi = { version = "3.2.0-alpha.1", path = "../wasi", default-features = false, features = ["sys-default"] } wcgi = { version = "0.1.1" } wcgi-host = { version = "0.1.0" } webc = { version = "5.0.0-rc.1", default-features = false } [dev-dependencies] -reqwest = { version = "0.11.0", default-features = false, features = [ - "rustls-tls", -] } +anyhow = { version = "1", features = ["backtrace"] } +clap = { version = "4", features = ["derive", "env"] } +tempfile = "3.3.0" +tracing-subscriber = { version = "0.3.16", features = ["fmt", "env-filter"] } ureq = "2.6.2" + +[features] +default = ["wasmer/sys", "wasmer/singlepass"] diff --git a/lib/wcgi-runner/src/bin/wcgi-runner.rs b/lib/wcgi-runner/examples/wcgi-runner.rs similarity index 100% rename from lib/wcgi-runner/src/bin/wcgi-runner.rs rename to lib/wcgi-runner/examples/wcgi-runner.rs diff --git a/lib/wcgi-runner/src/builder.rs b/lib/wcgi-runner/src/builder.rs index 974ce8b36ab..3086709cd2f 100644 --- a/lib/wcgi-runner/src/builder.rs +++ b/lib/wcgi-runner/src/builder.rs @@ -25,7 +25,7 @@ use crate::{ /// ```rust,no_run /// use wcgi_runner::Runner; /// # #[tokio::main] -/// # async fn main() -> Result<(), Box> { +/// # async fn main() -> Result<(), Box> { /// let webc = std::fs::read("path/to/server.webc")?; /// let runner = Runner::builder().build_webc(webc)?; /// # Ok(()) diff --git a/lib/wcgi-runner/src/lib.rs b/lib/wcgi-runner/src/lib.rs index c72e74d54a0..32279661917 100644 --- a/lib/wcgi-runner/src/lib.rs +++ b/lib/wcgi-runner/src/lib.rs @@ -1,3 +1,5 @@ +// Rust 1.64 doesn't understand tool-specific lints +#![warn(unknown_lints)] // For now, let's ignore the fact that some of our Error variants are really big #![allow(clippy::result_large_err)] diff --git a/lib/wcgi-runner/src/module_loader/wasm.rs b/lib/wcgi-runner/src/module_loader/wasm.rs index 595bc86b8de..a7542a99b78 100644 --- a/lib/wcgi-runner/src/module_loader/wasm.rs +++ b/lib/wcgi-runner/src/module_loader/wasm.rs @@ -7,7 +7,7 @@ use crate::{ Error, }; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct WasmLoader { program: String, wasm: Bytes, diff --git a/lib/wcgi-runner/src/runner.rs b/lib/wcgi-runner/src/runner.rs index d28fd8f706c..bc16de9064a 100644 --- a/lib/wcgi-runner/src/runner.rs +++ b/lib/wcgi-runner/src/runner.rs @@ -15,10 +15,12 @@ use crate::{context::Context, Builder, Error}; /// so it can be easily integrated with a Hyper server and the Tower ecosystem. /// /// ```rust,no_run -/// # use std::net::SocketAddr; +/// use std::net::SocketAddr; +/// use wcgi_runner::Runner; +/// /// # #[tokio::main] /// # async fn main() -> Result<(), Box> { -/// let webc = b"..."; +/// let webc: &[u8] = b"..."; /// let address: SocketAddr = ([127, 0, 0, 1], 3000).into(); /// let runner = Runner::builder().build_webc(webc)?; /// diff --git a/lib/wcgi-runner/tests/integration.rs b/lib/wcgi-runner/tests/integration.rs index 43761375dc0..8164ed2e809 100644 --- a/lib/wcgi-runner/tests/integration.rs +++ b/lib/wcgi-runner/tests/integration.rs @@ -1,15 +1,15 @@ -use std::{ - path::{Path, PathBuf}, - process::{Command, Output, Stdio}, -}; +use std::path::Path; use bytes::Bytes; use http::{Request, StatusCode}; use hyper::{body::HttpBody, Body}; +use tempfile::TempDir; use wcgi_host::CgiDialect; use wcgi_runner::Builder; const FERRIS_SAYS: &str = "https://registry-cdn.wapm.dev/packages/wasmer-examples/ferris-says/ferris-says-0.2.0-2f5dfb76-77a9-11ed-a646-d2c429a5b858.webc"; +const STATICSERVER: &str = + "https://registry-cdn.wapm.dev/contents/syrusakbary/staticserver/1.0.2/serve.wasm"; #[tokio::test] async fn execute_a_webc_server() { @@ -35,22 +35,19 @@ async fn execute_a_webc_server() { #[tokio::test] async fn execute_a_webassembly_server_with_mounted_directories() { - let static_server = build_wasi("staticserver").join("serve.wasm"); - let wasm = std::fs::read(&static_server).unwrap(); + let wasm = cached(STATICSERVER); + let temp = TempDir::new().unwrap(); + let example = temp.path().join("example"); + std::fs::create_dir_all(&example).unwrap(); + std::fs::write(example.join("file.txt"), b"Hello, World!").unwrap(); let runner = Builder::default() - .program(static_server.display().to_string()) - .map_dir( - "example", - project_root() - .join("examples") - .join("staticserver") - .join("example"), - ) + .program("staticserver") + .map_dir("example", example) .build_wasm(wasm) .unwrap(); let req = Request::builder() - .uri("/example/index.html") + .uri("/example/file.txt") .body(Body::default()) .unwrap(); let response = runner.handle(req).await.unwrap(); @@ -63,7 +60,7 @@ async fn execute_a_webassembly_server_with_mounted_directories() { buffer.extend(chunk); } let body = String::from_utf8(buffer).unwrap(); - assert!(body.contains("

Welcome to WebC

")); + assert!(body.contains("Hello, World!")); } /// Download a file, caching it in $CARGO_TARGET_TMPDIR to avoid unnecessary @@ -96,58 +93,3 @@ fn cached(url: &str) -> Bytes { body.into() } - -/// Compile a package in this workspace to `wasm32-wasi` and get the directory -/// the final binary was saved to. -fn build_wasi(name: &str) -> PathBuf { - let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string()); - - let mut cmd = Command::new(cargo); - cmd.arg("build") - .arg("--target=wasm32-wasi") - .args(["--package", name]) - .stdin(Stdio::null()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - - // Note: We were seeing build failures in CI because "cargo llvm-cov" - // automatically sets $RUSTFLAGS to include "-Zinstrument-coverage" and - // "profielr_builtins" isn't available for WebAssembly. - // See https://github.com/taiki-e/cargo-llvm-cov/issues/221 - cmd.env("RUSTFLAGS", ""); - - let Output { - status, - stdout, - stderr, - } = cmd.output().expect("Unable to invoke cargo"); - - if !status.success() { - if !stdout.is_empty() { - eprintln!("---- STDOUT ----"); - eprintln!("{}", String::from_utf8_lossy(&stdout)); - } - if !stderr.is_empty() { - eprintln!("---- STDERR ----"); - eprintln!("{}", String::from_utf8_lossy(&stderr)); - } - panic!("{cmd:?} failed with {status}"); - } - - let target_dir = match std::env::var("CARGO_TARGET_DIR") { - Ok(s) => PathBuf::from(s), - Err(_) => project_root().join("target"), - }; - target_dir.join("wasm32-wasi").join("debug") -} - -fn project_root() -> &'static Path { - let path = Path::new(env!("CARGO_MANIFEST_DIR")) - .ancestors() - .nth(2) - .unwrap(); - - assert!(path.join(".git").exists()); - - path -} From 7d409f8b079e3b452040b4350eabdfd7bc2387e8 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 24 Feb 2023 22:47:14 +0800 Subject: [PATCH 03/26] Upgraded everything to use webc v5 --- Cargo.lock | 587 ++++++++++++++------------- lib/c-api/Cargo.toml | 2 +- lib/cli/Cargo.toml | 2 +- lib/cli/src/commands/create_exe.rs | 9 +- lib/cli/src/commands/create_obj.rs | 2 +- lib/cli/src/commands/gen_c_header.rs | 4 +- lib/registry/src/lib.rs | 18 +- lib/vfs/Cargo.toml | 2 +- lib/vfs/src/static_fs.rs | 11 +- lib/vfs/src/webc_fs.rs | 4 +- lib/wasi/Cargo.toml | 8 +- lib/wasi/src/runners/container.rs | 10 + lib/wasi/src/runners/emscripten.rs | 2 +- lib/wasi/src/runners/mod.rs | 21 +- lib/wasi/src/runners/wasi.rs | 2 +- lib/wasi/src/wapm/mod.rs | 19 +- lib/wcgi-runner/Cargo.toml | 2 +- 17 files changed, 376 insertions(+), 329 deletions(-) create mode 100644 lib/wasi/src/runners/container.rs diff --git a/Cargo.lock b/Cargo.lock index a60d858066b..7b2ab33c386 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,7 +8,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97" dependencies = [ - "gimli 0.27.1", + "gimli 0.27.2", ] [[package]] @@ -63,18 +63,18 @@ checksum = "70033777eb8b5124a81a1889416543dddef2de240019b674c81285a2635a7e1e" [[package]] name = "anyhow" -version = "1.0.68" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2cb2f989d18dd141ab8ae82f64d1a8cdd37e0840f73a406896cf5e99502fab61" +checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" dependencies = [ "backtrace", ] [[package]] name = "arbitrary" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0224938f92e7aef515fac2ff2d18bd1115c1394ddf4a092e0c87e8be9499ee5" +checksum = "3e90af4de65aa7b293ef2d09daff88501eb254f58edde2e1ac02c82d873eadad" dependencies = [ "derive_arbitrary", ] @@ -119,9 +119,9 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.64" +version = "0.1.66" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd7fce9ba8c3c042128ce72d8b2ddbf3a05747efb67ea0313c635e10bda47a2" +checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" dependencies = [ "proc-macro2", "quote", @@ -187,6 +187,15 @@ version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" +[[package]] +name = "basic-toml" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0de75129aa8d0cceaf750b89013f0e08804d6ec61416da787b35ad0d7cddf1" +dependencies = [ + "serde", +] + [[package]] name = "bincode" version = "1.3.3" @@ -234,7 +243,6 @@ dependencies = [ "lazy_static", "memchr", "regex-automata", - "serde", ] [[package]] @@ -260,19 +268,20 @@ checksum = "0d261e256854913907f67ed06efbc3338dfe6179796deefc1ff763fc1aee5535" [[package]] name = "bytecheck" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d11cac2c12b5adc6570dad2ee1b87eff4955dac476fe12d81e5fdd352e52406f" +checksum = "13fe11640a23eb24562225322cd3e452b93a3d4091d62fab69c70542fcd17d1f" dependencies = [ "bytecheck_derive", "ptr_meta", + "simdutf8", ] [[package]] name = "bytecheck_derive" -version = "0.6.9" +version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e576ebe98e605500b3c8041bb888e966653577172df6dd97398714eb30b9bf" +checksum = "e31225543cb46f81a7e224762764f4a6a0f097b1db0b175f69e8065efaa42de5" dependencies = [ "proc-macro2", "quote", @@ -296,15 +305,15 @@ dependencies = [ [[package]] name = "bytesize" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c58ec36aac5066d5ca17df51b3e70279f5670a72102f5752cb7e7c856adfc70" +checksum = "38fcc2979eff34a4b84e1cf9a1e3da42a7d44b3b690a40cdcb23e3d556cfb2e5" [[package]] name = "camino" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77df041dc383319cc661b428b6961a005db4d6808d5e12536931b1ca9556055" +checksum = "6031a462f977dd38968b6f23378356512feeace69cef817e1a4475108093cec3" dependencies = [ "serde", ] @@ -320,9 +329,9 @@ dependencies = [ [[package]] name = "cargo_metadata" -version = "0.15.2" +version = "0.15.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "982a0cf6a99c350d7246035613882e376d58cebe571785abc5da4f648d53ac0a" +checksum = "08a1ec454bc3eead8719cb56e15dbbfecdbc14e4b3a3ae4936cc6e31f5fc0d07" dependencies = [ "camino", "cargo-platform", @@ -345,7 +354,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6358dedf60f4d9b8db43ad187391afe959746101346fe51bb978126bec61dfb" dependencies = [ "clap 3.2.23", - "heck 0.4.0", + "heck 0.4.1", "indexmap", "log", "proc-macro2", @@ -359,9 +368,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.78" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a20104e2335ce8a659d6dd92a51a767a0c062599c73b343fd152cb401e828c3d" +checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" dependencies = [ "jobserver", ] @@ -432,13 +441,13 @@ dependencies = [ [[package]] name = "clap" -version = "4.1.6" +version = "4.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0b0588d44d4d63a87dbd75c136c166bbfd9a86a31cb89e09906521c7d3f5e3" +checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" dependencies = [ "bitflags", - "clap_derive 4.1.0", - "clap_lex 0.3.1", + "clap_derive 4.1.8", + "clap_lex 0.3.2", "is-terminal", "once_cell", "strsim", @@ -451,7 +460,7 @@ version = "3.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea0c8bce528c4be4da13ea6fead8965e95b6073585a2f05204bd8f4119f82a65" dependencies = [ - "heck 0.4.0", + "heck 0.4.1", "proc-macro-error", "proc-macro2", "quote", @@ -460,11 +469,11 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.1.0" +version = "4.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "684a277d672e91966334af371f1a7b5833f9aa00b07c84e92fbce95e00208ce8" +checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" dependencies = [ - "heck 0.4.0", + "heck 0.4.1", "proc-macro-error", "proc-macro2", "quote", @@ -482,9 +491,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "783fe232adfca04f90f56201b26d79682d4cd2625e0bc7290b95123afe558ade" +checksum = "350b9cf31731f9957399229e9b2adc51eeabdfbe9d71d9a0552275fd12710d09" dependencies = [ "os_str_bytes", ] @@ -673,7 +682,7 @@ dependencies = [ "log", "regalloc2", "smallvec", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", ] [[package]] @@ -721,7 +730,7 @@ dependencies = [ "hashbrown 0.12.3", "log", "smallvec", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", ] [[package]] @@ -792,9 +801,9 @@ checksum = "6548a0ad5d2549e111e1f6a11a6c2e2d00ce6a3dafe22948d67c2b443f775e52" [[package]] name = "crossbeam-channel" -version = "0.5.6" +version = "0.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" +checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -802,9 +811,9 @@ dependencies = [ [[package]] name = "crossbeam-deque" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc" +checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef" dependencies = [ "cfg-if 1.0.0", "crossbeam-epoch", @@ -813,22 +822,22 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.13" +version = "0.9.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a" +checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695" dependencies = [ "autocfg", "cfg-if 1.0.0", "crossbeam-utils", - "memoffset 0.7.1", + "memoffset 0.8.0", "scopeguard", ] [[package]] name = "crossbeam-utils" -version = "0.8.14" +version = "0.8.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f" +checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b" dependencies = [ "cfg-if 1.0.0", ] @@ -845,13 +854,12 @@ dependencies = [ [[package]] name = "csv" -version = "1.1.6" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22813a6dc45b335f9bade10bf7271dc477e81113e89eb251a0bc2a8a81c536e1" +checksum = "af91f40b7355f82b0a891f50e70399475945bb0b0da4f1700ce60761c9d3e359" dependencies = [ - "bstr", "csv-core", - "itoa 0.4.8", + "itoa", "ryu", "serde", ] @@ -883,9 +891,9 @@ checksum = "b365fabc795046672053e29c954733ec3b05e4be654ab130fe8f1f94d7051f35" [[package]] name = "cxx" -version = "1.0.87" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b61a7545f753a88bcbe0a70de1fcc0221e10bfc752f576754fa91e663db1622e" +checksum = "9a140f260e6f3f79013b8bfc65e7ce630c9ab4388c6a89c71e07226f49487b72" dependencies = [ "cc", "cxxbridge-flags", @@ -895,9 +903,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.87" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f464457d494b5ed6905c63b0c4704842aba319084a0a3561cdc1359536b53200" +checksum = "da6383f459341ea689374bf0a42979739dc421874f112ff26f829b8040b8e613" dependencies = [ "cc", "codespan-reporting", @@ -910,15 +918,15 @@ dependencies = [ [[package]] name = "cxxbridge-flags" -version = "1.0.87" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43c7119ce3a3701ed81aca8410b9acf6fc399d2629d057b87e2efa4e63a3aaea" +checksum = "90201c1a650e95ccff1c8c0bb5a343213bdd317c6e600a93075bca2eff54ec97" [[package]] name = "cxxbridge-macro" -version = "1.0.87" +version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e07508b90551e610910fa648a1878991d367064997a596135b86df30daf07e" +checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" dependencies = [ "proc-macro2", "quote", @@ -927,9 +935,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0dd3cd20dc6b5a876612a6e5accfe7f3dd883db6d07acfbf14c128f61550dfa" +checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8" dependencies = [ "darling_core", "darling_macro", @@ -937,9 +945,9 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a784d2ccaf7c98501746bf0be29b2022ba41fd62a2e622af997a03e9f972859f" +checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb" dependencies = [ "fnv", "ident_case", @@ -950,9 +958,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.14.2" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7618812407e9402654622dd402b0a89dff9ba93badd6540781526117b92aab7e" +checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685" dependencies = [ "darling_core", "quote", @@ -972,9 +980,9 @@ dependencies = [ [[package]] name = "derive_arbitrary" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf460bbff5f571bfc762da5102729f59f338be7db17a21fade44c5c4f5005350" +checksum = "8beee4701e2e229e8098bbdecdca12449bc3e322f137d269182fa1291e20bd00" dependencies = [ "proc-macro2", "quote", @@ -1118,9 +1126,9 @@ dependencies = [ [[package]] name = "either" -version = "1.8.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" +checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" [[package]] name = "encode_unicode" @@ -1136,9 +1144,9 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] name = "encoding_rs" -version = "0.8.31" +version = "0.8.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9852635589dc9f9ea1b6fe9f05b50ef208c85c834a562f0c6abb1c475736ec2b" +checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" dependencies = [ "cfg-if 1.0.0", ] @@ -1186,9 +1194,9 @@ dependencies = [ [[package]] name = "erased-serde" -version = "0.3.24" +version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ca605381c017ec7a5fef5e548f1cfaa419ed0f6df6367339300db74c92aa7d" +checksum = "4f2b0c2380453a92ea8b6c8e5f64ecaafccddde8ceab55ff7a8ac1029f894569" dependencies = [ "serde", ] @@ -1228,9 +1236,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "1.8.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a407cfaa3385c4ae6b23e84623d48c2798d06e3e6a1878f7f59f17b3f86499" +checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" dependencies = [ "instant", ] @@ -1247,24 +1255,24 @@ dependencies = [ [[package]] name = "field-offset" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e1c54951450cbd39f3dbcf1005ac413b49487dabf18a720ad2383eccfeffb92" +checksum = "a3cf3a800ff6e860c863ca6d4b16fd999db8b752819c1606884047b73e468535" dependencies = [ - "memoffset 0.6.5", - "rustc_version 0.3.3", + "memoffset 0.8.0", + "rustc_version 0.4.0", ] [[package]] name = "filetime" -version = "0.2.19" +version = "0.2.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e884668cd0c7480504233e951174ddc3b382f7c2666e3b7310b5c4e7b0c37f9" +checksum = "8a3de6e8d11b22ff9edc6d916f890800597d60f8b2da1caf2955c274638d6412" dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall 0.2.16", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -1303,15 +1311,15 @@ dependencies = [ [[package]] name = "fs_extra" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" [[package]] name = "futures" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38390104763dc37a5145a53c29c63c1290b5d316d6086ec32c293f6736051bb0" +checksum = "13e2792b0ff0340399d58445b88fd9770e3489eff258a4cbc1523418f12abf84" dependencies = [ "futures-channel", "futures-core", @@ -1324,9 +1332,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52ba265a92256105f45b719605a571ffe2d1f0fea3807304b522c1d778f79eed" +checksum = "2e5317663a9089767a1ec00a487df42e0ca174b61b4483213ac24448e4664df5" dependencies = [ "futures-core", "futures-sink", @@ -1334,15 +1342,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04909a7a7e4633ae6c4a9ab280aeb86da1236243a77b694a49eacd659a4bd3ac" +checksum = "ec90ff4d0fe1f57d600049061dc6bb68ed03c7d2fbd697274c41805dcb3f8608" [[package]] name = "futures-executor" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7acc85df6714c176ab5edf386123fafe217be88c0840ec11f199441134a074e2" +checksum = "e8de0a35a6ab97ec8869e32a2473f4b1324459e14c29275d14b10cb1fd19b50e" dependencies = [ "futures-core", "futures-task", @@ -1351,15 +1359,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00f5fb52a06bdcadeb54e8d3671f8888a39697dcb0b81b23b55174030427f4eb" +checksum = "bfb8371b6fb2aeb2d280374607aeabfc99d95c72edfe51692e42d3d7f0d08531" [[package]] name = "futures-macro" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdfb8ce053d86b91919aad980c220b1fb8401a9394410e1c289ed7e66b61835d" +checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ "proc-macro2", "quote", @@ -1368,21 +1376,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39c15cf1a4aa79df40f1bb462fb39676d0ad9e366c2a33b590d7c66f4f81fcf9" +checksum = "f310820bb3e8cfd46c80db4d7fb8353e15dfff853a127158425f31e0be6c8364" [[package]] name = "futures-task" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffb393ac5d9a6eaa9d3fdf37ae2776656b706e200c8e16b1bdb227f5198e6ea" +checksum = "dcf79a1bf610b10f42aea489289c5a2c478a786509693b80cd39c44ccd936366" [[package]] name = "futures-util" -version = "0.3.25" +version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "197676987abd2f9cadff84926f410af1c183608d36641465df73ae8211dc65d6" +checksum = "9c1d6de3acfef38d2be4b1f543f553131788603495be83da675e180c8d6b7bd1" dependencies = [ "futures-channel", "futures-core", @@ -1439,9 +1447,9 @@ dependencies = [ [[package]] name = "ghost" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41973d4c45f7a35af8753ba3457cc99d406d863941fd7f52663cff54a5ab99b3" +checksum = "69e0cd8a998937e25c6ba7cc276b96ec5cc3f4dc4ab5de9ede4fb152bdd5c5eb" dependencies = [ "proc-macro2", "quote", @@ -1461,9 +1469,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "221996f774192f0f718773def8201c4ae31f02616a54ccfc2d358bb0e5cefdec" +checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" [[package]] name = "glob" @@ -1509,7 +1517,7 @@ checksum = "f290ecfa3bea3e8a157899dc8a1d96ee7dd6405c18c8ddd213fc58939d18a0e9" dependencies = [ "graphql-introspection-query", "graphql-parser", - "heck 0.4.0", + "heck 0.4.1", "lazy_static", "proc-macro2", "quote", @@ -1551,9 +1559,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f9f29bc9dda355256b2916cf526ab02ce0aeaaaf2bad60d65ef3f12f11dd0f4" +checksum = "5be7b54589b581f624f566bf5d8eb2bab1db736c51528720b6bd36b96b55924d" dependencies = [ "bytes", "fnv", @@ -1634,9 +1642,9 @@ dependencies = [ [[package]] name = "heck" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2540771e65fc8cb83cd6e8a237f70c319bd5c29f78ed1084ba5d50eeac86f7f9" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] name = "hermit-abi" @@ -1679,13 +1687,13 @@ dependencies = [ [[package]] name = "http" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75f43d41e26995c17e71ee126451dd3941010b0514a81a9d11f3b341debc2399" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" dependencies = [ "bytes", "fnv", - "itoa 1.0.5", + "itoa", ] [[package]] @@ -1735,9 +1743,9 @@ checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" [[package]] name = "hyper" -version = "0.14.23" +version = "0.14.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "034711faac9d2166cb1baf1a2fb0b60b1f277f8492fd72176c17f3515e1abd3c" +checksum = "5e011372fa0b68db8350aa7a248930ecc7839bf46d8485577d69f117a75f164c" dependencies = [ "bytes", "futures-channel", @@ -1748,7 +1756,7 @@ dependencies = [ "http-body", "httparse", "httpdate", - "itoa 1.0.5", + "itoa", "pin-project-lite", "socket2", "tokio", @@ -1966,9 +1974,9 @@ checksum = "30e22bd8629359895450b59ea7a776c850561b96a3b1d31321c1949d9e6c9146" [[package]] name = "is-terminal" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0a45d56fe973d6db23972bf5bc46f988a4a2385deac9cc29572f09daef" +checksum = "21b6b32576413a8e69b90e952e4a026476040d81017b80445deda5f2d3921857" dependencies = [ "hermit-abi 0.3.1", "io-lifetimes", @@ -1999,30 +2007,24 @@ dependencies = [ [[package]] name = "itoa" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" - -[[package]] -name = "itoa" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440" +checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" [[package]] name = "jobserver" -version = "0.1.25" +version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "068b1ee6743e4d11fb9c6a1e6064b3693a1b600e7f5f5988047d98b3dc9fb90b" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49409df3e3bf0856b916e2ceaca09ee28e6871cf7d9ce97a692cacfdb2a25a47" +checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" dependencies = [ "wasm-bindgen", ] @@ -2069,9 +2071,9 @@ checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79" [[package]] name = "libfuzzer-sys" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8fff891139ee62800da71b7fd5b508d570b9ad95e614a53c6f453ca08366038" +checksum = "beb09950ae85a0a94b27676cccf37da5ff13f27076aa1adbc6545dd0d0e1bd4e" dependencies = [ "arbitrary", "cc", @@ -2234,9 +2236,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" [[package]] name = "memmap2" -version = "0.5.8" +version = "0.5.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b182332558b18d807c4ce1ca8ca983b34c3ee32765e47b3f0f69b90355cc1dc" +checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" dependencies = [ "libc", ] @@ -2252,9 +2254,9 @@ dependencies = [ [[package]] name = "memoffset" -version = "0.7.1" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4" +checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1" dependencies = [ "autocfg", ] @@ -2329,14 +2331,14 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5d732bc30207a6423068df043e3d02e0735b155ad7ce1a6f76fe2baa5b158de" +checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", "wasi", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -2401,15 +2403,6 @@ dependencies = [ "minimal-lexical", ] -[[package]] -name = "nom8" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae01545c9c7fc4486ab7debaf2aad7003ac19431791868fb2e8066df97fad2f8" -dependencies = [ - "memchr", -] - [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -2463,18 +2456,18 @@ dependencies = [ [[package]] name = "num_enum" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e0072973714303aa6e3631c7e8e777970cf4bdd25dc4932e41031027b8bcc4e" +checksum = "1f646caf906c20226733ed5b1374287eb97e3c2a5c227ce668c1f2ce20ae57c9" dependencies = [ "num_enum_derive", ] [[package]] name = "num_enum_derive" -version = "0.5.10" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0629cbd6b897944899b1f10496d9c4a7ac5878d45fd61bc22e9e79bfbbc29597" +checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2512,9 +2505,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.17.0" +version = "1.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66" +checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" [[package]] name = "oorandom" @@ -2577,7 +2570,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" dependencies = [ "lock_api", - "parking_lot_core 0.9.6", + "parking_lot_core 0.9.7", ] [[package]] @@ -2596,22 +2589,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.6" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba1ef8814b5c993410bb3adfad7a5ed269563e4a2f90c41f5d85be7fb47133bf" +checksum = "9069cbb9f99e3a5083476ccb29ceb1de18b9118cafa53e90c9551235de2b9521" dependencies = [ "cfg-if 1.0.0", "libc", "redox_syscall 0.2.16", "smallvec", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] name = "paste" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba" +checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79" [[package]] name = "path-clean" @@ -2642,9 +2635,9 @@ checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" [[package]] name = "pest" -version = "2.5.3" +version = "2.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4257b4a04d91f7e9e6290be5d3da4804dd5784fafde3a497d73eb2b4a158c30a" +checksum = "8cbd939b234e95d72bc393d51788aec68aeeb5d51e748ca08ff3aad58cb722f7" dependencies = [ "thiserror", "ucd-trie", @@ -2786,9 +2779,9 @@ dependencies = [ [[package]] name = "proc-macro-crate" -version = "1.3.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66618389e4ec1c7afe67d51a9bf34ff9236480f8d51e7489b7d5ab0303c13f34" +checksum = "7f4c021e1093a56626774e81216a4ce732a735e5bad4868a03f3ed65ca0c3919" dependencies = [ "once_cell", "toml_edit", @@ -2826,9 +2819,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.50" +version = "1.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ef7d57beacfaf2d8aee5937dab7b7f28de3cb8b1828479bb5de2a7106f2bae2" +checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" dependencies = [ "unicode-ident", ] @@ -2948,9 +2941,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7" +checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b" dependencies = [ "either", "rayon-core", @@ -2958,9 +2951,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.10.2" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b" +checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -3224,9 +3217,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.36.8" +version = "0.36.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "fd5c6ff11fecd55b40746d1995a02f2eb375bf8c00d192d521ee09f42bef37bc" dependencies = [ "bitflags", "errno", @@ -3272,15 +3265,15 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.11" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5583e89e108996506031660fe09baa5011b9dd0341b89029313006d1fb508d70" +checksum = "4f3208ce4d8448b3f3e7d168a73f5e0c43a61e32930de3bceeccedb388b6bf06" [[package]] name = "ryu" -version = "1.0.12" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde" +checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" [[package]] name = "salsa20" @@ -3314,9 +3307,9 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "scratch" -version = "1.0.3" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddccb15bcce173023b3fedd9436f882a0739b8dfb45e4f6b6002bee5929f61b2" +checksum = "1792db035ce95be60c3f8853017b3999209281c24e2ba5bc8e59bf97a0c590c1" [[package]] name = "scrypt" @@ -3445,9 +3438,9 @@ dependencies = [ [[package]] name = "serde_bytes" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "718dc5fff5b36f99093fc49b280cfc96ce6fc824317783bff5a1fed0c7a64819" +checksum = "416bda436f9aab92e02c8e10d49a15ddd339cea90b6e340fe51ed97abb548294" dependencies = [ "serde", ] @@ -3475,11 +3468,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.91" +version = "1.0.94" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877c235533714907a8c2464236f5c4b2a17262ef1bd71f38f35ea592c8da6883" +checksum = "1c533a59c9d8a93a09c6ab31f0fd5e5f4dd1b8fc9434804029839884765d04ea" dependencies = [ - "itoa 1.0.5", + "itoa", "ryu", "serde", ] @@ -3491,7 +3484,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" dependencies = [ "form_urlencoded", - "itoa 1.0.5", + "itoa", "ryu", "serde", ] @@ -3510,12 +3503,12 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.17" +version = "0.9.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fb06d4b6cdaef0e0c51fa881acb721bed3c924cfaa71d9c94a3b771dfdf6567" +checksum = "f82e6c8c047aa50a7328632d067bcae6ef38772a79e28daf32f735e0e4f3dd10" dependencies = [ "indexmap", - "itoa 1.0.5", + "itoa", "ryu", "serde", "unsafe-libyaml", @@ -3602,6 +3595,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "similar" version = "2.2.1" @@ -3610,9 +3609,9 @@ checksum = "420acb44afdae038210c99e69aae24109f32f15500aa708e81d46c9f29d55fcf" [[package]] name = "slab" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4614a76b2a8be0058caa9dbbaf66d988527d86d003c11a94fbd335d7661edcef" +checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" dependencies = [ "autocfg", ] @@ -3631,9 +3630,9 @@ checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" [[package]] name = "socket2" -version = "0.4.7" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2d2db9033d13a1567121ddd7a095ee144db4e1ca1b1bda3419bc0da294ebd" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" dependencies = [ "libc", "winapi", @@ -3757,7 +3756,7 @@ version = "0.24.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ - "heck 0.4.0", + "heck 0.4.1", "proc-macro2", "quote", "rustversion", @@ -3772,9 +3771,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.107" +version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ "proc-macro2", "quote", @@ -3800,9 +3799,9 @@ checksum = "422045212ea98508ae3d28025bc5aaa2bd4a9cdaecd442a08da2ee620ee9ea95" [[package]] name = "target-lexicon" -version = "0.12.5" +version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9410d0f6853b1d94f0e519fb95df60f29d2c1eff2d921ffdf01a4c8a3b54f12d" +checksum = "8ae9980cab1db3fceee2f6c6f643d5d8de2997c58ee8d25fb0cc8a9e9e7348e5" [[package]] name = "tempfile" @@ -3867,7 +3866,7 @@ name = "test-generator" version = "0.1.0" dependencies = [ "anyhow", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", ] [[package]] @@ -3911,18 +3910,18 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0" +checksum = "a5ab016db510546d856297882807df8da66a16fb8c4101cb8b30054b0d5b2d9c" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.38" +version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f" +checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" dependencies = [ "proc-macro2", "quote", @@ -3931,10 +3930,11 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.4" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5516c27b78311c50bf42c071425c560ac799b11c30b31f87e3081965fe5e0180" +checksum = "3fdd6f064ccff2d6567adcb3873ca630700f00b5ad3f060c25b5dcfd9a4ce152" dependencies = [ + "cfg-if 1.0.0", "once_cell", ] @@ -3955,14 +3955,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a561bf4617eebd33bca6434b988f39ed798e527f51a1e797d0ee4f61c0a38376" +checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" dependencies = [ - "itoa 1.0.5", + "itoa", "serde", "time-core", - "time-macros 0.2.6", + "time-macros 0.2.8", ] [[package]] @@ -3983,9 +3983,9 @@ dependencies = [ [[package]] name = "time-macros" -version = "0.2.6" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d967f99f534ca7e495c575c62638eebc2898a8c84c119b89e250477bc4ba16b2" +checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" dependencies = [ "time-core", ] @@ -4024,9 +4024,9 @@ dependencies = [ [[package]] name = "tinyvec_macros" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tldextract" @@ -4044,9 +4044,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.24.2" +version = "1.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a12a59981d9e3c38d216785b0c37399f6e415e8d0712047620f189371b0bb" +checksum = "03201d01c3c27a29c8a5cee5b55a93ddae1ccf6f08f65365c2c918f8c1b76f64" dependencies = [ "autocfg", "bytes", @@ -4058,7 +4058,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "windows-sys 0.42.0", + "windows-sys 0.45.0", ] [[package]] @@ -4085,9 +4085,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.4" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bb2e075f03b3d66d8d8785356224ba688d2906a371015e225beeb65ca92c740" +checksum = "5427d89453009325de0d8f342c9490009f76e999cb7672d77e46267448f7e6b2" dependencies = [ "bytes", "futures-core", @@ -4108,19 +4108,19 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.5.1" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4553f467ac8e3d374bc9a177a26801e5d0f9b211aa1673fb137a403afd1c9cf5" +checksum = "3ab8ed2edee10b50132aed5f331333428b011c99402b5a534154ed15746f9622" [[package]] name = "toml_edit" -version = "0.18.1" +version = "0.19.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c59d8dd7d0dcbc6428bf7aa2f0e823e26e43b3c9aca15bbc9475d23e5fa12b" +checksum = "9a1eb0622d28f4b9c90adc4ea4b2b46b47663fde9ac5fafcb14a1369d5508825" dependencies = [ "indexmap", - "nom8", "toml_datetime", + "winnow", ] [[package]] @@ -4243,17 +4243,17 @@ checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" [[package]] name = "trybuild" -version = "1.0.76" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ed2c57956f91546d4d33614265a85d55c8e1ab91484853a10335894786d7db6" +checksum = "db3115bddce1b5f52dd4b5e0ec8298a66ce733e4cc6759247dc2d1c11508ec38" dependencies = [ + "basic-toml", "glob", "once_cell", "serde", "serde_derive", "serde_json", "termcolor", - "toml", ] [[package]] @@ -4309,9 +4309,9 @@ checksum = "d54675592c1dbefd78cbd98db9bacd89886e1ca50692a0692baefffdeb92dd58" [[package]] name = "unicode-ident" -version = "1.0.6" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc" +checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" [[package]] name = "unicode-normalization" @@ -4324,9 +4324,9 @@ dependencies = [ [[package]] name = "unicode-segmentation" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fdbf052a0783de01e944a6ce7a8cb939e295b1e7be835a1112c3b9a7f047a5a" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" [[package]] name = "unicode-width" @@ -4357,9 +4357,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.5" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc7ed8ba44ca06be78ea1ad2c3682a43349126c8818054231ee6f4748012aed2" +checksum = "ad2024452afd3874bf539695e04af6732ba06517424dbf958fdb16a01f3bef6c" [[package]] name = "untrusted" @@ -4600,9 +4600,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eaf9f5aceeec8be17c128b2e93e031fb8a4d469bb9c4ae2d7dc1888b26887268" +checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" dependencies = [ "cfg-if 1.0.0", "wasm-bindgen-macro", @@ -4610,9 +4610,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c8ffb332579b0557b52d268b91feab8df3615f265d5270fec2a8c95b17c1142" +checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" dependencies = [ "bumpalo", "log", @@ -4648,9 +4648,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23639446165ca5a5de86ae1d8896b737ae80319560fbaa4c2887b7da6e7ebd7d" +checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -4660,9 +4660,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "052be0f94026e6cbc75cdefc9bae13fd6052cdcaf532fa6c45e7ae33a1e6c810" +checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4670,9 +4670,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bc0c051dc5f23e307b13285f9d75df86bfdf816c5721e573dec1f9b8aa193c" +checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ "proc-macro2", "quote", @@ -4683,15 +4683,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.83" +version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c38c045535d93ec4f0b4defec448e4291638ee608530863b1e2ba115d4fff7f" +checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "wasm-bindgen-test" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09d2fff962180c3fadf677438054b1db62bee4aa32af26a45388af07d1287e1d" +checksum = "6db36fc0f9fb209e88fb3642590ae0205bb5a56216dabd963ba15879fe53a30b" dependencies = [ "console_error_panic_hook", "js-sys", @@ -4703,9 +4703,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.33" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4683da3dfc016f704c9f82cf401520c4f1cb3ee440f7f52b3d6ac29506a49ca7" +checksum = "0734759ae6b3b1717d661fe4f016efcfb9828f5edb4520c18eaee05af3b43be9" dependencies = [ "proc-macro2", "quote", @@ -4713,9 +4713,9 @@ dependencies = [ [[package]] name = "wasm-coredump-builder" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "158180f35c9ba89a3e7763f20be93e77d5e41535c18e22c85d6dd5b5bce18108" +checksum = "189a9a7d8952ac4103a59ab849a291a40b7d97c55e24a859344e9e240ac08aed" dependencies = [ "wasm-coredump-encoder", "wasm-coredump-types", @@ -4724,9 +4724,9 @@ dependencies = [ [[package]] name = "wasm-coredump-encoder" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0c99cdf3a88363570f1027e2f337de6647cac9fed5d474f86103d7c45c8700" +checksum = "b855c0e9989e3e54ddbd744b3b5d3d7e2ad39e6a3357d2168f63e93d044c65b3" dependencies = [ "leb128", "wasm-coredump-types", @@ -4734,9 +4734,9 @@ dependencies = [ [[package]] name = "wasm-coredump-types" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10e35729a021e44c20511e23ac2b215df05da243bdc4bad336fd3686552539fc" +checksum = "349ae066a33052159feb4261988bb813f3b58f3ad9c60ded5dfce7c75ff7d064" [[package]] name = "wasm-encoder" @@ -4749,18 +4749,18 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef126be0e14bdf355ac1a8b41afc89195289e5c7179f80118e3abddb472f0810" +checksum = "1c3e4bc09095436c8e7584d86d33e6c3ee67045af8fb262cbb9cc321de553428" dependencies = [ "leb128", ] [[package]] name = "wasm-encoder" -version = "0.23.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c3e4bc09095436c8e7584d86d33e6c3ee67045af8fb262cbb9cc321de553428" +checksum = "68f7d56227d910901ce12dfd19acc40c12687994dfb3f57c90690f80be946ec5" dependencies = [ "leb128", ] @@ -4805,7 +4805,7 @@ dependencies = [ "more-asserts", "serde", "serde-wasm-bindgen", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", "tempfile", "thiserror", "tracing", @@ -4865,7 +4865,7 @@ dependencies = [ "wasmer-types", "wasmer-vfs", "wasmer-wasi", - "webc 4.1.1", + "webc 5.0.0-rc.5", ] [[package]] @@ -4940,10 +4940,10 @@ dependencies = [ "sha2", "spinoff", "tar", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", "tempfile", "thiserror", - "time 0.3.17", + "time 0.3.20", "tldextract", "toml", "unix_mode", @@ -4970,7 +4970,7 @@ dependencies = [ "wasmer-wasm-interface", "wasmer-wast", "wasmparser 0.51.4", - "webc 4.1.1", + "webc 5.0.0-rc.5", ] [[package]] @@ -5012,7 +5012,7 @@ dependencies = [ "distance", "fern", "log", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", "unix_mode", "wasmer-compiler", "wasmer-compiler-cranelift", @@ -5033,7 +5033,7 @@ dependencies = [ "more-asserts", "rayon", "smallvec", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", "tracing", "wasmer-compiler", "wasmer-types", @@ -5055,7 +5055,7 @@ dependencies = [ "rustc_version 0.4.0", "semver 1.0.16", "smallvec", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", "wasmer-compiler", "wasmer-types", "wasmer-vm", @@ -5075,7 +5075,7 @@ dependencies = [ "more-asserts", "rayon", "smallvec", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", "wasmer-compiler", "wasmer-types", ] @@ -5149,7 +5149,7 @@ dependencies = [ "rand", "serde", "tar", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", "tempfile", ] @@ -5236,7 +5236,7 @@ dependencies = [ "serde", "serde_cbor", "serde_json", - "serde_yaml 0.9.17", + "serde_yaml 0.9.19", "thiserror", "toml", ] @@ -5253,7 +5253,7 @@ dependencies = [ "rkyv", "serde", "serde_bytes", - "target-lexicon 0.12.5", + "target-lexicon 0.12.6", "thiserror", ] @@ -5276,7 +5276,7 @@ dependencies = [ "tokio", "tracing", "typetag", - "webc 4.1.1", + "webc 5.0.0-rc.5", ] [[package]] @@ -5331,6 +5331,7 @@ dependencies = [ "heapless", "hex", "http", + "hyper", "lazy_static", "libc", "linked_hash_set", @@ -5367,7 +5368,9 @@ dependencies = [ "wasmer-vnet", "wasmer-wasi-local-networking", "wasmer-wasi-types", - "webc 4.1.1", + "wcgi", + "wcgi-host", + "webc 5.0.0-rc.5", "weezl", "winapi", ] @@ -5501,14 +5504,24 @@ dependencies = [ "url", ] +[[package]] +name = "wasmparser" +version = "0.101.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf2f22ef84ac5666544afa52f326f13e16f3d019d2e61e704fd8091c9358b130" +dependencies = [ + "indexmap", + "url", +] + [[package]] name = "wasmprinter" -version = "0.2.48" +version = "0.2.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322949f382cd5e4bad4330e144bf2124b3182846194ac01e2423c07a6a15ba85" +checksum = "003f2e37b9b7caac949d388e185ecd9139f51441249a23880b0cf38e10cdf647" dependencies = [ "anyhow", - "wasmparser 0.98.1", + "wasmparser 0.101.1", ] [[package]] @@ -5531,23 +5544,23 @@ dependencies = [ [[package]] name = "wast" -version = "52.0.1" +version = "54.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829fb867c8e82d21557a2c6c5b3ed8e8f7cdd534ea782b9ecf68bede5607fe4b" +checksum = "3d48d9d731d835f4f8dacbb8de7d47be068812cb9877f5c60d408858778d8d2a" dependencies = [ "leb128", "memchr", "unicode-width", - "wasm-encoder 0.22.0", + "wasm-encoder 0.24.1", ] [[package]] name = "wat" -version = "1.0.55" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3493e7c82d8e9a75e69ecbfe6f324ca1c4e2ae89f67ccbb22f92282e2e27bb23" +checksum = "d1db2e3ed05ea31243761439194bec3af6efbbaf87c4c8667fb879e4f23791a0" dependencies = [ - "wast 52.0.1", + "wast 54.0.1", ] [[package]] @@ -5656,7 +5669,7 @@ dependencies = [ "anyhow", "async-trait", "bytes", - "clap 4.1.6", + "clap 4.1.8", "futures", "http", "hyper", @@ -5674,14 +5687,14 @@ dependencies = [ "wasmer-wasi", "wcgi", "wcgi-host", - "webc 5.0.0-rc.1", + "webc 5.0.0-rc.5", ] [[package]] name = "web-sys" -version = "0.3.60" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcda906d8be16e728fd5adc5b729afad4e444e106ab28cd1c7256e54fa61510f" +checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" dependencies = [ "js-sys", "wasm-bindgen", @@ -5713,9 +5726,9 @@ dependencies = [ [[package]] name = "webc" -version = "5.0.0-rc.1" +version = "5.0.0-rc.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4f6a21de5a426f0ee97316b4f363ca28524489fd065ea437011a5f70f70e34f" +checksum = "418bfd8fc298ce60295203a6960d53af48c8e10c5a021a5e7db8bc06c4830148" dependencies = [ "anyhow", "base64 0.21.0", @@ -5724,9 +5737,10 @@ dependencies = [ "indexmap", "leb128", "lexical-sort", + "memmap2", "once_cell", "path-clean", - "rand 0.8.5", + "rand", "serde", "serde_cbor", "serde_json", @@ -5945,6 +5959,15 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "winnow" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c95fb4ff192527911dd18eb138ac30908e7165b8944e528b6af93aa4c842d345" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.10.1" diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index cb6ab75466b..04413d379ac 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -32,7 +32,7 @@ wasmer-middlewares = { version = "=3.2.0-alpha.1", path = "../middlewares", opti wasmer-wasi = { version = "=3.2.0-alpha.1", path = "../wasi", features = ["host-fs", "host-vnet"], optional = true } wasmer-types = { version = "=3.2.0-alpha.1", path = "../types" } wasmer-vfs = { version = "=3.2.0-alpha.1", path = "../vfs", optional = true, default-features = false, features = ["static-fs"] } -webc = { version = "4.0.0", optional = true } +webc = { version = "5.0.0-rc.5", optional = true } enumset = "1.0.2" cfg-if = "1.0" lazy_static = "1.4" diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index b8f87bb988e..4fe65e42d8d 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -72,7 +72,7 @@ toml = "0.5.9" url = "2.3.1" libc = { version = "^0.2", default-features = false } nuke-dir = { version = "0.1.0", optional = true } -webc = { version = "4.0.0", optional = true } +webc = { version = "5.0.0-rc.5", optional = true } isatty = "0.1.9" dialoguer = "0.10.2" tldextract = "0.6.0" diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 400c62badda..8c5379f061a 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -15,9 +15,10 @@ use std::process::Stdio; use tar::Archive; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; -use wasmer_types::compilation::symbols::ModuleMetadataSymbolRegistry; -use wasmer_types::ModuleInfo; -use webc::{ParseOptions, WebCMmap}; +use wasmer_types::{ + compilation::symbols::ModuleMetadataSymbolRegistry, ModuleInfo, SymbolRegistry, +}; +use webc::v1::{ParseOptions, WebCMmap}; const LINK_SYSTEM_LIBRARIES_WINDOWS: &[&str] = &["userenv", "Ws2_32", "advapi32", "bcrypt"]; @@ -520,7 +521,7 @@ impl PrefixMapCompilation { // if prefixes are specified, have to match the atom names exactly if prefixes.len() != atoms.len() { println!( - "WARNING: invalid mapping of prefix and atoms: expected prefixes for {} atoms, got {} prefixes", + "WARNING: invalid mapping of prefix and atoms: expected prefixes for {} atoms, got {} prefixes", atoms.len(), prefixes.len() ); } diff --git a/lib/cli/src/commands/create_obj.rs b/lib/cli/src/commands/create_obj.rs index a1b4be96b2e..c1873a7aef3 100644 --- a/lib/cli/src/commands/create_obj.rs +++ b/lib/cli/src/commands/create_obj.rs @@ -85,7 +85,7 @@ impl CreateObj { println!("Target: {}", target.triple()); let atoms = if let Ok(pirita) = - webc::WebCMmap::parse(input_path.clone(), &webc::ParseOptions::default()) + webc::v1::WebCMmap::parse(input_path.clone(), &webc::v1::ParseOptions::default()) { crate::commands::create_exe::compile_pirita_into_directory( &pirita, diff --git a/lib/cli/src/commands/gen_c_header.rs b/lib/cli/src/commands/gen_c_header.rs index 2a309af21a6..15da738d364 100644 --- a/lib/cli/src/commands/gen_c_header.rs +++ b/lib/cli/src/commands/gen_c_header.rs @@ -5,7 +5,7 @@ use std::path::PathBuf; use wasmer_compiler::Artifact; use wasmer_types::compilation::symbols::ModuleMetadataSymbolRegistry; use wasmer_types::{CpuFeature, MetadataHeader, Triple}; -use webc::WebC; +use webc::v1::WebC; #[derive(Debug, Parser)] /// The options for the `wasmer gen-c-header` subcommand @@ -57,7 +57,7 @@ impl GenCHeader { None => crate::commands::PrefixMapCompilation::hash_for_bytes(&file), }; - if let Ok(pirita) = WebC::parse(&file, &webc::ParseOptions::default()) { + if let Ok(pirita) = WebC::parse(&file, &webc::v1::ParseOptions::default()) { let atoms = pirita .manifest .atoms diff --git a/lib/registry/src/lib.rs b/lib/registry/src/lib.rs index c4fea9cfe83..4bd8764ae92 100644 --- a/lib/registry/src/lib.rs +++ b/lib/registry/src/lib.rs @@ -690,7 +690,7 @@ pub fn get_all_available_registries(wasmer_dir: &Path) -> Result, St #[derive(Debug, PartialEq, Clone)] pub struct RemoteWebcInfo { pub checksum: String, - pub manifest: webc::Manifest, + pub manifest: webc::metadata::Manifest, } pub fn install_webc_package( @@ -772,9 +772,9 @@ fn get_all_installed_webc_packages_inner(wasmer_dir: &Path) -> Vec String { /// Returns the checksum of the .webc file, so that we can check whether the /// file is already installed before downloading it pub fn get_remote_webc_checksum(url: &Url) -> Result { - let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8; + let request_max_bytes = webc::v1::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8; let data = get_webc_bytes(url, Some(0..request_max_bytes), None) .with_context(|| anyhow::anyhow!("note: use --registry to change the registry URL"))? .unwrap(); - let checksum = webc::WebC::get_checksum_bytes(&data) + let checksum = webc::v1::WebC::get_checksum_bytes(&data) .map_err(|e| anyhow::anyhow!("{e}"))? .to_vec(); Ok(get_checksum_hash(&checksum)) @@ -826,20 +826,20 @@ pub fn get_remote_webc_checksum(url: &Url) -> Result { /// so we can see if the package has already been installed pub fn get_remote_webc_manifest(url: &Url) -> Result { // Request up unti manifest size / manifest len - let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8; + let request_max_bytes = webc::v1::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8; let data = get_webc_bytes(url, Some(0..request_max_bytes), None)?.unwrap(); - let checksum = webc::WebC::get_checksum_bytes(&data) + let checksum = webc::v1::WebC::get_checksum_bytes(&data) .map_err(|e| anyhow::anyhow!("{e}")) .context("WebC::get_checksum_bytes failed")? .to_vec(); let hex_string = get_checksum_hash(&checksum); - let (manifest_start, manifest_len) = webc::WebC::get_manifest_offset_size(&data) + let (manifest_start, manifest_len) = webc::v1::WebC::get_manifest_offset_size(&data) .map_err(|e| anyhow::anyhow!("{e}")) .context("WebC::get_manifest_offset_size failed")?; let data_with_manifest = get_webc_bytes(url, Some(0..manifest_start + manifest_len), None)?.unwrap(); - let manifest = webc::WebC::get_manifest(&data_with_manifest) + let manifest = webc::v1::WebC::get_manifest(&data_with_manifest) .map_err(|e| anyhow::anyhow!("{e}")) .context("WebC::get_manifest failed")?; Ok(RemoteWebcInfo { diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 1cef239e0eb..55831a00e33 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -12,7 +12,7 @@ thiserror = "1" tracing = { version = "0.1" } typetag = { version = "0.1", optional = true } serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } -webc = { version = "4.0.0", optional = true } +webc = { version = "5.0.0-rc.5", optional = true } slab = { version = "0.4" } derivative = "2.2.0" anyhow = { version = "1.0.66", optional = true } diff --git a/lib/vfs/src/static_fs.rs b/lib/vfs/src/static_fs.rs index f33199cfb87..fc18b5c9951 100644 --- a/lib/vfs/src/static_fs.rs +++ b/lib/vfs/src/static_fs.rs @@ -13,19 +13,22 @@ use crate::mem_fs::FileSystem as MemFileSystem; use crate::{ FileOpener, FileSystem, FsError, Metadata, OpenOptions, OpenOptionsConfig, ReadDir, VirtualFile, }; -use webc::{FsEntry, FsEntryType, OwnedFsEntryFile}; +use webc::{ + metadata::IndexMap, + v1::{FsEntry, FsEntryType, OwnedFsEntryFile}, +}; /// Custom file system wrapper to map requested file paths #[derive(Debug)] pub struct StaticFileSystem { pub package: String, - pub volumes: Arc>>, + pub volumes: Arc>>, pub memory: Arc, } impl StaticFileSystem { pub fn init(bytes: &'static [u8], package: &str) -> Option { - let volumes = Arc::new(webc::WebC::parse_volumes_from_fileblock(bytes).ok()?); + let volumes = Arc::new(webc::v1::WebC::parse_volumes_from_fileblock(bytes).ok()?); let fs = Self { package: package.to_string(), volumes: volumes.clone(), @@ -90,7 +93,7 @@ impl FileOpener for StaticFileSystem { #[derive(Debug)] pub struct WebCFile { - pub volumes: Arc>>, + pub volumes: Arc>>, pub package: String, pub volume: String, pub path: PathBuf, diff --git a/lib/vfs/src/webc_fs.rs b/lib/vfs/src/webc_fs.rs index 441e382be71..11db43c7eff 100644 --- a/lib/vfs/src/webc_fs.rs +++ b/lib/vfs/src/webc_fs.rs @@ -12,7 +12,7 @@ use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; use tokio::io::{AsyncRead, AsyncSeek, AsyncWrite}; -use webc::{FsEntry, FsEntryType, OwnedFsEntryFile, WebC}; +use webc::v1::{FsEntry, FsEntryType, OwnedFsEntryFile, WebC}; /// Custom file system wrapper to map requested file paths #[derive(Debug)] @@ -23,7 +23,7 @@ where pub webc: Arc, pub memory: Arc, top_level_dirs: Vec, - volumes: Vec>, + volumes: Vec>, } impl WebcFileSystem diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index bcead472aba..fae1d7da61d 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -29,7 +29,7 @@ bincode = { version = "1.3" } chrono = { version = "^0.4", default-features = false, features = [ "wasmbind", "std", "clock" ], optional = true } derivative = { version = "^2" } bytes = "1" -webc = { version = "4.0.0", default-features = false, features = ["std"] } +webc = { version = "5.0.0-rc.5", default-features = false } serde_cbor = { version = "0.11.2", optional = true } anyhow = { version = "1.0.66" } lazy_static = "1.4" @@ -59,6 +59,10 @@ wai-bindgen-wasmer = { path = "../wai-bindgen-wasmer", version = "0.2.3", featur heapless = "0.7.16" once_cell = "1.17.0" pin-project = "1.0.12" +# Used by the WCGI runner +hyper = { version = "0.14", features = ["server", "stream"], optional = true } +wcgi = { version = "0.1.1", optional = true } +wcgi-host = { version = "0.1.0", optional = true } [dependencies.reqwest] version = "0.11" @@ -94,6 +98,7 @@ time = ["tokio/time"] webc_runner = ["serde_cbor", "wasmer/compiler"] webc_runner_rt_emscripten = ["wasmer-emscripten"] +wcgi_runner = ["hyper", "wcgi", "wcgi-host"] webc_runner_rt_wasi = [] sys = ["wasmer/sys", "wasmer-wasi-types/sys", "webc/mmap", "wasmer-vm", "time"] @@ -123,3 +128,4 @@ enable-serde = [ "wasmer-vfs/enable-serde", "wasmer-wasi-types/enable-serde", ] + diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs new file mode 100644 index 00000000000..fb39fe413c9 --- /dev/null +++ b/lib/wasi/src/runners/container.rs @@ -0,0 +1,10 @@ +use webc::{v1::WebCOwned, v2::read::OwnedReader}; + +pub struct WebcContainer { + inner: Container, +} + +enum Container { + V1(WebCOwned), + V2(OwnedReader), +} diff --git a/lib/wasi/src/runners/emscripten.rs b/lib/wasi/src/runners/emscripten.rs index 168f109c5c9..ab39348d18b 100644 --- a/lib/wasi/src/runners/emscripten.rs +++ b/lib/wasi/src/runners/emscripten.rs @@ -11,7 +11,7 @@ use wasmer_emscripten::{ generate_emscripten_env, is_emscripten_module, run_emscripten_instance, EmEnv, EmscriptenGlobals, }; -use webc::{Command, WebCMmap}; +use webc::{metadata::Command, v1::WebCMmap}; #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct EmscriptenRunner { diff --git a/lib/wasi/src/runners/mod.rs b/lib/wasi/src/runners/mod.rs index 4965bbd6415..394fc2babe0 100644 --- a/lib/wasi/src/runners/mod.rs +++ b/lib/wasi/src/runners/mod.rs @@ -3,7 +3,8 @@ use std::error::Error as StdError; use std::path::PathBuf; use std::sync::Arc; -use webc::*; + +use webc::v1::{Command, WebC, WebCMmap}; pub mod emscripten; pub mod wasi; @@ -16,7 +17,7 @@ pub struct WapmContainer { } impl core::ops::Deref for WapmContainer { - type Target = webc::WebC<'static>; + type Target = WebC<'static>; fn deref<'a>(&'a self) -> &WebC<'static> { &self.webc.webc } @@ -26,11 +27,11 @@ impl core::ops::Deref for WapmContainer { #[derive(Debug, PartialEq, Eq, Clone)] pub enum WebcParseError { /// Parse error - Parse(webc::Error), + Parse(webc::v1::Error), } -impl From for WebcParseError { - fn from(e: webc::Error) -> Self { +impl From for WebcParseError { + fn from(e: webc::v1::Error) -> Self { WebcParseError::Parse(e) } } @@ -39,7 +40,7 @@ impl WapmContainer { /// Parses a .webc container file. Since .webc files /// can be very large, only file paths are allowed. pub fn new(path: PathBuf) -> std::result::Result { - let webc = webc::WebCMmap::parse(path, &webc::ParseOptions::default())?; + let webc = webc::v1::WebCMmap::parse(path, &webc::v1::ParseOptions::default())?; Ok(Self { webc: Arc::new(webc), }) @@ -107,7 +108,7 @@ impl Bindings for WitBindings { container: &WapmContainer, value: &serde_cbor::Value, ) -> Result { - let value: webc::BindingsExtended = + let value: webc::metadata::BindingsExtended = serde_cbor::from_slice(&serde_cbor::to_vec(value).unwrap()) .map_err(|e| format!("could not parse WitBindings annotations: {e}"))?; @@ -161,7 +162,7 @@ pub trait Runner { Some(s) => s, None => { let path = format!("{}", container.webc.path.display()); - return Err(Box::new(webc::Error(format!( + return Err(Box::new(webc::v1::Error(format!( "Cannot run {path:?}: not executable (no entrypoint in manifest)" )))); } @@ -190,13 +191,13 @@ pub trait Runner { match self.can_run_command(cmd, command_to_exec) { Ok(true) => {} Ok(false) => { - return Err(Box::new(webc::Error(format!( + return Err(Box::new(webc::v1::Error(format!( "Cannot run command {cmd:?} with runner {:?}", command_to_exec.runner )))); } Err(e) => { - return Err(Box::new(webc::Error(format!( + return Err(Box::new(webc::v1::Error(format!( "Cannot run command {cmd:?} with runner {:?}: {e}", command_to_exec.runner )))); diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 07cc0c911c6..3d864951e62 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -8,7 +8,7 @@ use std::error::Error as StdError; use std::sync::Arc; use wasmer::{Module, Store}; use wasmer_vfs::webc_fs::WebcFileSystem; -use webc::{Command, WebCMmap}; +use webc::{metadata::Command, v1::WebCMmap}; #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct WasiRunner { diff --git a/lib/wasi/src/wapm/mod.rs b/lib/wasi/src/wapm/mod.rs index 4adaa99c04f..721ca216cc2 100644 --- a/lib/wasi/src/wapm/mod.rs +++ b/lib/wasi/src/wapm/mod.rs @@ -9,7 +9,10 @@ use wasmer_vfs::FileSystem; use tracing::*; #[allow(unused_imports)] use tracing::{error, warn}; -use webc::{Annotation, UrlOrManifest, WebC}; +use webc::{ + metadata::{Annotation, UrlOrManifest}, + v1::WebC, +}; use crate::{ bin_factory::{BinaryPackage, BinaryPackageCommand}, @@ -125,8 +128,8 @@ fn wapm_extract_version(data: &WapmWebQuery) -> Option } pub fn parse_static_webc(data: Vec) -> Result { - let options = webc::ParseOptions::default(); - match webc::WebCOwned::parse(data, &options) { + let options = webc::v1::ParseOptions::default(); + match webc::v1::WebCOwned::parse(data, &options) { Ok(webc) => unsafe { let webc = Arc::new(webc); return parse_webc(webc.as_webc_ref(), webc.clone()) @@ -164,14 +167,14 @@ async fn download_webc( }; // build the parse options - let options = webc::ParseOptions::default(); + let options = webc::v1::ParseOptions::default(); // fast path let path = compute_path(cache_dir, name); #[cfg(feature = "sys")] if path.exists() { - match webc::WebCMmap::parse(path.clone(), &options) { + match webc::v1::WebCMmap::parse(path.clone(), &options) { Ok(webc) => unsafe { let webc = Arc::new(webc); return parse_webc(webc.as_webc_ref(), webc.clone()).with_context(|| { @@ -230,7 +233,7 @@ async fn download_webc( ); } - match webc::WebCMmap::parse(path.clone(), &options) { + match webc::v1::WebCMmap::parse(path.clone(), &options) { Ok(webc) => unsafe { let webc = Arc::new(webc); return parse_webc(webc.as_webc_ref(), webc.clone()) @@ -242,7 +245,7 @@ async fn download_webc( } } - let webc_raw = webc::WebCOwned::parse(data, &options) + let webc_raw = webc::v1::WebCOwned::parse(data, &options) .with_context(|| format!("Failed to parse downloaded from '{pirita_download_url}'"))?; let webc = Arc::new(webc_raw); // FIXME: add SAFETY comment @@ -275,7 +278,7 @@ async fn download_package( } // TODO: should return Result<_, anyhow::Error> -unsafe fn parse_webc<'a, T>(webc: webc::WebC<'a>, ownership: Arc) -> Option +unsafe fn parse_webc<'a, T>(webc: webc::v1::WebC<'a>, ownership: Arc) -> Option where T: std::fmt::Debug + Send + Sync + 'static, T: Deref>, diff --git a/lib/wcgi-runner/Cargo.toml b/lib/wcgi-runner/Cargo.toml index ba2042cf861..56517746845 100644 --- a/lib/wcgi-runner/Cargo.toml +++ b/lib/wcgi-runner/Cargo.toml @@ -27,7 +27,7 @@ wasmer-vfs = { version = "3.2.0-alpha.1", path = "../vfs", default-features = fa wasmer-wasi = { version = "3.2.0-alpha.1", path = "../wasi", default-features = false, features = ["sys-default"] } wcgi = { version = "0.1.1" } wcgi-host = { version = "0.1.0" } -webc = { version = "5.0.0-rc.1", default-features = false } +webc = { version = "5.0.0-rc.5", default-features = false } [dev-dependencies] anyhow = { version = "1", features = ["backtrace"] } From e3aca2f192e55f7cdc602e038fd09d238ed8ca96 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 24 Feb 2023 23:09:57 +0800 Subject: [PATCH 04/26] Moved the WapmContainer into its own module so we can abstract over WEBC v1 --- lib/wasi/src/runners/container.rs | 130 +++++++++++++++++- lib/wasi/src/runners/emscripten.rs | 5 +- lib/wasi/src/runners/mod.rs | 210 +---------------------------- lib/wasi/src/runners/runner.rs | 79 +++++++++++ lib/wasi/src/runners/wasi.rs | 3 +- 5 files changed, 214 insertions(+), 213 deletions(-) create mode 100644 lib/wasi/src/runners/runner.rs diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index fb39fe413c9..b33053c4695 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -1,10 +1,128 @@ -use webc::{v1::WebCOwned, v2::read::OwnedReader}; +use std::{path::PathBuf, sync::Arc}; -pub struct WebcContainer { - inner: Container, +use webc::{metadata::Manifest, v1::WebCMmap}; + +/// Parsed WAPM file, memory-mapped to an on-disk path +#[derive(Debug, Clone)] +pub struct WapmContainer { + /// WebC container + webc: Arc, +} + +impl WapmContainer { + /// Parses a .webc container file. Since .webc files + /// can be very large, only file paths are allowed. + pub fn new(path: PathBuf) -> std::result::Result { + let webc = webc::v1::WebCMmap::parse(path, &webc::v1::ParseOptions::default())?; + Ok(Self { + webc: Arc::new(webc), + }) + } + + /// Returns the bytes of a file or a stringified error + pub fn get_file<'b>(&'b self, path: &str) -> Result<&'b [u8], String> { + self.webc + .get_file(&self.webc.get_package_name(), path) + .map_err(|e| e.0) + } + + /// Returns a list of volumes in this container + pub fn get_volumes(&self) -> Vec { + self.webc.volumes.keys().cloned().collect::>() + } + + /// Lookup .wit bindings by name and parse them + pub fn get_bindings( + &self, + bindings: &str, + ) -> std::result::Result { + let bindings = self + .webc + .manifest + .bindings + .iter() + .find(|b| b.name == bindings) + .ok_or_else(|| ParseBindingsError::NoBindings(bindings.to_string()))?; + + T::parse_bindings(self, &bindings.annotations).map_err(ParseBindingsError::ParseBindings) + } + + pub fn manifest(&self) -> &Manifest { + &self.webc.manifest + } + + pub fn v1(&self) -> &Arc { + &self.webc + } +} + +/// Error that happened while parsing .wit bindings +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)] +pub enum ParseBindingsError { + /// No bindings are available for the given lookup key + NoBindings(String), + /// Error happened during parsing + ParseBindings(String), +} + +/// Trait to parse bindings (any kind of bindings) for container .wasm files (usually .wit format) +pub trait Bindings { + /// Function that takes annotations in a free-form `Value` struct and returns the parsed bindings or an error + fn parse_bindings(_: &WapmContainer, _: &serde_cbor::Value) -> Result + where + Self: Sized; +} + +/// WIT bindings +#[derive(Default, Debug, Copy, Clone)] +pub struct WitBindings {} + +impl WitBindings { + /// Unused: creates default wit bindings + pub fn parse(_s: &str) -> Result { + Ok(Self::default()) + } +} + +impl Bindings for WitBindings { + fn parse_bindings( + container: &WapmContainer, + value: &serde_cbor::Value, + ) -> Result { + let value: webc::metadata::BindingsExtended = + serde_cbor::from_slice(&serde_cbor::to_vec(value).unwrap()) + .map_err(|e| format!("could not parse WitBindings annotations: {e}"))?; + + let mut wit_bindgen_filepath = value.exports().unwrap_or_default().to_string(); + + for v in container.get_volumes() { + let schema = format!("{v}://"); + if wit_bindgen_filepath.starts_with(&schema) { + wit_bindgen_filepath = wit_bindgen_filepath.replacen(&schema, "", 1); + break; + } + } + + let wit_bindings = container + .get_file(&wit_bindgen_filepath) + .map_err(|e| format!("could not get WitBindings file {wit_bindgen_filepath:?}: {e}"))?; + + let wit_bindings_str = std::str::from_utf8(wit_bindings) + .map_err(|e| format!("could not get WitBindings file {wit_bindgen_filepath:?}: {e}"))?; + + Self::parse(wit_bindings_str) + } +} + +/// Error that ocurred while parsing the .webc file +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum WebcParseError { + /// Parse error + Parse(webc::v1::Error), } -enum Container { - V1(WebCOwned), - V2(OwnedReader), +impl From for WebcParseError { + fn from(e: webc::v1::Error) -> Self { + WebcParseError::Parse(e) + } } diff --git a/lib/wasi/src/runners/emscripten.rs b/lib/wasi/src/runners/emscripten.rs index ab39348d18b..8a9885c6d37 100644 --- a/lib/wasi/src/runners/emscripten.rs +++ b/lib/wasi/src/runners/emscripten.rs @@ -62,6 +62,7 @@ impl crate::runners::Runner for EmscriptenRunner { _command: &Command, container: &WapmContainer, ) -> Result> { + let container = container.v1(); let atom_name = container.get_atom_name_for_command("emscripten", command_name)?; let main_args = container.get_main_args_for_command(command_name); let atom_bytes = container.get_atom(&container.get_package_name(), &atom_name)?; @@ -70,14 +71,14 @@ impl crate::runners::Runner for EmscriptenRunner { module.set_name(&atom_name); let (mut globals, env) = - prepare_emscripten_env(&mut self.store, &module, container.webc.clone(), &atom_name)?; + prepare_emscripten_env(&mut self.store, &module, container.clone(), &atom_name)?; exec_module( &mut self.store, &module, &mut globals, env, - container.webc.clone(), + container.clone(), &atom_name, main_args.unwrap_or_default(), )?; diff --git a/lib/wasi/src/runners/mod.rs b/lib/wasi/src/runners/mod.rs index 394fc2babe0..1ab2bd87676 100644 --- a/lib/wasi/src/runners/mod.rs +++ b/lib/wasi/src/runners/mod.rs @@ -1,209 +1,11 @@ #![cfg(feature = "webc_runner")] -use std::error::Error as StdError; -use std::path::PathBuf; -use std::sync::Arc; - -use webc::v1::{Command, WebC, WebCMmap}; - +mod container; pub mod emscripten; +mod runner; pub mod wasi; -/// Parsed WAPM file, memory-mapped to an on-disk path -#[derive(Debug, Clone)] -pub struct WapmContainer { - /// WebC container - pub webc: Arc, -} - -impl core::ops::Deref for WapmContainer { - type Target = WebC<'static>; - fn deref<'a>(&'a self) -> &WebC<'static> { - &self.webc.webc - } -} - -/// Error that ocurred while parsing the .webc file -#[derive(Debug, PartialEq, Eq, Clone)] -pub enum WebcParseError { - /// Parse error - Parse(webc::v1::Error), -} - -impl From for WebcParseError { - fn from(e: webc::v1::Error) -> Self { - WebcParseError::Parse(e) - } -} - -impl WapmContainer { - /// Parses a .webc container file. Since .webc files - /// can be very large, only file paths are allowed. - pub fn new(path: PathBuf) -> std::result::Result { - let webc = webc::v1::WebCMmap::parse(path, &webc::v1::ParseOptions::default())?; - Ok(Self { - webc: Arc::new(webc), - }) - } - - /// Returns the bytes of a file or a stringified error - pub fn get_file<'b>(&'b self, path: &str) -> Result<&'b [u8], String> { - self.webc - .get_file(&self.webc.get_package_name(), path) - .map_err(|e| e.0) - } - - /// Returns a list of volumes in this container - pub fn get_volumes(&self) -> Vec { - self.webc.volumes.keys().cloned().collect::>() - } - - /// Lookup .wit bindings by name and parse them - pub fn get_bindings( - &self, - bindings: &str, - ) -> std::result::Result { - let bindings = self - .webc - .manifest - .bindings - .iter() - .find(|b| b.name == bindings) - .ok_or_else(|| ParseBindingsError::NoBindings(bindings.to_string()))?; - - T::parse_bindings(self, &bindings.annotations).map_err(ParseBindingsError::ParseBindings) - } -} - -/// Error that happened while parsing .wit bindings -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd)] -pub enum ParseBindingsError { - /// No bindings are available for the given lookup key - NoBindings(String), - /// Error happened during parsing - ParseBindings(String), -} - -/// Trait to parse bindings (any kind of bindings) for container .wasm files (usually .wit format) -pub trait Bindings { - /// Function that takes annotations in a free-form `Value` struct and returns the parsed bindings or an error - fn parse_bindings(_: &WapmContainer, _: &serde_cbor::Value) -> Result - where - Self: Sized; -} - -/// WIT bindings -#[derive(Default, Debug, Copy, Clone)] -pub struct WitBindings {} - -impl WitBindings { - /// Unused: creates default wit bindings - pub fn parse(_s: &str) -> Result { - Ok(Self::default()) - } -} - -impl Bindings for WitBindings { - fn parse_bindings( - container: &WapmContainer, - value: &serde_cbor::Value, - ) -> Result { - let value: webc::metadata::BindingsExtended = - serde_cbor::from_slice(&serde_cbor::to_vec(value).unwrap()) - .map_err(|e| format!("could not parse WitBindings annotations: {e}"))?; - - let mut wit_bindgen_filepath = value.exports().unwrap_or_default().to_string(); - - for v in container.get_volumes() { - let schema = format!("{v}://"); - if wit_bindgen_filepath.starts_with(&schema) { - wit_bindgen_filepath = wit_bindgen_filepath.replacen(&schema, "", 1); - break; - } - } - - let wit_bindings = container - .get_file(&wit_bindgen_filepath) - .map_err(|e| format!("could not get WitBindings file {wit_bindgen_filepath:?}: {e}"))?; - - let wit_bindings_str = std::str::from_utf8(wit_bindings) - .map_err(|e| format!("could not get WitBindings file {wit_bindgen_filepath:?}: {e}"))?; - - Self::parse(wit_bindings_str) - } -} - -/// Trait that all runners have to implement -pub trait Runner { - /// The return value of the output of the runner - type Output; - - /// Returns whether the Runner will be able to run the `Command` - fn can_run_command( - &self, - command_name: &str, - command: &Command, - ) -> Result>; - - /// Implementation to run the given command - /// - /// - use `cmd.annotations` to get the metadata for the given command - /// - use `container.get_atom()` to get the - fn run_command( - &mut self, - command_name: &str, - cmd: &Command, - container: &WapmContainer, - ) -> Result>; - - /// Runs the container if the container has an `entrypoint` in the manifest - fn run(&mut self, container: &WapmContainer) -> Result> { - let cmd = match container.webc.webc.manifest.entrypoint.as_ref() { - Some(s) => s, - None => { - let path = format!("{}", container.webc.path.display()); - return Err(Box::new(webc::v1::Error(format!( - "Cannot run {path:?}: not executable (no entrypoint in manifest)" - )))); - } - }; - - self.run_cmd(container, cmd) - } - - /// Runs the given `cmd` on the container - fn run_cmd( - &mut self, - container: &WapmContainer, - cmd: &str, - ) -> Result> { - let path = format!("{}", container.webc.path.display()); - let command_to_exec = container - .webc - .webc - .manifest - .commands - .get(cmd) - .ok_or_else(|| anyhow::anyhow!("{path}: command {cmd:?} not found in manifest"))?; - - let _path = format!("{}", container.webc.path.display()); - - match self.can_run_command(cmd, command_to_exec) { - Ok(true) => {} - Ok(false) => { - return Err(Box::new(webc::v1::Error(format!( - "Cannot run command {cmd:?} with runner {:?}", - command_to_exec.runner - )))); - } - Err(e) => { - return Err(Box::new(webc::v1::Error(format!( - "Cannot run command {cmd:?} with runner {:?}: {e}", - command_to_exec.runner - )))); - } - } - - self.run_command(cmd, command_to_exec, container) - } -} +pub use self::{ + container::{Bindings, WapmContainer, WebcParseError, WitBindings}, + runner::Runner, +}; diff --git a/lib/wasi/src/runners/runner.rs b/lib/wasi/src/runners/runner.rs new file mode 100644 index 00000000000..6f006c5dbd2 --- /dev/null +++ b/lib/wasi/src/runners/runner.rs @@ -0,0 +1,79 @@ +use std::error::Error as StdError; + +use webc::v1::Command; + +use crate::runners::WapmContainer; + +/// Trait that all runners have to implement +pub trait Runner { + /// The return value of the output of the runner + type Output; + + /// Returns whether the Runner will be able to run the `Command` + fn can_run_command( + &self, + command_name: &str, + command: &Command, + ) -> Result>; + + /// Implementation to run the given command + /// + /// - use `cmd.annotations` to get the metadata for the given command + /// - use `container.get_atom()` to get the + fn run_command( + &mut self, + command_name: &str, + cmd: &Command, + container: &WapmContainer, + ) -> Result>; + + /// Runs the container if the container has an `entrypoint` in the manifest + fn run(&mut self, container: &WapmContainer) -> Result> { + let cmd = match container.manifest().entrypoint.as_ref() { + Some(s) => s, + None => { + let path = format!("{}", container.v1().path.display()); + return Err(Box::new(webc::v1::Error(format!( + "Cannot run {path:?}: not executable (no entrypoint in manifest)" + )))); + } + }; + + self.run_cmd(container, cmd) + } + + /// Runs the given `cmd` on the container + fn run_cmd( + &mut self, + container: &WapmContainer, + cmd: &str, + ) -> Result> { + let webc = container.v1(); + let path = format!("{}", webc.path.display()); + let command_to_exec = webc + .manifest + .commands + .get(cmd) + .ok_or_else(|| anyhow::anyhow!("{path}: command {cmd:?} not found in manifest"))?; + + let _path = format!("{}", webc.path.display()); + + match self.can_run_command(cmd, command_to_exec) { + Ok(true) => {} + Ok(false) => { + return Err(Box::new(webc::v1::Error(format!( + "Cannot run command {cmd:?} with runner {:?}", + command_to_exec.runner + )))); + } + Err(e) => { + return Err(Box::new(webc::v1::Error(format!( + "Cannot run command {cmd:?} with runner {:?}: {e}", + command_to_exec.runner + )))); + } + } + + self.run_command(cmd, command_to_exec, container) + } +} diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 3d864951e62..8996a689f56 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -61,13 +61,14 @@ impl crate::runners::Runner for WasiRunner { _command: &Command, container: &WapmContainer, ) -> Result> { + let container = container.v1(); let atom_name = container.get_atom_name_for_command("wasi", command_name)?; let atom_bytes = container.get_atom(&container.get_package_name(), &atom_name)?; let mut module = Module::new(&self.store, atom_bytes)?; module.set_name(&atom_name); - let builder = prepare_webc_env(container.webc.clone(), &atom_name, &self.args)?; + let builder = prepare_webc_env(container.clone(), &atom_name, &self.args)?; let init = builder.build_init()?; From 7303fabf35e5df38d326538d8dad5dcb2dc78014 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Mon, 27 Feb 2023 19:34:22 +0800 Subject: [PATCH 05/26] Started adding a WcgiRunner implementation to the wasmer-wasi crate --- Cargo.lock | 37 ++--------- lib/c-api/Cargo.toml | 2 +- lib/cli/Cargo.toml | 2 +- lib/cli/src/commands/run.rs | 68 ++++++++----------- lib/registry/Cargo.toml | 6 +- lib/vfs/Cargo.toml | 6 +- lib/vfs/src/static_fs.rs | 6 +- lib/wasi/Cargo.toml | 4 +- lib/wasi/src/lib.rs | 1 + lib/wasi/src/runners/container.rs | 4 ++ lib/wasi/src/runners/emscripten.rs | 16 +++-- lib/wasi/src/runners/mod.rs | 9 ++- lib/wasi/src/runners/runner.rs | 33 ++++------ lib/wasi/src/runners/wasi.rs | 15 ++--- lib/wasi/src/runners/wcgi.rs | 101 +++++++++++++++++++++++++++++ 15 files changed, 183 insertions(+), 127 deletions(-) create mode 100644 lib/wasi/src/runners/wcgi.rs diff --git a/Cargo.lock b/Cargo.lock index 7b2ab33c386..987dedc7776 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4865,7 +4865,7 @@ dependencies = [ "wasmer-types", "wasmer-vfs", "wasmer-wasi", - "webc 5.0.0-rc.5", + "webc", ] [[package]] @@ -4970,7 +4970,7 @@ dependencies = [ "wasmer-wasm-interface", "wasmer-wast", "wasmparser 0.51.4", - "webc 5.0.0-rc.5", + "webc", ] [[package]] @@ -5206,7 +5206,7 @@ dependencies = [ "toml", "url", "wasmer-toml", - "webc 4.1.1", + "webc", "whoami", ] @@ -5267,6 +5267,7 @@ dependencies = [ "derivative", "filetime", "fs_extra", + "indexmap", "lazy_static", "libc", "pin-project-lite", @@ -5276,7 +5277,7 @@ dependencies = [ "tokio", "tracing", "typetag", - "webc 5.0.0-rc.5", + "webc", ] [[package]] @@ -5370,7 +5371,7 @@ dependencies = [ "wasmer-wasi-types", "wcgi", "wcgi-host", - "webc 5.0.0-rc.5", + "webc", "weezl", "winapi", ] @@ -5687,7 +5688,7 @@ dependencies = [ "wasmer-wasi", "wcgi", "wcgi-host", - "webc 5.0.0-rc.5", + "webc", ] [[package]] @@ -5700,30 +5701,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webc" -version = "4.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4af0f2f0b6bf2e4366375c3cd6635aef1a2eb66cd02454d79525aa7eecf0751" -dependencies = [ - "anyhow", - "base64 0.13.1", - "byteorder", - "indexmap", - "leb128", - "lexical-sort", - "memchr", - "memmap2", - "path-clean", - "rand", - "serde", - "serde_cbor", - "serde_json", - "sha2", - "url", - "walkdir", -] - [[package]] name = "webc" version = "5.0.0-rc.5" diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index 04413d379ac..3760dfbc2cf 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -92,7 +92,7 @@ wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] static-artifact-load = ["wasmer-compiler/static-artifact-load"] static-artifact-create = ["wasmer-compiler/static-artifact-create"] -webc_runner = ["wasmer-wasi/webc_runner", "wasmer-vfs", "webc"] +webc_runner = ["wasmer-wasi/webc_runner", "webc"] # Deprecated features. jit = ["compiler"] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 4fe65e42d8d..2d72e3890ff 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -118,7 +118,7 @@ wast = ["wasmer-wast"] wasi = ["wasmer-wasi", "wasmer-wasi-local-networking"] emscripten = ["wasmer-emscripten"] wat = ["wasmer/wat"] -webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "nuke-dir", "webc"] +webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "wasmer-wasi/webc_runner_rt_wcgi", "nuke-dir", "webc"] compiler = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 1889fc64f73..0004245716d 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -225,11 +225,7 @@ impl RunWithPathBuf { { if let Ok(pf) = WapmContainer::new(self.path.clone()) { return self - .run_container( - pf, - &self.command_name.clone().unwrap_or_default(), - &self.args, - ) + .run_container(pf, self.command_name.as_deref(), &self.args) .map_err(|e| anyhow!("Could not run PiritaFile: {e}")); } } @@ -376,48 +372,38 @@ impl RunWithPathBuf { fn run_container( &self, container: WapmContainer, - id: &str, + id: Option<&str>, args: &[String], ) -> Result<(), anyhow::Error> { - let mut result = None; - - #[cfg(feature = "wasi")] - { - if let Some(r) = result { - return r; - } - - let (store, _compiler_type) = self.store.get_store()?; - let mut runner = wasmer_wasi::runners::wasi::WasiRunner::new(store); - runner.set_args(args.to_vec()); - result = Some(if id.is_empty() { - runner.run(&container).map_err(|e| anyhow::anyhow!("{e}")) - } else { - runner - .run_cmd(&container, id) - .map_err(|e| anyhow::anyhow!("{e}")) - }); + let id = id + .or_else(|| container.manifest().entrypoint.as_deref()) + .context("No command specified")?; + + let command = container + .manifest() + .commands + .get(id) + .with_context(|| format!("No metadata found for the command, \"{id}\""))?; + + let (store, _compiler_type) = self.store.get_store()?; + let mut runner = wasmer_wasi::runners::wasi::WasiRunner::new(store); + runner.set_args(args.to_vec()); + if runner.can_run_command(id, command).unwrap_or(false) { + runner + .run_cmd(&container, id) + .context("WASI runner failed")?; } - #[cfg(feature = "emscripten")] - { - if let Some(r) = result { - return r; - } - - let (store, _compiler_type) = self.store.get_store()?; - let mut runner = wasmer_wasi::runners::emscripten::EmscriptenRunner::new(store); - runner.set_args(args.to_vec()); - result = Some(if id.is_empty() { - runner.run(&container).map_err(|e| anyhow::anyhow!("{e}")) - } else { - runner - .run_cmd(&container, id) - .map_err(|e| anyhow::anyhow!("{e}")) - }); + let (store, _compiler_type) = self.store.get_store()?; + let mut runner = wasmer_wasi::runners::emscripten::EmscriptenRunner::new(store); + runner.set_args(args.to_vec()); + if runner.can_run_command(id, command).unwrap_or(false) { + runner + .run_cmd(&container, id) + .context("Emscripten runner failed")?; } - result.unwrap_or_else(|| Err(anyhow::anyhow!("neither emscripten or wasi file"))) + anyhow::bail!("No runner"); } fn get_store_module(&self) -> Result<(Store, Module)> { diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index b7147bf1d58..bd2a32d0671 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -12,10 +12,10 @@ rand = "0.8.5" dirs = "4.0.0" graphql_client = "0.11.0" serde = { version = "1.0.145", features = ["derive"] } -anyhow = "1.0.65" +anyhow = "1.0.65" reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json", "stream"] } futures-util = "0.3.25" -whoami = "1.2.3" +whoami = "1.2.3" serde_json = "1.0.85" url = "2.3.1" thiserror = "1.0.37" @@ -25,7 +25,7 @@ tar = "0.4.38" flate2 = "1.0.24" semver = "1.0.14" lzma-rs = "0.2.0" -webc = { version ="4.0.0", features = ["mmap"] } +webc = { version ="5.0.0-rc.5", features = ["mmap"] } hex = "0.4.3" tokio = "1.24.0" log = "0.4.17" diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 55831a00e33..3b0df8c7b85 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -23,6 +23,7 @@ filetime = { version = "0.2.18", optional = true } bytes = "1" tokio = { version = "1", features = [ "io-util", "sync", "macros" ], default_features = false } pin-project-lite = "0.2.9" +indexmap = "1.9.2" [dev-dependencies] tokio = { version = "1", features = [ "io-util", "rt" ], default_features = false } @@ -32,8 +33,5 @@ default = ["host-fs", "webc-fs", "static-fs"] host-fs = ["libc", "fs_extra", "filetime", "tokio/fs", "tokio/io-std"] webc-fs = ["webc", "anyhow"] static-fs = ["webc", "anyhow"] -enable-serde = [ - "serde", - "typetag" -] +enable-serde = ["typetag"] no-time = [] diff --git a/lib/vfs/src/static_fs.rs b/lib/vfs/src/static_fs.rs index fc18b5c9951..48131c70eb6 100644 --- a/lib/vfs/src/static_fs.rs +++ b/lib/vfs/src/static_fs.rs @@ -13,10 +13,8 @@ use crate::mem_fs::FileSystem as MemFileSystem; use crate::{ FileOpener, FileSystem, FsError, Metadata, OpenOptions, OpenOptionsConfig, ReadDir, VirtualFile, }; -use webc::{ - metadata::IndexMap, - v1::{FsEntry, FsEntryType, OwnedFsEntryFile}, -}; +use indexmap::IndexMap; +use webc::v1::{FsEntry, FsEntryType, OwnedFsEntryFile}; /// Custom file system wrapper to map requested file paths #[derive(Debug)] diff --git a/lib/wasi/Cargo.toml b/lib/wasi/Cargo.toml index fae1d7da61d..8d9bad051d9 100644 --- a/lib/wasi/Cargo.toml +++ b/lib/wasi/Cargo.toml @@ -97,9 +97,9 @@ default = ["sys-default"] time = ["tokio/time"] webc_runner = ["serde_cbor", "wasmer/compiler"] -webc_runner_rt_emscripten = ["wasmer-emscripten"] -wcgi_runner = ["hyper", "wcgi", "wcgi-host"] webc_runner_rt_wasi = [] +webc_runner_rt_wcgi = ["hyper", "wcgi", "wcgi-host"] +webc_runner_rt_emscripten = ["wasmer-emscripten"] sys = ["wasmer/sys", "wasmer-wasi-types/sys", "webc/mmap", "wasmer-vm", "time"] sys-default = ["wasmer/wat", "wasmer/compiler", "sys", "logging", "host-fs", "sys-poll", "sys-thread", "host-vnet", "host-threads", "host-reqwest" ] diff --git a/lib/wasi/src/lib.rs b/lib/wasi/src/lib.rs index cb513f3581a..28ad07cd286 100644 --- a/lib/wasi/src/lib.rs +++ b/lib/wasi/src/lib.rs @@ -39,6 +39,7 @@ pub mod net; // TODO: should this be pub? pub mod fs; pub mod http; +#[cfg(feature = "webc_runner")] pub mod runners; pub mod runtime; mod state; diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index b33053c4695..fd77a755d3a 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -31,6 +31,10 @@ impl WapmContainer { self.webc.volumes.keys().cloned().collect::>() } + pub fn get_atom(&self, name: &str) -> Option<&[u8]> { + self.webc.get_atom(&self.webc.get_package_name(), name).ok() + } + /// Lookup .wit bindings by name and parse them pub fn get_bindings( &self, diff --git a/lib/wasi/src/runners/emscripten.rs b/lib/wasi/src/runners/emscripten.rs index 8a9885c6d37..6e9704131d7 100644 --- a/lib/wasi/src/runners/emscripten.rs +++ b/lib/wasi/src/runners/emscripten.rs @@ -1,10 +1,8 @@ -#![cfg(feature = "webc_runner_rt_emscripten")] //! WebC container support for running Emscripten modules use crate::runners::WapmContainer; -use anyhow::anyhow; +use anyhow::{anyhow, Error}; use serde::{Deserialize, Serialize}; -use std::error::Error as StdError; use std::sync::Arc; use wasmer::{FunctionEnv, Instance, Module, Store}; use wasmer_emscripten::{ @@ -49,7 +47,7 @@ impl EmscriptenRunner { impl crate::runners::Runner for EmscriptenRunner { type Output = (); - fn can_run_command(&self, _: &str, command: &Command) -> Result> { + fn can_run_command(&self, _: &str, command: &Command) -> Result { Ok(command .runner .starts_with("https://webc.org/runner/emscripten")) @@ -61,11 +59,15 @@ impl crate::runners::Runner for EmscriptenRunner { command_name: &str, _command: &Command, container: &WapmContainer, - ) -> Result> { + ) -> Result { let container = container.v1(); - let atom_name = container.get_atom_name_for_command("emscripten", command_name)?; + let atom_name = container + .get_atom_name_for_command("emscripten", command_name) + .map_err(Error::msg)?; let main_args = container.get_main_args_for_command(command_name); - let atom_bytes = container.get_atom(&container.get_package_name(), &atom_name)?; + let atom_bytes = container + .get_atom(&container.get_package_name(), &atom_name) + .map_err(Error::msg)?; let mut module = Module::new(&self.store, atom_bytes)?; module.set_name(&atom_name); diff --git a/lib/wasi/src/runners/mod.rs b/lib/wasi/src/runners/mod.rs index 1ab2bd87676..d156753fe93 100644 --- a/lib/wasi/src/runners/mod.rs +++ b/lib/wasi/src/runners/mod.rs @@ -1,9 +1,12 @@ -#![cfg(feature = "webc_runner")] - mod container; -pub mod emscripten; mod runner; + +#[cfg(feature = "webc_runner_rt_emscripten")] +pub mod emscripten; +#[cfg(feature = "webc_runner_rt_wasi")] pub mod wasi; +#[cfg(feature = "webc_runner_rt_wcgi")] +pub mod wcgi; pub use self::{ container::{Bindings, WapmContainer, WebcParseError, WitBindings}, diff --git a/lib/wasi/src/runners/runner.rs b/lib/wasi/src/runners/runner.rs index 6f006c5dbd2..a42ee363409 100644 --- a/lib/wasi/src/runners/runner.rs +++ b/lib/wasi/src/runners/runner.rs @@ -1,6 +1,5 @@ -use std::error::Error as StdError; - -use webc::v1::Command; +use anyhow::Error; +use webc::metadata::Command; use crate::runners::WapmContainer; @@ -10,11 +9,7 @@ pub trait Runner { type Output; /// Returns whether the Runner will be able to run the `Command` - fn can_run_command( - &self, - command_name: &str, - command: &Command, - ) -> Result>; + fn can_run_command(&self, command_name: &str, command: &Command) -> Result; /// Implementation to run the given command /// @@ -25,17 +20,15 @@ pub trait Runner { command_name: &str, cmd: &Command, container: &WapmContainer, - ) -> Result>; + ) -> Result; /// Runs the container if the container has an `entrypoint` in the manifest - fn run(&mut self, container: &WapmContainer) -> Result> { + fn run(&mut self, container: &WapmContainer) -> Result { let cmd = match container.manifest().entrypoint.as_ref() { Some(s) => s, None => { let path = format!("{}", container.v1().path.display()); - return Err(Box::new(webc::v1::Error(format!( - "Cannot run {path:?}: not executable (no entrypoint in manifest)" - )))); + anyhow::bail!("Cannot run {path:?}: not executable (no entrypoint in manifest)"); } }; @@ -43,11 +36,7 @@ pub trait Runner { } /// Runs the given `cmd` on the container - fn run_cmd( - &mut self, - container: &WapmContainer, - cmd: &str, - ) -> Result> { + fn run_cmd(&mut self, container: &WapmContainer, cmd: &str) -> Result { let webc = container.v1(); let path = format!("{}", webc.path.display()); let command_to_exec = webc @@ -61,16 +50,16 @@ pub trait Runner { match self.can_run_command(cmd, command_to_exec) { Ok(true) => {} Ok(false) => { - return Err(Box::new(webc::v1::Error(format!( + anyhow::bail!( "Cannot run command {cmd:?} with runner {:?}", command_to_exec.runner - )))); + ); } Err(e) => { - return Err(Box::new(webc::v1::Error(format!( + anyhow::bail!( "Cannot run command {cmd:?} with runner {:?}: {e}", command_to_exec.runner - )))); + ); } } diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 8996a689f56..b1a529a69b5 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -1,10 +1,9 @@ -#![cfg(feature = "webc_runner_rt_wasi")] //! WebC container support for running WASI modules use crate::runners::WapmContainer; use crate::{WasiEnv, WasiEnvBuilder}; +use anyhow::Error; use serde::{Deserialize, Serialize}; -use std::error::Error as StdError; use std::sync::Arc; use wasmer::{Module, Store}; use wasmer_vfs::webc_fs::WebcFileSystem; @@ -46,11 +45,7 @@ impl WasiRunner { impl crate::runners::Runner for WasiRunner { type Output = (); - fn can_run_command( - &self, - _command_name: &str, - command: &Command, - ) -> Result> { + fn can_run_command(&self, _command_name: &str, command: &Command) -> Result { Ok(command.runner.starts_with("https://webc.org/runner/wasi")) } @@ -60,9 +55,11 @@ impl crate::runners::Runner for WasiRunner { command_name: &str, _command: &Command, container: &WapmContainer, - ) -> Result> { + ) -> Result { let container = container.v1(); - let atom_name = container.get_atom_name_for_command("wasi", command_name)?; + let atom_name = container + .get_atom_name_for_command("wasi", command_name) + .map_err(Error::msg)?; let atom_bytes = container.get_atom(&container.get_package_name(), &atom_name)?; let mut module = Module::new(&self.store, atom_bytes)?; diff --git a/lib/wasi/src/runners/wcgi.rs b/lib/wasi/src/runners/wcgi.rs new file mode 100644 index 00000000000..6bb2c10f219 --- /dev/null +++ b/lib/wasi/src/runners/wcgi.rs @@ -0,0 +1,101 @@ +use anyhow::{Context, Error}; +use wasmer::{Engine, Module, Store}; +use wasmer_vfs::FileSystem; +use webc::metadata::{Command, Manifest}; + +use crate::runners::WapmContainer; + +pub struct WcgiRunner {} + +// TODO(Michael-F-Bryan): When we rewrite the existing runner infrastructure, +// make the "Runner" trait contain just these two methods. +impl WcgiRunner { + fn supports(cmd: &Command) -> Result { + Ok(cmd.runner.starts_with("https://webc.org/runner/wcgi")) + } + + #[tracing::instrument(skip(self, ctx))] + fn run_(&self, command_name: &str, ctx: &RunnerContext<'_>) -> Result<(), Error> { + let wasi: webc::metadata::annotations::Wasi = ctx + .command() + .annotations + .get("wasi") + .cloned() + .and_then(|v| serde_cbor::value::from_value(v).ok()) + .context("Unable to retrieve the WASI metadata")?; + + let atom_name = &wasi.atom; + let atom = ctx + .get_atom(&atom_name) + .with_context(|| format!("Unable to retrieve the \"{atom_name}\" atom"))?; + + let module = ctx.compile(atom).context("Unable to compile the atom")?; + todo!(); + } +} + +// TODO(Michael-F-Bryan): Turn this into an object-safe trait when we rewrite +// the "Runner" trait. +struct RunnerContext<'a> { + container: &'a WapmContainer, + command: &'a Command, + engine: Engine, + store: Store, +} + +#[allow(dead_code)] +impl RunnerContext<'_> { + fn command(&self) -> &Command { + self.command + } + + fn manifest(&self) -> &Manifest { + self.container.manifest() + } + + fn engine(&self) -> &Engine { + &self.engine + } + + fn store(&self) -> &Store { + &self.store + } + + fn volume(&self, _name: &str) -> Option> { + todo!(); + } + + fn get_atom(&self, name: &str) -> Option<&[u8]> { + self.container.get_atom(name) + } + + fn compile(&self, wasm: &[u8]) -> Result { + // TODO: wire this up to wasmer-cache + Module::new(&self.engine, wasm).map_err(Error::from) + } +} + +impl crate::runners::Runner for WcgiRunner { + type Output = (); + + fn can_run_command(&self, _: &str, command: &Command) -> Result { + WcgiRunner::supports(command) + } + + fn run_command( + &mut self, + command_name: &str, + command: &Command, + container: &WapmContainer, + ) -> Result { + let store = Store::default(); + let ctx = RunnerContext { + container, + command, + engine: store.engine().clone(), + store, + }; + + self.run_(command_name, &ctx) + } +} From 7c8c67deb2070f50e4d6ec751c4cbd9b6ddc47d9 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 28 Feb 2023 00:40:46 +0800 Subject: [PATCH 06/26] Copied the original WCGI runner code into the wasmer_wasi::runner format --- lib/wasi/src/runners/wcgi.rs | 101 ---------- lib/wasi/src/runners/wcgi/handler.rs | 244 ++++++++++++++++++++++++ lib/wasi/src/runners/wcgi/mod.rs | 12 ++ lib/wasi/src/runners/wcgi/runner.rs | 275 +++++++++++++++++++++++++++ 4 files changed, 531 insertions(+), 101 deletions(-) delete mode 100644 lib/wasi/src/runners/wcgi.rs create mode 100644 lib/wasi/src/runners/wcgi/handler.rs create mode 100644 lib/wasi/src/runners/wcgi/mod.rs create mode 100644 lib/wasi/src/runners/wcgi/runner.rs diff --git a/lib/wasi/src/runners/wcgi.rs b/lib/wasi/src/runners/wcgi.rs deleted file mode 100644 index 6bb2c10f219..00000000000 --- a/lib/wasi/src/runners/wcgi.rs +++ /dev/null @@ -1,101 +0,0 @@ -use anyhow::{Context, Error}; -use wasmer::{Engine, Module, Store}; -use wasmer_vfs::FileSystem; -use webc::metadata::{Command, Manifest}; - -use crate::runners::WapmContainer; - -pub struct WcgiRunner {} - -// TODO(Michael-F-Bryan): When we rewrite the existing runner infrastructure, -// make the "Runner" trait contain just these two methods. -impl WcgiRunner { - fn supports(cmd: &Command) -> Result { - Ok(cmd.runner.starts_with("https://webc.org/runner/wcgi")) - } - - #[tracing::instrument(skip(self, ctx))] - fn run_(&self, command_name: &str, ctx: &RunnerContext<'_>) -> Result<(), Error> { - let wasi: webc::metadata::annotations::Wasi = ctx - .command() - .annotations - .get("wasi") - .cloned() - .and_then(|v| serde_cbor::value::from_value(v).ok()) - .context("Unable to retrieve the WASI metadata")?; - - let atom_name = &wasi.atom; - let atom = ctx - .get_atom(&atom_name) - .with_context(|| format!("Unable to retrieve the \"{atom_name}\" atom"))?; - - let module = ctx.compile(atom).context("Unable to compile the atom")?; - todo!(); - } -} - -// TODO(Michael-F-Bryan): Turn this into an object-safe trait when we rewrite -// the "Runner" trait. -struct RunnerContext<'a> { - container: &'a WapmContainer, - command: &'a Command, - engine: Engine, - store: Store, -} - -#[allow(dead_code)] -impl RunnerContext<'_> { - fn command(&self) -> &Command { - self.command - } - - fn manifest(&self) -> &Manifest { - self.container.manifest() - } - - fn engine(&self) -> &Engine { - &self.engine - } - - fn store(&self) -> &Store { - &self.store - } - - fn volume(&self, _name: &str) -> Option> { - todo!(); - } - - fn get_atom(&self, name: &str) -> Option<&[u8]> { - self.container.get_atom(name) - } - - fn compile(&self, wasm: &[u8]) -> Result { - // TODO: wire this up to wasmer-cache - Module::new(&self.engine, wasm).map_err(Error::from) - } -} - -impl crate::runners::Runner for WcgiRunner { - type Output = (); - - fn can_run_command(&self, _: &str, command: &Command) -> Result { - WcgiRunner::supports(command) - } - - fn run_command( - &mut self, - command_name: &str, - command: &Command, - container: &WapmContainer, - ) -> Result { - let store = Store::default(); - let ctx = RunnerContext { - container, - command, - engine: store.engine().clone(), - store, - }; - - self.run_(command_name, &ctx) - } -} diff --git a/lib/wasi/src/runners/wcgi/handler.rs b/lib/wasi/src/runners/wcgi/handler.rs new file mode 100644 index 00000000000..8144e0e61f1 --- /dev/null +++ b/lib/wasi/src/runners/wcgi/handler.rs @@ -0,0 +1,244 @@ +use std::{ + collections::HashMap, + path::{Path, PathBuf}, + pin::Pin, + sync::Arc, + task::Poll, +}; + +use anyhow::Error; +use futures::{Future, FutureExt, StreamExt, TryFutureExt}; +use http::{Request, Response}; +use hyper::{service::Service, Body}; +use tokio::{ + io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt}, + runtime::Handle, +}; +use wasmer::Module; +use wasmer_vfs::{FileSystem, PassthruFileSystem, RootFileSystemBuilder, TmpFileSystem}; +use wcgi_host::CgiDialect; + +use crate::{ + http::HttpClientCapabilityV1, runners::wcgi::MappedDirectory, Capabilities, Pipe, + VirtualTaskManager, WasiEnv, +}; + +/// The shared object that manages the instantiaion of WASI executables and +/// communicating with them via the CGI protocol. +#[derive(Debug, Clone)] +pub(crate) struct Handler { + pub(crate) program: Arc, + pub(crate) env: Arc>, + pub(crate) args: Arc<[String]>, + pub(crate) mapped_dirs: Arc<[MappedDirectory]>, + pub(crate) task_manager: Arc, + pub(crate) module: Module, + pub(crate) dialect: CgiDialect, +} + +impl Handler { + pub(crate) async fn handle(&self, req: Request) -> Result, Error> { + let (parts, body) = req.into_parts(); + + let (req_body_sender, req_body_receiver) = Pipe::channel(); + let (res_body_sender, res_body_receiver) = Pipe::channel(); + let (stderr_sender, stderr_receiver) = Pipe::channel(); + + let builder = WasiEnv::builder(self.program.to_string()); + + let mut request_specific_env = HashMap::new(); + self.dialect + .prepare_environment_variables(parts, &mut request_specific_env); + + let builder = builder + .envs(self.env.iter()) + .envs(request_specific_env) + .args(self.args.iter()) + .stdin(Box::new(req_body_receiver)) + .stdout(Box::new(res_body_sender)) + .stderr(Box::new(stderr_sender)) + .capabilities(Capabilities { + insecure_allow_all: true, + http_client: HttpClientCapabilityV1::new_allow_all(), + }) + .sandbox_fs(self.fs()?) + .preopen_dir(Path::new("/"))?; + + let module = self.module.clone(); + + let done = self + .task_manager + .runtime() + .spawn_blocking(move || builder.run(module)) + .map_err(Error::from) + .and_then(|r| async { r.map_err(Error::from) }); + + let handle = self.task_manager.runtime().clone(); + self.task_manager.runtime().spawn(async move { + if let Err(e) = + drive_request_to_completion(&handle, done, body, req_body_sender, stderr_receiver) + .await + { + tracing::error!( + error = &*e as &dyn std::error::Error, + "Unable to drive the request to completion" + ); + } + }); + + let mut res_body_receiver = tokio::io::BufReader::new(res_body_receiver); + + let parts = self + .dialect + .extract_response_header(&mut res_body_receiver) + .await?; + let chunks = futures::stream::try_unfold(res_body_receiver, |mut r| async move { + match r.fill_buf().await { + Ok(chunk) if chunk.is_empty() => Ok(None), + Ok(chunk) => { + let chunk = chunk.to_vec(); + r.consume(chunk.len()); + Ok(Some((chunk, r))) + } + Err(e) => Err(e), + } + }); + let body = hyper::Body::wrap_stream(chunks); + + let response = hyper::Response::from_parts(parts, body); + + Ok(response) + } + + fn fs(&self) -> Result { + let root_fs = RootFileSystemBuilder::new().build(); + + if !self.mapped_dirs.is_empty() { + let fs_backing: Arc = + Arc::new(PassthruFileSystem::new(crate::default_fs_backing())); + + for MappedDirectory { host, guest } in self.mapped_dirs.iter() { + let guest = match guest.starts_with('/') { + true => PathBuf::from(guest), + false => Path::new("/").join(guest), + }; + tracing::trace!( + host=%host.display(), + guest=%guest.display(), + "mounting directory to instance fs", + ); + + root_fs + .mount(host.clone(), &fs_backing, guest.clone()) + .map_err(|error| { + anyhow::anyhow!( + "Unable to mount \"{}\" to \"{}\": {error}", + host.display(), + guest.display() + ) + })?; + } + } + Ok(root_fs) + } +} + +/// Drive the request to completion by streaming the request body to the +/// instance and waiting for it to exit. +async fn drive_request_to_completion( + handle: &Handle, + done: impl Future>, + mut request_body: hyper::Body, + mut instance_stdin: impl AsyncWrite + Send + Unpin + 'static, + instance_stderr: impl AsyncRead + Send + Unpin + 'static, +) -> Result<(), Error> { + let request_body_send = handle + .spawn(async move { + // Copy the request into our instance, chunk-by-chunk. If the instance + // dies before we finish writing the body, the instance's side of the + // pipe will be automatically closed and we'll error out. + while let Some(res) = request_body.next().await { + // FIXME(theduke): figure out how to propagate a body error to the + // CGI instance. + let chunk = res?; + instance_stdin.write_all(chunk.as_ref()).await?; + } + + instance_stdin.shutdown().await?; + + Ok::<(), Error>(()) + }) + .map_err(Error::from) + .and_then(|r| async { r }); + + handle.spawn(async move { + consume_stderr(instance_stderr).await; + }); + + futures::try_join!(done, request_body_send)?; + + Ok(()) +} + +/// Read the instance's stderr, taking care to preserve output even when WASI +/// pipe errors occur so users still have *something* they use for +/// troubleshooting. +async fn consume_stderr(stderr: impl AsyncRead + Send + Unpin + 'static) { + let mut stderr = tokio::io::BufReader::new(stderr); + + // FIXME: this could lead to unbound memory usage + let mut buffer = Vec::new(); + + // Note: we don't want to just read_to_end() because a reading error + // would cause us to lose all of stderr. At least this way we'll be + // able to show users the partial result. + loop { + match stderr.fill_buf().await { + Ok(chunk) if chunk.is_empty() => { + // EOF - the instance's side of the pipe was closed. + break; + } + Ok(chunk) => { + buffer.extend(chunk); + let bytes_read = chunk.len(); + stderr.consume(bytes_read); + } + Err(e) => { + tracing::error!( + error = &e as &dyn std::error::Error, + bytes_read = buffer.len(), + "Unable to read the complete stderr", + ); + break; + } + } + } + + let stderr = String::from_utf8(buffer).unwrap_or_else(|e| { + tracing::warn!( + error = &e as &dyn std::error::Error, + "Stdout wasn't valid UTF-8", + ); + String::from_utf8_lossy(e.as_bytes()).into_owned() + }); + + tracing::info!(%stderr); +} + +impl Service> for Handler { + type Response = Response; + type Error = Error; + type Future = Pin, Error>> + Send>>; + + fn poll_ready(&mut self, _cx: &mut std::task::Context<'_>) -> Poll> { + // TODO: We probably should implement some sort of backpressure here... + Poll::Ready(Ok(())) + } + + fn call(&mut self, request: Request) -> Self::Future { + // Note: all fields are reference-counted so cloning is pretty cheap + let handler = self.clone(); + let fut = async move { handler.handle(request).await }; + fut.boxed() + } +} diff --git a/lib/wasi/src/runners/wcgi/mod.rs b/lib/wasi/src/runners/wcgi/mod.rs new file mode 100644 index 00000000000..4b3df172778 --- /dev/null +++ b/lib/wasi/src/runners/wcgi/mod.rs @@ -0,0 +1,12 @@ +mod handler; +mod runner; + +use std::path::PathBuf; + +pub use self::runner::WcgiRunner; + +#[derive(Debug, Clone, PartialEq)] +pub(crate) struct MappedDirectory { + pub host: PathBuf, + pub guest: String, +} diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs new file mode 100644 index 00000000000..599709376b2 --- /dev/null +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -0,0 +1,275 @@ +use std::{collections::HashMap, convert::Infallible, net::SocketAddr, path::PathBuf, sync::Arc}; + +use anyhow::{Context, Error}; +use wasmer::{Engine, Module, Store}; +use wasmer_vfs::FileSystem; +use wcgi_host::CgiDialect; +use webc::metadata::{Command, Manifest}; + +use crate::{ + runners::{ + wcgi::{handler::Handler, MappedDirectory}, + WapmContainer, + }, + runtime::task_manager::tokio::TokioTaskManager, + VirtualTaskManager, +}; + +pub struct WcgiRunner { + program_name: Arc, + config: Config, +} + +// TODO(Michael-F-Bryan): When we rewrite the existing runner infrastructure, +// make the "Runner" trait contain just these two methods. +impl WcgiRunner { + fn supports(cmd: &Command) -> Result { + Ok(cmd.runner.starts_with("https://webc.org/runner/wcgi")) + } + + #[tracing::instrument(skip(self, ctx))] + fn run_(&mut self, command_name: &str, ctx: &RunnerContext<'_>) -> Result<(), Error> { + let module = self.load_module(ctx).context("Couldn't load the module")?; + + let handler = self.create_handler(module, ctx)?; + let task_manager = Arc::clone(&handler.task_manager); + + let make_service = hyper::service::make_service_fn(move |_| { + let handler = handler.clone(); + async { Ok::<_, Infallible>(handler) } + }); + + let address = self.config.addr; + tracing::info!(%address, "Starting the server"); + + task_manager + .block_on(async { hyper::Server::bind(&address).serve(make_service).await }) + .context("Unable to start the server")?; + + todo!(); + } +} + +impl WcgiRunner { + pub fn new(program_name: impl Into>) -> Self { + WcgiRunner { + program_name: program_name.into(), + config: Config::default(), + } + } + + pub fn config(&mut self) -> &mut Config { + &mut self.config + } + + fn load_module(&self, ctx: &RunnerContext<'_>) -> Result { + let wasi: webc::metadata::annotations::Wasi = ctx + .command() + .annotations + .get("wasi") + .cloned() + .and_then(|v| serde_cbor::value::from_value(v).ok()) + .context("Unable to retrieve the WASI metadata")?; + + let atom_name = &wasi.atom; + let atom = ctx + .get_atom(&atom_name) + .with_context(|| format!("Unable to retrieve the \"{atom_name}\" atom"))?; + + let module = ctx.compile(atom).context("Unable to compile the atom")?; + + Ok(module) + } + + fn create_handler(&self, module: Module, ctx: &RunnerContext<'_>) -> Result { + let mut env = HashMap::new(); + + if self.config.forward_host_env { + env.extend(std::env::vars()); + } + + env.extend(self.config.env.clone()); + + let webc::metadata::annotations::Wcgi { dialect, .. } = ctx + .command() + .annotations + .get("wcgi") + .cloned() + .and_then(|v| serde_cbor::value::from_value(v).ok()) + .context("No \"wcgi\" annotations associated with this command")?; + + let dialect = match dialect { + Some(d) => d.parse().context("Unable to parse the CGI dialect")?, + None => CgiDialect::Wcgi, + }; + + let handler = Handler { + program: Arc::clone(&self.program_name), + env: Arc::new(env), + args: self.config.args.clone().into(), + mapped_dirs: self.config.mapped_dirs.clone().into(), + task_manager: self + .config + .task_manager + .clone() + .unwrap_or_else(|| Arc::new(TokioTaskManager::default())), + module, + dialect, + }; + + Ok(handler) + } +} + +// TODO(Michael-F-Bryan): Pass this to Runner::run() as "&dyn RunnerContext" +// when we rewrite the "Runner" trait. +struct RunnerContext<'a> { + container: &'a WapmContainer, + command: &'a Command, + engine: Engine, + store: Store, +} + +#[allow(dead_code)] +impl RunnerContext<'_> { + fn command(&self) -> &Command { + self.command + } + + fn manifest(&self) -> &Manifest { + self.container.manifest() + } + + fn engine(&self) -> &Engine { + &self.engine + } + + fn store(&self) -> &Store { + &self.store + } + + fn volume(&self, _name: &str) -> Option> { + todo!("Implement a read-only filesystem backed by a volume"); + } + + fn get_atom(&self, name: &str) -> Option<&[u8]> { + self.container.get_atom(name) + } + + fn compile(&self, wasm: &[u8]) -> Result { + // TODO(Michael-F-Bryan): wire this up to wasmer-cache + Module::new(&self.engine, wasm).map_err(Error::from) + } +} + +impl crate::runners::Runner for WcgiRunner { + type Output = (); + + fn can_run_command(&self, _: &str, command: &Command) -> Result { + WcgiRunner::supports(command) + } + + fn run_command( + &mut self, + command_name: &str, + command: &Command, + container: &WapmContainer, + ) -> Result { + let store = Store::default(); + let ctx = RunnerContext { + container, + command, + engine: store.engine().clone(), + store, + }; + + self.run_(command_name, &ctx) + } +} + +#[derive(Debug)] +pub struct Config { + task_manager: Option>, + addr: SocketAddr, + args: Vec, + env: HashMap, + forward_host_env: bool, + mapped_dirs: Vec, +} + +impl Config { + pub fn task_manager(&mut self, task_manager: impl VirtualTaskManager) -> &mut Self { + self.task_manager = Some(Arc::new(task_manager)); + self + } + + pub fn addr(&mut self, addr: SocketAddr) -> &mut Self { + self.addr = addr; + self + } + + /// Add an argument to the WASI executable's command-line arguments. + pub fn arg(&mut self, arg: impl Into) -> &mut Self { + self.args.push(arg.into()); + self + } + + /// Add multiple arguments to the WASI executable's command-line arguments. + pub fn args(&mut self, args: A) -> &mut Self + where + A: IntoIterator, + S: Into, + { + self.args.extend(args.into_iter().map(|s| s.into())); + self + } + + /// Expose an environment variable to the guest. + pub fn env(&mut self, name: impl Into, value: impl Into) -> &mut Self { + self.env.insert(name.into(), value.into()); + self + } + + /// Expose multiple environment variables to the guest. + pub fn envs(&mut self, variables: I) -> &mut Self + where + I: IntoIterator, + K: Into, + V: Into, + { + self.env + .extend(variables.into_iter().map(|(k, v)| (k.into(), v.into()))); + self + } + + /// Forward all of the host's environment variables to the guest. + pub fn forward_host_env(&mut self) -> &mut Self { + self.forward_host_env = true; + self + } + + pub fn map_directory( + &mut self, + host: impl Into, + guest: impl Into, + ) -> &mut Self { + self.mapped_dirs.push(MappedDirectory { + host: host.into(), + guest: guest.into(), + }); + self + } +} + +impl Default for Config { + fn default() -> Self { + Self { + task_manager: None, + addr: ([127, 0, 0, 1], 8000).into(), + env: HashMap::new(), + forward_host_env: false, + mapped_dirs: Vec::new(), + args: Vec::new(), + } + } +} From 91a8d4c5986bff9391f7d99bf11e16171c8a6437 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 28 Feb 2023 22:24:25 +0800 Subject: [PATCH 07/26] Wrote integration tests for the WASI and WCGI runners --- lib/cli/src/commands/run.rs | 11 ++- lib/wasi/src/runners/container.rs | 103 ++++++++++++++++++++++----- lib/wasi/src/runners/emscripten.rs | 9 +-- lib/wasi/src/runners/runner.rs | 13 ++-- lib/wasi/src/runners/wasi.rs | 59 +++++++++++---- lib/wasi/src/runners/wcgi/handler.rs | 4 +- lib/wasi/src/runners/wcgi/runner.rs | 40 ++++++++--- lib/wasi/tests/runners.rs | 100 ++++++++++++++++++++++++++ 8 files changed, 281 insertions(+), 58 deletions(-) create mode 100644 lib/wasi/tests/runners.rs diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 0004245716d..5d800fa90a8 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -223,7 +223,7 @@ impl RunWithPathBuf { fn inner_execute(&self) -> Result<()> { #[cfg(feature = "webc_runner")] { - if let Ok(pf) = WapmContainer::new(self.path.clone()) { + if let Ok(pf) = WapmContainer::from_path(self.path.clone()) { return self .run_container(pf, self.command_name.as_deref(), &self.args) .map_err(|e| anyhow!("Could not run PiritaFile: {e}")); @@ -403,6 +403,15 @@ impl RunWithPathBuf { .context("Emscripten runner failed")?; } + let (store, _compiler_type) = self.store.get_store()?; + let mut runner = wasmer_wasi::runners::wcgi::WcgiRunner::new(store); + runner.set_args(args.to_vec()); + if runner.can_run_command(id, command).unwrap_or(false) { + runner + .run_cmd(&container, id) + .context("Emscripten runner failed")?; + } + anyhow::bail!("No runner"); } diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index fd77a755d3a..2eb01baef5c 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -1,38 +1,67 @@ use std::{path::PathBuf, sync::Arc}; -use webc::{metadata::Manifest, v1::WebCMmap}; +use bytes::Bytes; +use wasmer_vfs::{webc_fs::WebcFileSystem, FileSystem}; +use webc::{ + metadata::Manifest, + v1::{ParseOptions, WebC, WebCMmap, WebCOwned}, + Version, +}; /// Parsed WAPM file, memory-mapped to an on-disk path #[derive(Debug, Clone)] pub struct WapmContainer { - /// WebC container - webc: Arc, + repr: Repr, } impl WapmContainer { /// Parses a .webc container file. Since .webc files /// can be very large, only file paths are allowed. - pub fn new(path: PathBuf) -> std::result::Result { - let webc = webc::v1::WebCMmap::parse(path, &webc::v1::ParseOptions::default())?; + pub fn from_path(path: PathBuf) -> std::result::Result { + let webc = webc::v1::WebCMmap::parse(path, &ParseOptions::default())?; Ok(Self { - webc: Arc::new(webc), + repr: Repr::V1Mmap(Arc::new(webc)), }) } + pub fn from_bytes(bytes: Bytes) -> std::result::Result { + match webc::detect(bytes.as_ref())? { + Version::V1 => { + let webc = WebCOwned::parse(bytes.into(), &ParseOptions::default())?; + Ok(WapmContainer { + repr: Repr::V1Owned(Arc::new(webc)), + }) + } + Version::V2 => todo!(), + other => Err(WebcParseError::UnsupportedVersion(other)), + } + } + /// Returns the bytes of a file or a stringified error pub fn get_file<'b>(&'b self, path: &str) -> Result<&'b [u8], String> { - self.webc - .get_file(&self.webc.get_package_name(), path) - .map_err(|e| e.0) + match &self.repr { + Repr::V1Mmap(mapped) => mapped + .get_file(&mapped.get_package_name(), path) + .map_err(|e| e.0), + Repr::V1Owned(owned) => owned + .get_file(&owned.get_package_name(), path) + .map_err(|e| e.0), + } } /// Returns a list of volumes in this container pub fn get_volumes(&self) -> Vec { - self.webc.volumes.keys().cloned().collect::>() + match &self.repr { + Repr::V1Mmap(mapped) => mapped.volumes.keys().cloned().collect(), + Repr::V1Owned(owned) => owned.volumes.keys().cloned().collect(), + } } pub fn get_atom(&self, name: &str) -> Option<&[u8]> { - self.webc.get_atom(&self.webc.get_package_name(), name).ok() + match &self.repr { + Repr::V1Mmap(mapped) => mapped.get_atom(&mapped.get_package_name(), name).ok(), + Repr::V1Owned(owned) => owned.get_atom(&owned.get_package_name(), name).ok(), + } } /// Lookup .wit bindings by name and parse them @@ -41,8 +70,7 @@ impl WapmContainer { bindings: &str, ) -> std::result::Result { let bindings = self - .webc - .manifest + .manifest() .bindings .iter() .find(|b| b.name == bindings) @@ -52,14 +80,47 @@ impl WapmContainer { } pub fn manifest(&self) -> &Manifest { - &self.webc.manifest + match &self.repr { + Repr::V1Mmap(mapped) => &mapped.manifest, + Repr::V1Owned(owned) => &owned.manifest, + } + } + + // HACK(Michael-F-Bryan): WapmContainer originally exposed its Arc + // field, so every man and his dog accessed it directly instead of going + // through the WapmContainer abstraction. This is an escape hatch to make + // that code w + pub fn v1(&self) -> &WebC<'_> { + match &self.repr { + Repr::V1Mmap(mapped) => &*mapped, + Repr::V1Owned(owned) => &*owned, + } } - pub fn v1(&self) -> &Arc { - &self.webc + pub(crate) fn volume_fs(&self, package_name: &str) -> Box { + match &self.repr { + Repr::V1Mmap(mapped) => { + Box::new(WebcFileSystem::init(Arc::clone(mapped), package_name)) + } + Repr::V1Owned(owned) => Box::new(WebcFileSystem::init(Arc::clone(owned), package_name)), + } + } + + /// Get the entire container as a single filesystem. + pub(crate) fn container_fs(&self) -> Box { + match &self.repr { + Repr::V1Mmap(mapped) => Box::new(WebcFileSystem::init_all(Arc::clone(mapped))), + Repr::V1Owned(owned) => Box::new(WebcFileSystem::init_all(Arc::clone(owned))), + } } } +#[derive(Debug, Clone)] +enum Repr { + V1Mmap(Arc), + V1Owned(Arc), +} + /// Error that happened while parsing .wit bindings #[derive(Debug, Clone, PartialEq, Eq, PartialOrd)] pub enum ParseBindingsError { @@ -119,10 +180,12 @@ impl Bindings for WitBindings { } /// Error that ocurred while parsing the .webc file -#[derive(Debug, PartialEq, Eq, Clone)] +#[derive(Debug)] pub enum WebcParseError { /// Parse error Parse(webc::v1::Error), + Detect(webc::DetectError), + UnsupportedVersion(Version), } impl From for WebcParseError { @@ -130,3 +193,9 @@ impl From for WebcParseError { WebcParseError::Parse(e) } } + +impl From for WebcParseError { + fn from(e: webc::DetectError) -> Self { + WebcParseError::Detect(e) + } +} diff --git a/lib/wasi/src/runners/emscripten.rs b/lib/wasi/src/runners/emscripten.rs index 6e9704131d7..bd8a6e9777c 100644 --- a/lib/wasi/src/runners/emscripten.rs +++ b/lib/wasi/src/runners/emscripten.rs @@ -3,13 +3,12 @@ use crate::runners::WapmContainer; use anyhow::{anyhow, Error}; use serde::{Deserialize, Serialize}; -use std::sync::Arc; use wasmer::{FunctionEnv, Instance, Module, Store}; use wasmer_emscripten::{ generate_emscripten_env, is_emscripten_module, run_emscripten_instance, EmEnv, EmscriptenGlobals, }; -use webc::{metadata::Command, v1::WebCMmap}; +use webc::metadata::Command; #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct EmscriptenRunner { @@ -72,15 +71,13 @@ impl crate::runners::Runner for EmscriptenRunner { let mut module = Module::new(&self.store, atom_bytes)?; module.set_name(&atom_name); - let (mut globals, env) = - prepare_emscripten_env(&mut self.store, &module, container.clone(), &atom_name)?; + let (mut globals, env) = prepare_emscripten_env(&mut self.store, &module, &atom_name)?; exec_module( &mut self.store, &module, &mut globals, env, - container.clone(), &atom_name, main_args.unwrap_or_default(), )?; @@ -92,7 +89,6 @@ impl crate::runners::Runner for EmscriptenRunner { fn prepare_emscripten_env( store: &mut Store, module: &Module, - _atom: Arc, name: &str, ) -> Result<(EmscriptenGlobals, FunctionEnv), anyhow::Error> { if !is_emscripten_module(module) { @@ -113,7 +109,6 @@ fn exec_module( module: &Module, globals: &mut EmscriptenGlobals, em_env: FunctionEnv, - _atom: Arc, name: &str, args: Vec, ) -> Result<(), anyhow::Error> { diff --git a/lib/wasi/src/runners/runner.rs b/lib/wasi/src/runners/runner.rs index a42ee363409..ef330b05801 100644 --- a/lib/wasi/src/runners/runner.rs +++ b/lib/wasi/src/runners/runner.rs @@ -27,8 +27,7 @@ pub trait Runner { let cmd = match container.manifest().entrypoint.as_ref() { Some(s) => s, None => { - let path = format!("{}", container.v1().path.display()); - anyhow::bail!("Cannot run {path:?}: not executable (no entrypoint in manifest)"); + anyhow::bail!("Cannot run the package: not executable (no entrypoint in manifest)"); } }; @@ -37,15 +36,11 @@ pub trait Runner { /// Runs the given `cmd` on the container fn run_cmd(&mut self, container: &WapmContainer, cmd: &str) -> Result { - let webc = container.v1(); - let path = format!("{}", webc.path.display()); - let command_to_exec = webc - .manifest + let command_to_exec = container + .manifest() .commands .get(cmd) - .ok_or_else(|| anyhow::anyhow!("{path}: command {cmd:?} not found in manifest"))?; - - let _path = format!("{}", webc.path.display()); + .ok_or_else(|| anyhow::anyhow!("command {cmd:?} not found in manifest"))?; match self.can_run_command(cmd, command_to_exec) { Ok(true) => {} diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index b1a529a69b5..5bd802c7ef9 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -1,19 +1,21 @@ //! WebC container support for running WASI modules -use crate::runners::WapmContainer; +use std::sync::Arc; + +use crate::{runners::WapmContainer, PluggableRuntimeImplementation, VirtualTaskManager}; use crate::{WasiEnv, WasiEnvBuilder}; use anyhow::Error; use serde::{Deserialize, Serialize}; -use std::sync::Arc; use wasmer::{Module, Store}; -use wasmer_vfs::webc_fs::WebcFileSystem; -use webc::{metadata::Command, v1::WebCMmap}; +use webc::metadata::Command; -#[derive(Debug, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] pub struct WasiRunner { args: Vec, #[serde(skip, default)] store: Store, + #[serde(skip, default)] + tasks: Option>, } impl WasiRunner { @@ -22,6 +24,7 @@ impl WasiRunner { Self { args: Vec::new(), store, + tasks: None, } } @@ -40,6 +43,15 @@ impl WasiRunner { pub fn set_args(&mut self, args: Vec) { self.args = args; } + + pub fn with_task_manager(mut self, tasks: impl VirtualTaskManager) -> Self { + self.set_task_manager(tasks); + self + } + + pub fn set_task_manager(&mut self, tasks: impl VirtualTaskManager) { + self.tasks = Some(Arc::new(tasks)); + } } impl crate::runners::Runner for WasiRunner { @@ -56,16 +68,23 @@ impl crate::runners::Runner for WasiRunner { _command: &Command, container: &WapmContainer, ) -> Result { - let container = container.v1(); - let atom_name = container + let webc = container.v1(); + let atom_name = webc .get_atom_name_for_command("wasi", command_name) .map_err(Error::msg)?; - let atom_bytes = container.get_atom(&container.get_package_name(), &atom_name)?; + let atom_bytes = webc.get_atom(&webc.get_package_name(), &atom_name)?; let mut module = Module::new(&self.store, atom_bytes)?; module.set_name(&atom_name); - let builder = prepare_webc_env(container.clone(), &atom_name, &self.args)?; + let mut builder = prepare_webc_env(container, &atom_name, &self.args)?; + + if let Some(tasks) = &self.tasks { + eprintln!("Aasdfasf"); + + let rt = PluggableRuntimeImplementation::new(Arc::clone(&tasks)); + builder.set_runtime(Arc::new(rt)); + } let init = builder.build_init()?; @@ -82,15 +101,29 @@ impl crate::runners::Runner for WasiRunner { // https://github.com/tokera-com/ate/blob/42c4ce5a0c0aef47aeb4420cc6dc788ef6ee8804/term-lib/src/eval/exec.rs#L444 fn prepare_webc_env( - webc: Arc, + container: &WapmContainer, command: &str, args: &[String], ) -> Result { - let filesystem = Box::new(WebcFileSystem::init_all(webc)); + let filesystem = container.container_fs(); let mut builder = WasiEnv::builder(command).args(args); - for f_name in filesystem.top_level_dirs() { - builder.add_preopen_build(|p| p.directory(f_name).read(true).write(true).create(true))?; + + if let Ok(dir) = filesystem.read_dir("/".as_ref()) { + let entries = dir.filter_map(|entry| entry.ok()).filter(|entry| { + if let Ok(file_type) = entry.file_type() { + file_type.dir + } else { + false + } + }); + + for entry in entries { + builder.add_preopen_build(|p| { + p.directory(&entry.path).read(true).write(true).create(true) + })?; + } } + builder.set_fs(filesystem); Ok(builder) diff --git a/lib/wasi/src/runners/wcgi/handler.rs b/lib/wasi/src/runners/wcgi/handler.rs index 8144e0e61f1..c0c419fe771 100644 --- a/lib/wasi/src/runners/wcgi/handler.rs +++ b/lib/wasi/src/runners/wcgi/handler.rs @@ -20,7 +20,7 @@ use wcgi_host::CgiDialect; use crate::{ http::HttpClientCapabilityV1, runners::wcgi::MappedDirectory, Capabilities, Pipe, - VirtualTaskManager, WasiEnv, + PluggableRuntimeImplementation, VirtualTaskManager, WasiEnv, }; /// The shared object that manages the instantiaion of WASI executables and @@ -50,6 +50,7 @@ impl Handler { self.dialect .prepare_environment_variables(parts, &mut request_specific_env); + let rt = PluggableRuntimeImplementation::new(Arc::clone(&self.task_manager)); let builder = builder .envs(self.env.iter()) .envs(request_specific_env) @@ -61,6 +62,7 @@ impl Handler { insecure_allow_all: true, http_client: HttpClientCapabilityV1::new_allow_all(), }) + .runtime(Arc::new(rt)) .sandbox_fs(self.fs()?) .preopen_dir(Path::new("/"))?; diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index 599709376b2..e2277b147e2 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -1,10 +1,11 @@ use std::{collections::HashMap, convert::Infallible, net::SocketAddr, path::PathBuf, sync::Arc}; use anyhow::{Context, Error}; +use futures::channel::oneshot::Receiver; use wasmer::{Engine, Module, Store}; use wasmer_vfs::FileSystem; use wcgi_host::CgiDialect; -use webc::metadata::{Command, Manifest}; +use webc::metadata::{annotations::Wcgi, Command, Manifest}; use crate::{ runners::{ @@ -24,7 +25,7 @@ pub struct WcgiRunner { // make the "Runner" trait contain just these two methods. impl WcgiRunner { fn supports(cmd: &Command) -> Result { - Ok(cmd.runner.starts_with("https://webc.org/runner/wcgi")) + Ok(cmd.runner.starts_with("https://webc.org/runner/wasi")) } #[tracing::instrument(skip(self, ctx))] @@ -42,8 +43,23 @@ impl WcgiRunner { let address = self.config.addr; tracing::info!(%address, "Starting the server"); + let abort = self.config.abort.take(); + task_manager - .block_on(async { hyper::Server::bind(&address).serve(make_service).await }) + .block_on(async { + let server = hyper::Server::bind(&address).serve(make_service); + + match abort { + Some(abort) => { + server + .with_graceful_shutdown(async move { + let _ = abort.await; + }) + .await + } + None => server.await, + } + }) .context("Unable to start the server")?; todo!(); @@ -90,13 +106,10 @@ impl WcgiRunner { env.extend(self.config.env.clone()); - let webc::metadata::annotations::Wcgi { dialect, .. } = ctx - .command() - .annotations - .get("wcgi") - .cloned() - .and_then(|v| serde_cbor::value::from_value(v).ok()) - .context("No \"wcgi\" annotations associated with this command")?; + let Wcgi { dialect, .. } = match ctx.command().annotations.get("wcgi") { + Some(v) => serde_cbor::value::from_value(v.clone())?, + None => Wcgi::default(), + }; let dialect = match dialect { Some(d) => d.parse().context("Unable to parse the CGI dialect")?, @@ -195,6 +208,7 @@ pub struct Config { env: HashMap, forward_host_env: bool, mapped_dirs: Vec, + abort: Option>, } impl Config { @@ -259,6 +273,11 @@ impl Config { }); self } + + pub fn abort_channel(&mut self, rx: Receiver<()>) -> &mut Self { + self.abort = Some(rx); + self + } } impl Default for Config { @@ -270,6 +289,7 @@ impl Default for Config { forward_host_env: false, mapped_dirs: Vec::new(), args: Vec::new(), + abort: None, } } } diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs new file mode 100644 index 00000000000..620fab3fbc1 --- /dev/null +++ b/lib/wasi/tests/runners.rs @@ -0,0 +1,100 @@ +#![cfg(feature = "webc_runner")] + +use std::path::Path; + +use reqwest::Client; +use wasmer_wasi::runners::{Runner, WapmContainer}; + +#[cfg(feature = "webc_runner_rt_wasi")] +mod wasi { + use tokio::runtime::Handle; + use wasmer::Store; + use wasmer_wasi::{ + runners::wasi::WasiRunner, runtime::task_manager::tokio::TokioTaskManager, WasiRuntimeError, + }; + + use super::*; + + #[tokio::test] + async fn wat_2_wasm() { + let webc = download_cached("https://wapm.io/wasmer/wabt").await; + let store = Store::default(); + let tasks = TokioTaskManager::new(Handle::current()); + let container = WapmContainer::from_bytes(webc).unwrap(); + + // Note: we don't have any way to intercept stdin or stdout, so blindly + // assume that everything is fine if it runs successfully. + let err = WasiRunner::new(store) + .with_task_manager(tasks) + .run_cmd(&container, "wat2wasm") + .unwrap_err(); + + let runtime_error: &WasiRuntimeError = err.downcast().unwrap(); + let exit_code = runtime_error.as_exit_code().unwrap(); + assert_eq!(exit_code, 1); + } +} + +#[cfg(feature = "webc_runner_rt_wcgi")] +mod wcgi { + use rand::Rng; + use tokio::runtime::Handle; + use wasmer::Store; + use wasmer_wasi::{runners::wcgi::WcgiRunner, runtime::task_manager::tokio::TokioTaskManager}; + + use super::*; + + #[tokio::test] + async fn static_server() { + let webc = download_cached("https://wapm.dev/syrusakbary/staticserver").await; + let tasks = TokioTaskManager::new(Handle::current()); + let container = WapmContainer::from_bytes(webc).unwrap(); + + let mut runner = WcgiRunner::new("staticserver"); + let port = rand::thread_rng().gen_range(10000_u16..65535_u16); + let (tx, rx) = futures::channel::oneshot::channel(); + runner + .config() + .addr(([127, 0, 0, 1], port).into()) + .task_manager(tasks) + .abort_channel(rx); + runner.run_cmd(&container, "wcgi").unwrap(); + + todo!(); + } +} + +async fn download_cached(url: &str) -> bytes::Bytes { + let uri: http::Uri = url.parse().unwrap(); + + let file_name = Path::new(uri.path()).file_name().unwrap(); + let cache_dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(module_path!()); + let cached_path = cache_dir.join(file_name); + + if cached_path.exists() { + return std::fs::read(&cached_path).unwrap().into(); + } + + let client = Client::new(); + + let response = client + .get(url) + .header("Accept", "application/webc") + .send() + .await + .unwrap(); + + assert_eq!( + response.status(), + 200, + "Unable to get \"{url}\": {}", + response.status(), + ); + + let body = response.bytes().await.unwrap(); + + std::fs::create_dir_all(&cache_dir).unwrap(); + std::fs::write(&cached_path, &body).unwrap(); + + body +} From d09b6a512dc8e0d5b5a38778e3462b16b5f46ef8 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 28 Feb 2023 22:51:37 +0800 Subject: [PATCH 08/26] Fleshed out the WCGI runner integration test --- lib/cli/src/commands/run.rs | 5 ++--- lib/wasi/src/runners/wasi.rs | 2 -- lib/wasi/tests/runners.rs | 39 ++++++++++++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 5d800fa90a8..f918663bde7 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -403,9 +403,8 @@ impl RunWithPathBuf { .context("Emscripten runner failed")?; } - let (store, _compiler_type) = self.store.get_store()?; - let mut runner = wasmer_wasi::runners::wcgi::WcgiRunner::new(store); - runner.set_args(args.to_vec()); + let mut runner = wasmer_wasi::runners::wcgi::WcgiRunner::new(id); + runner.config().args(args); if runner.can_run_command(id, command).unwrap_or(false) { runner .run_cmd(&container, id) diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 5bd802c7ef9..66c3b99d7e5 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -80,8 +80,6 @@ impl crate::runners::Runner for WasiRunner { let mut builder = prepare_webc_env(container, &atom_name, &self.args)?; if let Some(tasks) = &self.tasks { - eprintln!("Aasdfasf"); - let rt = PluggableRuntimeImplementation::new(Arc::clone(&tasks)); builder.set_runtime(Arc::new(rt)); } diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs index 620fab3fbc1..ef252de6da3 100644 --- a/lib/wasi/tests/runners.rs +++ b/lib/wasi/tests/runners.rs @@ -37,9 +37,10 @@ mod wasi { #[cfg(feature = "webc_runner_rt_wcgi")] mod wcgi { + use std::thread::JoinHandle; + use rand::Rng; use tokio::runtime::Handle; - use wasmer::Store; use wasmer_wasi::{runners::wcgi::WcgiRunner, runtime::task_manager::tokio::TokioTaskManager}; use super::*; @@ -49,18 +50,48 @@ mod wcgi { let webc = download_cached("https://wapm.dev/syrusakbary/staticserver").await; let tasks = TokioTaskManager::new(Handle::current()); let container = WapmContainer::from_bytes(webc).unwrap(); - let mut runner = WcgiRunner::new("staticserver"); let port = rand::thread_rng().gen_range(10000_u16..65535_u16); + let port = 12345; let (tx, rx) = futures::channel::oneshot::channel(); runner .config() .addr(([127, 0, 0, 1], port).into()) .task_manager(tasks) .abort_channel(rx); - runner.run_cmd(&container, "wcgi").unwrap(); + // Note: the server blocks, so spin it up in a background thread and kill it + // after we've made our request. + let _guard = thread_spawn(move || { + runner.run_cmd(&container, "wcgi").unwrap(); + }); + + // The way we test this is by fetching "/" and checking it contains + // something we expect + let resp = reqwest::get(format!("http://localhost:{port}/index.html")) + .await + .unwrap(); + let body = resp.error_for_status().unwrap().text().await.unwrap(); + + assert!(body.contains("asdf"), "{}", body); + + // Make sure we shut the server down afterwards + drop(tx); + } - todo!(); + fn thread_spawn(f: impl FnOnce() + Send + 'static) -> impl Drop { + struct JoinOnDrop(Option>); + impl Drop for JoinOnDrop { + fn drop(&mut self) { + if let Err(e) = self.0.take().unwrap().join() { + if !std::thread::panicking() { + std::panic::resume_unwind(e); + } + } + } + } + let handle = std::thread::spawn(f); + + JoinOnDrop(Some(handle)) } } From 124dacfd7dce7f58c65d5feef0ab028ccabaf529 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 28 Feb 2023 22:58:30 +0800 Subject: [PATCH 09/26] Remove the (now redundant) wcgi-runner crate --- Cargo.lock | 88 +----- Cargo.toml | 1 - lib/wcgi-runner/Cargo.toml | 40 --- lib/wcgi-runner/README.md | 9 - lib/wcgi-runner/examples/wcgi-runner.rs | 111 -------- lib/wcgi-runner/src/annotations.rs | 24 -- lib/wcgi-runner/src/builder.rs | 252 ------------------ lib/wcgi-runner/src/context.rs | 226 ---------------- lib/wcgi-runner/src/errors.rs | 65 ----- lib/wcgi-runner/src/lib.rs | 17 -- lib/wcgi-runner/src/module_loader/cached.rs | 48 ---- .../src/module_loader/file_loader.rs | 48 ---- lib/wcgi-runner/src/module_loader/mod.rs | 67 ----- lib/wcgi-runner/src/module_loader/wasm.rs | 40 --- lib/wcgi-runner/src/module_loader/webc.rs | 155 ----------- lib/wcgi-runner/src/runner.rs | 87 ------ lib/wcgi-runner/tests/integration.rs | 95 ------- 17 files changed, 2 insertions(+), 1371 deletions(-) delete mode 100644 lib/wcgi-runner/Cargo.toml delete mode 100644 lib/wcgi-runner/README.md delete mode 100644 lib/wcgi-runner/examples/wcgi-runner.rs delete mode 100644 lib/wcgi-runner/src/annotations.rs delete mode 100644 lib/wcgi-runner/src/builder.rs delete mode 100644 lib/wcgi-runner/src/context.rs delete mode 100644 lib/wcgi-runner/src/errors.rs delete mode 100644 lib/wcgi-runner/src/lib.rs delete mode 100644 lib/wcgi-runner/src/module_loader/cached.rs delete mode 100644 lib/wcgi-runner/src/module_loader/file_loader.rs delete mode 100644 lib/wcgi-runner/src/module_loader/mod.rs delete mode 100644 lib/wcgi-runner/src/module_loader/wasm.rs delete mode 100644 lib/wcgi-runner/src/module_loader/webc.rs delete mode 100644 lib/wcgi-runner/src/runner.rs delete mode 100644 lib/wcgi-runner/tests/integration.rs diff --git a/Cargo.lock b/Cargo.lock index 987dedc7776..091617d391b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -66,9 +66,6 @@ name = "anyhow" version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "224afbd727c3d6e4b90103ece64b8d1b67fbb1973b1046c2281eed3f3803f800" -dependencies = [ - "backtrace", -] [[package]] name = "arbitrary" @@ -430,8 +427,8 @@ checksum = "71655c45cb9845d3270c9d6df84ebe72b4dad3c2ba3f7023ad47c144e4e473a5" dependencies = [ "atty", "bitflags", - "clap_derive 3.2.18", - "clap_lex 0.2.4", + "clap_derive", + "clap_lex", "indexmap", "once_cell", "strsim", @@ -439,21 +436,6 @@ dependencies = [ "textwrap 0.16.0", ] -[[package]] -name = "clap" -version = "4.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d7ae14b20b94cb02149ed21a86c423859cbe18dc7ed69845cace50e52b40a5" -dependencies = [ - "bitflags", - "clap_derive 4.1.8", - "clap_lex 0.3.2", - "is-terminal", - "once_cell", - "strsim", - "termcolor", -] - [[package]] name = "clap_derive" version = "3.2.18" @@ -467,19 +449,6 @@ dependencies = [ "syn", ] -[[package]] -name = "clap_derive" -version = "4.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44bec8e5c9d09e439c4335b1af0abaab56dcf3b94999a936e1bb47b9134288f0" -dependencies = [ - "heck 0.4.1", - "proc-macro-error", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "clap_lex" version = "0.2.4" @@ -489,15 +458,6 @@ dependencies = [ "os_str_bytes", ] -[[package]] -name = "clap_lex" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350b9cf31731f9957399229e9b2adc51eeabdfbe9d71d9a0552275fd12710d09" -dependencies = [ - "os_str_bytes", -] - [[package]] name = "cmake" version = "0.1.49" @@ -4367,22 +4327,6 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" -[[package]] -name = "ureq" -version = "2.6.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "338b31dd1314f68f3aabf3ed57ab922df95ffcd902476ca7ba3c4ce7b908c46d" -dependencies = [ - "base64 0.13.1", - "flate2", - "log", - "once_cell", - "rustls 0.20.8", - "url", - "webpki 0.22.0", - "webpki-roots 0.22.6", -] - [[package]] name = "url" version = "2.3.1" @@ -5663,34 +5607,6 @@ dependencies = [ "wcgi", ] -[[package]] -name = "wcgi-runner" -version = "3.2.0-alpha.1" -dependencies = [ - "anyhow", - "async-trait", - "bytes", - "clap 4.1.8", - "futures", - "http", - "hyper", - "serde", - "serde_cbor", - "tempfile", - "thiserror", - "tokio", - "tower-service", - "tracing", - "tracing-subscriber 0.3.16", - "ureq", - "wasmer", - "wasmer-vfs", - "wasmer-wasi", - "wcgi", - "wcgi-host", - "webc", -] - [[package]] name = "web-sys" version = "0.3.61" diff --git a/Cargo.toml b/Cargo.toml index 43e3ec71f6f..2ed45f1da93 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,6 @@ members = [ "lib/wasi-local-networking", "lib/wasix/wasix-http-client", "lib/wasm-interface", - "lib/wcgi-runner", "lib/c-api/tests/wasmer-c-api-test-runner", "lib/c-api/examples/wasmer-capi-examples-runner", "lib/types", diff --git a/lib/wcgi-runner/Cargo.toml b/lib/wcgi-runner/Cargo.toml deleted file mode 100644 index 56517746845..00000000000 --- a/lib/wcgi-runner/Cargo.toml +++ /dev/null @@ -1,40 +0,0 @@ -[package] -name = "wcgi-runner" -version = "3.2.0-alpha.1" -description = "A runner that can serve WCGI programs locally" -authors = ["Wasmer Engineering Team "] -repository = "https://github.com/wasmerio/wasmer" -license = "MIT" -readme = "README.md" -edition = "2018" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -async-trait = "0.1.64" -bytes = "1.4.0" -futures = "0.3.25" -http = "0.2.8" -hyper = { version = "0.14", features = ["server", "stream"] } -serde = { version = "1.0", features = ["derive"] } -serde_cbor = "0.11" -thiserror = "1.0" -tokio = { version = "1", features = ["rt-multi-thread", "macros"] } -tower-service = "0.3" -tracing = "0.1.37" -wasmer = { version = "3.2.0-alpha.1", path = "../api", default-features = false } -wasmer-vfs = { version = "3.2.0-alpha.1", path = "../vfs", default-features = false } -wasmer-wasi = { version = "3.2.0-alpha.1", path = "../wasi", default-features = false, features = ["sys-default"] } -wcgi = { version = "0.1.1" } -wcgi-host = { version = "0.1.0" } -webc = { version = "5.0.0-rc.5", default-features = false } - -[dev-dependencies] -anyhow = { version = "1", features = ["backtrace"] } -clap = { version = "4", features = ["derive", "env"] } -tempfile = "3.3.0" -tracing-subscriber = { version = "0.3.16", features = ["fmt", "env-filter"] } -ureq = "2.6.2" - -[features] -default = ["wasmer/sys", "wasmer/singlepass"] diff --git a/lib/wcgi-runner/README.md b/lib/wcgi-runner/README.md deleted file mode 100644 index 18dc8a51abf..00000000000 --- a/lib/wcgi-runner/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# wcgi-runner - -A server that can serve [wcgi][wcgi] web servers. - -## Usage - -`wcgi-runner --watch ./path/to/module.wasm` - -[wcgi]: https://github.com/wasmerio/wcgi diff --git a/lib/wcgi-runner/examples/wcgi-runner.rs b/lib/wcgi-runner/examples/wcgi-runner.rs deleted file mode 100644 index f30b12b887d..00000000000 --- a/lib/wcgi-runner/examples/wcgi-runner.rs +++ /dev/null @@ -1,111 +0,0 @@ -use std::{convert::Infallible, net::SocketAddr, path::PathBuf}; - -use anyhow::{Context, Error}; -use clap::Parser; -use tracing_subscriber::fmt::format::FmtSpan; -use wcgi_runner::Runner; - -fn main() -> Result<(), Error> { - if std::env::var("RUST_LOG").is_err() { - std::env::set_var("RUST_LOG", "wcgi_runner=trace,info"); - } - tracing_subscriber::fmt() - .with_span_events(FmtSpan::CLOSE) - .init(); - - let Args { - address, - input, - env_all, - mapped_dirs, - } = Args::parse(); - - // Hack to initialize the global shared tokio task manager handle. - // Prevents "cannot drop runtime in async context" errors, because - // the default will be initialized to the current tokio context. - let rt = wasmer_wasi::runtime::task_manager::tokio::TokioTaskManager::default(); - - let mut builder = Runner::builder() - .map_dirs(mapped_dirs) - .tokio_handle(rt.runtime_handle()); - - if env_all { - builder = builder.forward_host_env(); - } - - let runner = builder - .watch(input) - .context("Unable to create the runner")?; - - let make_service = hyper::service::make_service_fn(move |_| { - let runner = runner.clone(); - async move { Ok::<_, Infallible>(runner) } - }); - - tracing::info!(%address, "Started the server"); - rt.runtime_handle() - .block_on(async { hyper::Server::bind(&address).serve(make_service).await }) - .context("Unable to start the server")?; - - Ok(()) -} - -#[derive(Debug, Clone, Parser)] -#[clap(about, version, author)] -struct Args { - /// Server address. - #[clap(long, short, env, default_value_t = ([127, 0, 0, 1], 8000).into())] - address: SocketAddr, - - /// Map a host directory to a different location for the Wasm module - /// - /// Example: - /// - /// --map-dir /www:./my-website - /// => will make the ./my-website directory available for wazsm at /www - #[clap( - long = "mapdir", - name = "GUEST_DIR:HOST_DIR", - value_parser = parse_mapdir, - )] - mapped_dirs: Vec<(String, PathBuf)>, - - /// Forward all host env variables to the wcgi task. - #[clap(long)] - env_all: bool, - - /// A WCGI program. - input: PathBuf, -} - -/// Parses a mapdir from a string -// NOTE: copied from wasmerio/wasmer lib/cli/src/utils.rs. -pub fn parse_mapdir(entry: &str) -> Result<(String, PathBuf), anyhow::Error> { - fn retrieve_alias_pathbuf( - alias: &str, - real_dir: &str, - ) -> Result<(String, PathBuf), anyhow::Error> { - let pb = PathBuf::from(&real_dir); - if let Ok(pb_metadata) = pb.metadata() { - if !pb_metadata.is_dir() { - anyhow::bail!("\"{real_dir}\" exists, but it is not a directory"); - } - } else { - anyhow::bail!("Directory \"{real_dir}\" does not exist"); - } - Ok((alias.to_string(), pb)) - } - - // We try first splitting by `::` - if let Some((alias, real_dir)) = entry.split_once("::") { - retrieve_alias_pathbuf(alias, real_dir) - } - // And then we try splitting by `:` (for compatibility with previous API) - else if let Some((alias, real_dir)) = entry.split_once(':') { - retrieve_alias_pathbuf(alias, real_dir) - } else { - anyhow::bail!( - "Directory mappings must consist of two paths separate by a `::` or `:`. Found {entry}", - ) - } -} diff --git a/lib/wcgi-runner/src/annotations.rs b/lib/wcgi-runner/src/annotations.rs deleted file mode 100644 index d32cb19d250..00000000000 --- a/lib/wcgi-runner/src/annotations.rs +++ /dev/null @@ -1,24 +0,0 @@ -use wcgi_host::CgiDialect; - -// FIXME(@Michael-F-Bryan): Make this public in the webc crate -#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct WasiCommandAnnotation { - #[serde(default)] - pub atom: Option, - #[serde(default)] - pub package: Option, - #[serde(default)] - pub env: Option>, - #[serde(default)] - pub main_args: Option>, - #[serde(default, rename = "mountAtomInVolume")] - pub mount_atom_in_volume: Option, -} - -// FIXME(@Michael-F-Bryan): Add this to the webc crate and update -// wapm-targz-to-pirita -#[derive(Debug, Default, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] -pub struct WcgiAnnotation { - #[serde(default)] - pub dialect: Option, -} diff --git a/lib/wcgi-runner/src/builder.rs b/lib/wcgi-runner/src/builder.rs deleted file mode 100644 index 3086709cd2f..00000000000 --- a/lib/wcgi-runner/src/builder.rs +++ /dev/null @@ -1,252 +0,0 @@ -use std::{ - collections::HashMap, - path::PathBuf, - sync::{Arc, Mutex}, -}; - -use bytes::Bytes; -use tokio::runtime::Handle; -use wasmer::Engine; -use wcgi_host::CgiDialect; - -use crate::{ - context::Context, - module_loader::{FileLoader, ModuleLoader, WasmLoader, WebcCommand, WebcLoader, WebcOptions}, - Error, Runner, -}; - -/// A builder for initializing a [`Runner`]. -/// -/// # Examples -/// -/// The easiest way to use the builder is by giving it a WEBC file where the -/// default entrypoint is a WCGI command. -/// -/// ```rust,no_run -/// use wcgi_runner::Runner; -/// # #[tokio::main] -/// # async fn main() -> Result<(), Box> { -/// let webc = std::fs::read("path/to/server.webc")?; -/// let runner = Runner::builder().build_webc(webc)?; -/// # Ok(()) -/// # } -#[derive(Default)] -pub struct Builder { - args: Vec, - program: Option, - dialect: Option, - engine: Option, - env: HashMap, - forward_host_env: bool, - mapped_dirs: Vec<(String, PathBuf)>, - tokio_handle: Option, -} - -impl Builder { - pub fn new() -> Self { - Builder::default() - } - - /// Set the name of the program. - pub fn program(self, program: impl Into) -> Self { - Builder { - program: Some(program.into()), - ..self - } - } - - /// Add an argument to the WASI executable's command-line arguments. - pub fn arg(mut self, arg: impl Into) -> Self { - self.args.push(arg.into()); - self - } - - /// Add multiple arguments to the WASI executable's command-line arguments. - pub fn args(mut self, args: A) -> Self - where - A: IntoIterator, - S: Into, - { - self.args.extend(args.into_iter().map(|s| s.into())); - self - } - - /// Expose an environment variable to the guest. - pub fn env(mut self, name: impl Into, value: impl Into) -> Self { - self.env.insert(name.into(), value.into()); - self - } - - /// Expose multiple environment variables to the guest. - pub fn envs(mut self, variables: I) -> Self - where - I: IntoIterator, - K: Into, - V: Into, - { - self.env - .extend(variables.into_iter().map(|(k, v)| (k.into(), v.into()))); - self - } - - /// Forward all of the host's environment variables to the guest. - pub fn forward_host_env(self) -> Self { - Builder { - forward_host_env: true, - ..self - } - } - - /// Override the CGI dialect. - pub fn cgi_dialect(self, dialect: CgiDialect) -> Self { - Builder { - dialect: Some(dialect), - ..self - } - } - - /// Map `guest_dir` to a directory on the host. - pub fn map_dir(mut self, guest_dir: impl Into, host_dir: impl Into) -> Self { - self.mapped_dirs.push((guest_dir.into(), host_dir.into())); - self - } - - /// Map one or more `guest_dir` to a directory on the host. - pub fn map_dirs(mut self, mapping: I) -> Self - where - I: IntoIterator, - G: Into, - H: Into, - { - for (guest_dir, host_dir) in mapping { - self.mapped_dirs.push((guest_dir.into(), host_dir.into())); - } - - self - } - - /// Pass in a [`Handle`] to a custom tokio runtime. - pub fn tokio_handle(self, handle: Handle) -> Self { - Builder { - tokio_handle: Some(handle), - ..self - } - } - - /// Set the engine used to compile the WebAssembly module. - pub fn engine(self, engine: Engine) -> Self { - Builder { - engine: Some(engine), - ..self - } - } - - /// Create a [`Runner`] that executes a WEBC file. - /// - /// If [`Builder::program`] was set, this will look for the command with - /// that name in the WEBC file. Otherwise, it will fall back to the WEBC - /// file's default entrypoint. - /// - /// This will infer the [`CgiDialect`] from the WEBC file's metadata - pub fn build_webc(self, webc: impl Into) -> Result { - let webc = webc.into(); - - let options = WebcOptions { - command: match &self.program { - Some(program) => WebcCommand::Named(program), - None => WebcCommand::Entrypoint, - }, - dialect: self.dialect, - }; - let loader = WebcLoader::new_with_options(&options, webc)?; - - self.build(loader.load_once()) - } - - /// Create a [`Runner`] that executes a WebAssembly module. - /// - /// This requires the [`Builder::program`] to have been set. - /// - /// Unless otherwise specified (i.e. via [`Builder::cgi_dialect`]), the - /// WebAssembly binary will be assumed to implement the [`CgiDialect::Wcgi`] - /// dialect. - pub fn build_wasm(self, wasm: impl Into) -> Result { - let wasm = wasm.into(); - let program = self.program.clone().ok_or(Error::ProgramNameRequired)?; - - let loader = match self.dialect { - Some(dialect) => WasmLoader::new_with_dialect(program, wasm, dialect), - None => WasmLoader::new(program, wasm), - }; - - self.build(loader.load_once()) - } - - /// Create a new [`Runner`] from a particular file and automatically reload - /// whenever that file changes. - pub fn watch(self, path: impl Into) -> Result { - let path = path.into(); - let loader = FileLoader::new(&path).cached(file_has_changed(path)); - - self.build(loader) - } - - fn build(self, loader: impl ModuleLoader + 'static) -> Result { - let Builder { - args, - engine, - mut env, - forward_host_env, - mapped_dirs, - tokio_handle, - .. - } = self; - - if forward_host_env { - env = std::env::vars().chain(env).collect(); - } - - let tokio_handle = tokio_handle.unwrap_or_else(|| { - Handle::try_current().expect("The builder can only be used inside a Tokio context") - }); - - let ctx = Context { - tokio_handle, - engine: engine.unwrap_or_else(|| { - let store = wasmer::Store::default(); - store.engine().clone() - }), - env: Arc::new(env), - args: args.into(), - loader: Box::new(loader), - mapped_dirs, - }; - - Ok(Runner::new(Arc::new(ctx))) - } -} - -fn file_has_changed(path: PathBuf) -> impl Fn() -> bool + Send + Sync + 'static { - let last_modified: Mutex> = Mutex::new(None); - - move || -> bool { - let modified = match path.metadata().and_then(|m| m.modified()).ok() { - Some(m) => m, - None => { - // we couldn't determine the last modified time so be - // conservative mark the cache as invalidated. - return true; - } - }; - - let mut last_modified = last_modified.lock().expect("Poisoned"); - - let invalidated = match *last_modified { - Some(last) => last != modified, - None => true, - }; - - *last_modified = Some(modified); - invalidated - } -} diff --git a/lib/wcgi-runner/src/context.rs b/lib/wcgi-runner/src/context.rs deleted file mode 100644 index c79333a4f07..00000000000 --- a/lib/wcgi-runner/src/context.rs +++ /dev/null @@ -1,226 +0,0 @@ -use std::{ - collections::HashMap, - path::{Path, PathBuf}, - sync::Arc, -}; - -use futures::{Future, StreamExt, TryFutureExt}; -use http::{Request, Response}; -use hyper::Body; -use tokio::{ - io::{AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt}, - runtime::Handle, -}; -use wasmer::Engine; -use wasmer_vfs::{FileSystem, PassthruFileSystem, RootFileSystemBuilder, TmpFileSystem}; -use wasmer_wasi::{http::HttpClientCapabilityV1, Capabilities, Pipe, WasiEnv}; - -use crate::{ - module_loader::{LoadedModule, ModuleLoader, ModuleLoaderContext}, - Error, -}; - -/// The shared object that manages the instantiaion of WASI executables and -/// communicating with them via the CGI protocol. -pub(crate) struct Context { - pub(crate) engine: Engine, - pub(crate) env: Arc>, - pub(crate) args: Arc<[String]>, - pub(crate) mapped_dirs: Vec<(String, PathBuf)>, - pub(crate) tokio_handle: Handle, - pub(crate) loader: Box, -} - -impl Context { - pub(crate) async fn handle(&self, req: Request) -> Result, Error> { - let LoadedModule { - program, - module, - dialect, - } = self - .loader - .load(ModuleLoaderContext::new(&self.engine, &self.tokio_handle)) - .await?; - - let (parts, body) = req.into_parts(); - - let (req_body_sender, req_body_receiver) = Pipe::channel(); - let (res_body_sender, res_body_receiver) = Pipe::channel(); - let (stderr_sender, stderr_receiver) = Pipe::channel(); - - let builder = WasiEnv::builder(program.to_string()); - - let mut request_specific_env = HashMap::new(); - dialect.prepare_environment_variables(parts, &mut request_specific_env); - - let builder = builder - .envs(self.env.iter()) - .envs(request_specific_env) - .args(self.args.iter()) - .stdin(Box::new(req_body_receiver)) - .stdout(Box::new(res_body_sender)) - .stderr(Box::new(stderr_sender)) - .capabilities(Capabilities { - insecure_allow_all: true, - http_client: HttpClientCapabilityV1::new_allow_all(), - }) - .sandbox_fs(self.fs()?) - .preopen_dir(Path::new("/"))?; - - let done = self - .tokio_handle - .spawn_blocking(move || builder.run(module)) - .map_err(Error::from) - .and_then(|r| async { r.map_err(Error::from) }); - - let handle = self.tokio_handle.clone(); - self.tokio_handle.spawn(async move { - if let Err(e) = - drive_request_to_completion(&handle, done, body, req_body_sender, stderr_receiver) - .await - { - tracing::error!( - error = &e as &dyn std::error::Error, - "Unable to drive the request to completion" - ); - } - }); - - let mut res_body_receiver = tokio::io::BufReader::new(res_body_receiver); - - let parts = dialect - .extract_response_header(&mut res_body_receiver) - .await?; - let chunks = futures::stream::try_unfold(res_body_receiver, |mut r| async move { - match r.fill_buf().await { - Ok(chunk) if chunk.is_empty() => Ok(None), - Ok(chunk) => { - let chunk = chunk.to_vec(); - r.consume(chunk.len()); - Ok(Some((chunk, r))) - } - Err(e) => Err(e), - } - }); - let body = hyper::Body::wrap_stream(chunks); - - let response = hyper::Response::from_parts(parts, body); - - Ok(response) - } - - fn fs(&self) -> Result { - let root_fs = RootFileSystemBuilder::new().build(); - - if !self.mapped_dirs.is_empty() { - let fs_backing: Arc = - Arc::new(PassthruFileSystem::new(wasmer_wasi::default_fs_backing())); - - for (src, dst) in &self.mapped_dirs { - let src = match src.starts_with('/') { - true => PathBuf::from(src), - false => Path::new("/").join(src), - }; - tracing::trace!( - source=%src.display(), - alias=%dst.display(), - "mounting directory to instance fs", - ); - - root_fs - .mount(PathBuf::from(&src), &fs_backing, dst.clone()) - .map_err(|error| Error::Mount { - error, - src, - dst: dst.to_path_buf(), - })?; - } - } - Ok(root_fs) - } -} - -/// Drive the request to completion by streaming the request body to the -/// instance and waiting for it to exit. -async fn drive_request_to_completion( - handle: &Handle, - done: impl Future>, - mut request_body: hyper::Body, - mut instance_stdin: impl AsyncWrite + Send + Unpin + 'static, - instance_stderr: impl AsyncRead + Send + Unpin + 'static, -) -> Result<(), Error> { - let request_body_send = handle - .spawn(async move { - // Copy the request into our instance, chunk-by-chunk. If the instance - // dies before we finish writing the body, the instance's side of the - // pipe will be automatically closed and we'll error out. - while let Some(res) = request_body.next().await { - // FIXME(theduke): figure out how to propagate a body error to the - // CGI instance. - let chunk = res?; - instance_stdin - .write_all(chunk.as_ref()) - .await - .map_err(Error::Io)?; - } - - instance_stdin.shutdown().await.map_err(Error::Io)?; - - Ok::<(), Error>(()) - }) - .map_err(Error::from) - .and_then(|r| async { r }); - - handle.spawn(async move { - consume_stderr(instance_stderr).await; - }); - - futures::try_join!(done, request_body_send)?; - - Ok(()) -} - -/// Read the instance's stderr, taking care to preserve output even when WASI -/// pipe errors occur so users still have *something* they use for -/// troubleshooting. -async fn consume_stderr(stderr: impl AsyncRead + Send + Unpin + 'static) { - let mut stderr = tokio::io::BufReader::new(stderr); - - // FIXME: this could lead to unbound memory usage - let mut buffer = Vec::new(); - - // Note: we don't want to just read_to_end() because a reading error - // would cause us to lose all of stderr. At least this way we'll be - // able to show users the partial result. - loop { - match stderr.fill_buf().await { - Ok(chunk) if chunk.is_empty() => { - // EOF - the instance's side of the pipe was closed. - break; - } - Ok(chunk) => { - buffer.extend(chunk); - let bytes_read = chunk.len(); - stderr.consume(bytes_read); - } - Err(e) => { - tracing::error!( - error = &e as &dyn std::error::Error, - bytes_read = buffer.len(), - "Unable to read the complete stderr", - ); - break; - } - } - } - - let stderr = String::from_utf8(buffer).unwrap_or_else(|e| { - tracing::warn!( - error = &e as &dyn std::error::Error, - "Stdout wasn't valid UTF-8", - ); - String::from_utf8_lossy(e.as_bytes()).into_owned() - }); - - tracing::info!(%stderr); -} diff --git a/lib/wcgi-runner/src/errors.rs b/lib/wcgi-runner/src/errors.rs deleted file mode 100644 index bd6655934ba..00000000000 --- a/lib/wcgi-runner/src/errors.rs +++ /dev/null @@ -1,65 +0,0 @@ -use std::path::PathBuf; - -use wasmer_wasi::{FsError, WasiRuntimeError, WasiStateCreationError}; -use webc::Version; - -/// Various errors that can be returned by the WCGI runner. -#[derive(Debug, thiserror::Error)] -#[non_exhaustive] -pub enum Error { - #[error("The provided binary is not in a format known by the runner")] - UnknownFormat, - #[error(transparent)] - Hyper(#[from] hyper::Error), - #[error(transparent)] - Io(std::io::Error), - #[error("Unable to read \"{}\"", path.display())] - File { - #[source] - error: std::io::Error, - path: PathBuf, - }, - #[error(transparent)] - Webc(#[from] WebcLoadError), - #[error("Unable to compile the WebAssembly module")] - Compile(#[from] wasmer::CompileError), - #[error("A spawned task didn't run to completion")] - Join(#[from] tokio::task::JoinError), - #[error("Unable to automatically infer the program name")] - ProgramNameRequired, - #[error("An error occurred while implementing the CGI protocol")] - Cgi(#[from] wcgi_host::CgiError), - #[error("Unable to set up the WASI environment")] - StateCreation(#[from] WasiStateCreationError), - #[error("Could not mount directory mapping: '{src}:{dst}'")] - Mount { - #[source] - error: FsError, - src: PathBuf, - dst: PathBuf, - }, - #[error("Executing the WASI executable failed")] - Exec(#[from] WasiRuntimeError), -} - -/// Errors that may occur when loading a WCGI program from a WEBC file. -#[derive(Debug, thiserror::Error)] -#[non_exhaustive] -pub enum WebcLoadError { - #[error("The WEBC file doesn't contain a \"{name}\" command")] - UnknownCommand { name: String }, - #[error("Unable to find the \"{name}\" atom")] - MissingAtom { name: String }, - #[error("Unable to parse the manifest")] - Manifest(#[from] serde_cbor::Error), - #[error("Unable to detect the WEBC version")] - Detect(#[from] webc::DetectError), - #[error("Unsupported WEBC version, {_0}")] - UnsupportedVersion(Version), - #[error(transparent)] - V1(#[from] webc::v1::Error), - #[error(transparent)] - V2(#[from] webc::v2::read::OwnedReaderError), - #[error("Unable to infer the command to execute")] - UnknownEntrypoint, -} diff --git a/lib/wcgi-runner/src/lib.rs b/lib/wcgi-runner/src/lib.rs deleted file mode 100644 index 32279661917..00000000000 --- a/lib/wcgi-runner/src/lib.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Rust 1.64 doesn't understand tool-specific lints -#![warn(unknown_lints)] -// For now, let's ignore the fact that some of our Error variants are really big -#![allow(clippy::result_large_err)] - -pub mod annotations; -mod builder; -mod context; -mod errors; -mod module_loader; -mod runner; - -pub use crate::{ - builder::Builder, - errors::{Error, WebcLoadError}, - runner::Runner, -}; diff --git a/lib/wcgi-runner/src/module_loader/cached.rs b/lib/wcgi-runner/src/module_loader/cached.rs deleted file mode 100644 index f43b8992968..00000000000 --- a/lib/wcgi-runner/src/module_loader/cached.rs +++ /dev/null @@ -1,48 +0,0 @@ -use tokio::sync::Mutex; - -use crate::{ - module_loader::{ModuleLoader, ModuleLoaderContext}, - Error, -}; - -use super::LoadedModule; - -pub(crate) struct Cached { - loader: L, - invalidated: Box bool + Send + Sync>, - cached: Mutex>, -} - -impl Cached { - pub(crate) fn new(loader: L, invalidated: impl Fn() -> bool + Send + Sync + 'static) -> Self { - Self { - loader, - invalidated: Box::new(invalidated), - cached: Mutex::new(None), - } - } -} - -#[async_trait::async_trait] -impl ModuleLoader for Cached { - async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result { - let mut cached = self.cached.lock().await; - - if (self.invalidated)() { - // Throw away the previous value to make sure we will always load - // the module again, even if invalidated() returns false in the - // future or calling the inner loader fails further down. - let _ = cached.take(); - } - - if let Some(module) = &*cached { - // Cache hit! - return Ok(module.clone()); - } - - let module = self.loader.load(ctx).await?; - *cached = Some(module.clone()); - - Ok(module) - } -} diff --git a/lib/wcgi-runner/src/module_loader/file_loader.rs b/lib/wcgi-runner/src/module_loader/file_loader.rs deleted file mode 100644 index cfdb528e9f9..00000000000 --- a/lib/wcgi-runner/src/module_loader/file_loader.rs +++ /dev/null @@ -1,48 +0,0 @@ -use std::path::PathBuf; - -use bytes::Bytes; - -use crate::{ - module_loader::{LoadedModule, ModuleLoader, ModuleLoaderContext, WasmLoader, WebcLoader}, - Error, -}; - -/// A [`ModuleLoader`] that will read a file to disk and try to interpret it as -/// a known module type. -/// -/// This will re-read the file on every [`ModuleLoader::load()`] call. You may -/// want to add a [`ModuleLoader::cached()`] in front of it to only reload when -/// the file has been changed. -#[derive(Debug, Clone, PartialEq)] -pub(crate) struct FileLoader { - path: PathBuf, -} - -impl FileLoader { - pub(crate) fn new(path: impl Into) -> Self { - FileLoader { path: path.into() } - } -} - -#[async_trait::async_trait] -impl ModuleLoader for FileLoader { - async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result { - let bytes: Bytes = tokio::fs::read(&self.path) - .await - .map_err(|error| Error::File { - error, - path: self.path.clone(), - })? - .into(); - - if webc::detect(bytes.as_ref()).is_ok() { - let loader = WebcLoader::new(bytes)?; - loader.load(ctx).await - } else if wasmer::is_wasm(&bytes) { - let loader = WasmLoader::new(self.path.display().to_string(), bytes); - loader.load(ctx).await - } else { - Err(Error::UnknownFormat) - } - } -} diff --git a/lib/wcgi-runner/src/module_loader/mod.rs b/lib/wcgi-runner/src/module_loader/mod.rs deleted file mode 100644 index 2b5135c03df..00000000000 --- a/lib/wcgi-runner/src/module_loader/mod.rs +++ /dev/null @@ -1,67 +0,0 @@ -mod cached; -mod file_loader; -mod wasm; -mod webc; - -use bytes::Bytes; -use tokio::runtime::Handle; -use wasmer::{Engine, Module}; -use wcgi_host::CgiDialect; - -use crate::Error; - -pub(crate) use self::{ - cached::Cached, - file_loader::FileLoader, - wasm::WasmLoader, - webc::{WebcCommand, WebcLoader, WebcOptions}, -}; - -#[async_trait::async_trait] -pub(crate) trait ModuleLoader: Send + Sync { - async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result; - - /// Wrap this [`ModuleLoader`] with a cache that will only reload the - /// module when `invalidated` returns `true`. - fn cached(self, invalidated: impl Fn() -> bool + Send + Sync + 'static) -> Cached - where - Self: Sized, - { - Cached::new(self, invalidated) - } - - fn load_once(self) -> Cached - where - Self: Sized, - { - Cached::new(self, || false) - } -} - -#[derive(Debug, Clone)] -pub(crate) struct LoadedModule { - pub(crate) program: String, - pub(crate) module: Module, - pub(crate) dialect: CgiDialect, -} - -#[derive(Debug, Clone)] -pub(crate) struct ModuleLoaderContext<'a> { - engine: &'a Engine, - handle: &'a Handle, -} - -impl<'a> ModuleLoaderContext<'a> { - pub(crate) fn new(engine: &'a Engine, handle: &'a Handle) -> Self { - ModuleLoaderContext { engine, handle } - } - - pub(crate) async fn compile_wasm(&self, wasm: Bytes) -> Result { - let engine = self.engine.clone(); - let module = self - .handle - .spawn_blocking(move || Module::new(&engine, &wasm)) - .await??; - Ok(module) - } -} diff --git a/lib/wcgi-runner/src/module_loader/wasm.rs b/lib/wcgi-runner/src/module_loader/wasm.rs deleted file mode 100644 index a7542a99b78..00000000000 --- a/lib/wcgi-runner/src/module_loader/wasm.rs +++ /dev/null @@ -1,40 +0,0 @@ -use bytes::Bytes; - -use wcgi_host::CgiDialect; - -use crate::{ - module_loader::{LoadedModule, ModuleLoader, ModuleLoaderContext}, - Error, -}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct WasmLoader { - program: String, - wasm: Bytes, - dialect: CgiDialect, -} - -impl WasmLoader { - pub fn new(program: String, wasm: Bytes) -> Self { - WasmLoader::new_with_dialect(program, wasm, CgiDialect::Wcgi) - } - - pub fn new_with_dialect(program: String, wasm: Bytes, dialect: CgiDialect) -> Self { - WasmLoader { - program, - wasm, - dialect, - } - } -} - -#[async_trait::async_trait] -impl ModuleLoader for WasmLoader { - async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result { - Ok(LoadedModule { - module: ctx.compile_wasm(self.wasm.clone()).await?, - dialect: self.dialect, - program: self.program.clone(), - }) - } -} diff --git a/lib/wcgi-runner/src/module_loader/webc.rs b/lib/wcgi-runner/src/module_loader/webc.rs deleted file mode 100644 index f10cb845eb0..00000000000 --- a/lib/wcgi-runner/src/module_loader/webc.rs +++ /dev/null @@ -1,155 +0,0 @@ -use bytes::Bytes; -use wcgi_host::CgiDialect; -use webc::{metadata::Manifest, v1::ParseOptions, v2::read::OwnedReader, Version}; - -use crate::{ - annotations::{WasiCommandAnnotation, WcgiAnnotation}, - errors::WebcLoadError, - module_loader::{ModuleLoader, ModuleLoaderContext}, - Error, -}; - -use super::LoadedModule; - -pub(crate) struct WebcLoader { - command: String, - atom: Bytes, - dialect: CgiDialect, -} - -impl WebcLoader { - /// Create a new [`WebcLoader`] which uses the WEBC file's default - /// entrypoint. - pub fn new(webc: impl Into) -> Result { - WebcLoader::new_with_options(&WebcOptions::default(), webc) - } - - pub fn new_with_options( - options: &WebcOptions<'_>, - webc: impl Into, - ) -> Result { - let webc = webc.into(); - match webc::detect(webc.as_ref())? { - Version::V1 => WebcLoader::v1(options, webc), - Version::V2 => WebcLoader::v2(options, webc), - other => Err(WebcLoadError::UnsupportedVersion(other)), - } - } - - fn v1(options: &WebcOptions<'_>, webc: Bytes) -> Result { - let parse_options = ParseOptions::default(); - let webc = webc::v1::WebC::parse(&webc, &parse_options)?; - - let (command, atom, dialect) = get_atom_and_dialect(&webc.manifest, options, |name| { - let pkg = webc.get_package_name(); - webc.get_atom(&pkg, name).ok().map(|b| b.to_vec().into()) - })?; - - Ok(WebcLoader { - command, - atom, - dialect, - }) - } - - fn v2(options: &WebcOptions<'_>, webc: Bytes) -> Result { - let webc = OwnedReader::parse(webc)?; - let (command, atom, dialect) = get_atom_and_dialect(webc.manifest(), options, |name| { - webc.get_atom(name).cloned() - })?; - - Ok(WebcLoader { - command, - atom, - dialect, - }) - } -} - -#[async_trait::async_trait] -impl ModuleLoader for WebcLoader { - async fn load(&self, ctx: ModuleLoaderContext<'_>) -> Result { - Ok(LoadedModule { - module: ctx.compile_wasm(self.atom.clone()).await?, - dialect: self.dialect, - program: self.command.clone(), - }) - } -} - -fn get_atom_and_dialect( - manifest: &Manifest, - options: &WebcOptions<'_>, - get_atom: impl FnOnce(&str) -> Option, -) -> Result<(String, Bytes, CgiDialect), WebcLoadError> { - let command = options - .command - .as_str() - .or(manifest.entrypoint.as_deref()) - .ok_or(WebcLoadError::UnknownEntrypoint)?; - - let cmd = manifest - .commands - .get(command) - .ok_or_else(|| WebcLoadError::UnknownCommand { - name: command.to_string(), - })?; - - let wcgi_annotations: WcgiAnnotation = cmd - .annotations - .get("wcgi") - .cloned() - .and_then(|a| serde_cbor::value::from_value(a).ok()) - .unwrap_or_default(); - - let wasi_annotations: WasiCommandAnnotation = cmd - .annotations - .get("wasi") - .cloned() - .and_then(|a| serde_cbor::value::from_value(a).ok()) - .unwrap_or_default(); - - // Note: Not all WCGI binaries have "wcgi" annotations (e.g. because they - // were published before "wasmer.toml" started using it), so we fall back - // to using the command as our atom name. - let atom_name = wasi_annotations.atom.as_deref().unwrap_or(command); - - let atom = get_atom(atom_name).ok_or_else(|| WebcLoadError::MissingAtom { - name: atom_name.to_string(), - })?; - - // Note: We explicitly use WCGI instead of CgiDialect::default() so we can - // use existing WCGI packages. We also prefer the user-provided CgiDialect - // so they can work around packages with bad metadata. - - Ok(( - command.to_string(), - atom, - options - .dialect - .or(wcgi_annotations.dialect) - .unwrap_or(CgiDialect::Wcgi), - )) -} - -#[derive(Debug, Default, Clone)] -pub(crate) struct WebcOptions<'a> { - pub command: WebcCommand<'a>, - pub dialect: Option, -} - -#[derive(Debug, Default, Copy, Clone)] -pub(crate) enum WebcCommand<'a> { - #[default] - Entrypoint, - Named(&'a str), -} - -impl<'a> WebcCommand<'a> { - fn as_str(self) -> Option<&'a str> { - match self { - WebcCommand::Entrypoint => None, - WebcCommand::Named(name) => Some(name), - } - } -} diff --git a/lib/wcgi-runner/src/runner.rs b/lib/wcgi-runner/src/runner.rs deleted file mode 100644 index bc16de9064a..00000000000 --- a/lib/wcgi-runner/src/runner.rs +++ /dev/null @@ -1,87 +0,0 @@ -use std::{future::Future, pin::Pin, sync::Arc, task::Poll}; - -use futures::FutureExt; -use http::{Request, Response}; -use hyper::Body; -use tower_service::Service; - -use crate::{context::Context, Builder, Error}; - -/// A runner for WCGI binaries. -/// -/// # Examples -/// -/// [`Runner`] implements the [`Service`] trait and is cheaply cloneable -/// so it can be easily integrated with a Hyper server and the Tower ecosystem. -/// -/// ```rust,no_run -/// use std::net::SocketAddr; -/// use wcgi_runner::Runner; -/// -/// # #[tokio::main] -/// # async fn main() -> Result<(), Box> { -/// let webc: &[u8] = b"..."; -/// let address: SocketAddr = ([127, 0, 0, 1], 3000).into(); -/// let runner = Runner::builder().build_webc(webc)?; -/// -/// let make_service = hyper::service::make_service_fn(move |_| { -/// let runner = runner.clone(); -/// async move { Ok::<_, std::convert::Infallible>(runner) } -/// }); -/// -/// hyper::Server::bind(&address).serve(make_service).await?; -/// # Ok(()) -/// # } -/// ``` -#[derive(Clone)] -pub struct Runner { - ctx: Arc, -} - -impl Runner { - pub(crate) fn new(ctx: Arc) -> Self { - Runner { ctx } - } - - /// Create a [`Builder`] that can be used to configure a [`Runner`]. - pub fn builder() -> Builder { - Builder::new() - } - - /// Handle a single HTTP request. - pub async fn handle(&self, request: Request) -> Result, Error> { - self.ctx.handle(request).await - } -} - -impl Service> for Runner { - type Response = Response; - type Error = Error; - type Future = Pin, Error>> + Send>>; - - fn poll_ready(&mut self, _cx: &mut std::task::Context<'_>) -> Poll> { - // TODO: We probably should implement some sort of backpressure here... - Poll::Ready(Ok(())) - } - - fn call(&mut self, request: Request) -> Self::Future { - let ctx = Arc::clone(&self.ctx); - let fut = async move { ctx.handle(request).await }; - fut.boxed() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - #[allow(clippy::extra_unused_type_parameters)] - fn send_and_sync() { - fn assert_send() {} - fn assert_sync() {} - - assert_send::(); - assert_sync::(); - } -} diff --git a/lib/wcgi-runner/tests/integration.rs b/lib/wcgi-runner/tests/integration.rs deleted file mode 100644 index 8164ed2e809..00000000000 --- a/lib/wcgi-runner/tests/integration.rs +++ /dev/null @@ -1,95 +0,0 @@ -use std::path::Path; - -use bytes::Bytes; -use http::{Request, StatusCode}; -use hyper::{body::HttpBody, Body}; -use tempfile::TempDir; -use wcgi_host::CgiDialect; -use wcgi_runner::Builder; - -const FERRIS_SAYS: &str = "https://registry-cdn.wapm.dev/packages/wasmer-examples/ferris-says/ferris-says-0.2.0-2f5dfb76-77a9-11ed-a646-d2c429a5b858.webc"; -const STATICSERVER: &str = - "https://registry-cdn.wapm.dev/contents/syrusakbary/staticserver/1.0.2/serve.wasm"; - -#[tokio::test] -async fn execute_a_webc_server() { - let ferris_says = cached(FERRIS_SAYS); - - let runner = Builder::default() - .cgi_dialect(CgiDialect::Wcgi) - .build_webc(ferris_says) - .unwrap(); - let req = Request::new(Body::default()); - let response = runner.handle(req).await.unwrap(); - - let (parts, mut body) = response.into_parts(); - assert_eq!(parts.status, StatusCode::OK); - let mut buffer = Vec::new(); - while let Some(result) = body.data().await { - let chunk = result.unwrap(); - buffer.extend(chunk); - } - let body = String::from_utf8(buffer).unwrap(); - assert!(body.contains("Wasmer Deploy <3 Rustaceans!")); -} - -#[tokio::test] -async fn execute_a_webassembly_server_with_mounted_directories() { - let wasm = cached(STATICSERVER); - let temp = TempDir::new().unwrap(); - let example = temp.path().join("example"); - std::fs::create_dir_all(&example).unwrap(); - std::fs::write(example.join("file.txt"), b"Hello, World!").unwrap(); - - let runner = Builder::default() - .program("staticserver") - .map_dir("example", example) - .build_wasm(wasm) - .unwrap(); - let req = Request::builder() - .uri("/example/file.txt") - .body(Body::default()) - .unwrap(); - let response = runner.handle(req).await.unwrap(); - - let (parts, mut body) = response.into_parts(); - assert_eq!(parts.status, StatusCode::OK); - let mut buffer = Vec::new(); - while let Some(result) = body.data().await { - let chunk = result.unwrap(); - buffer.extend(chunk); - } - let body = String::from_utf8(buffer).unwrap(); - assert!(body.contains("Hello, World!")); -} - -/// Download a file, caching it in $CARGO_TARGET_TMPDIR to avoid unnecessary -/// downloads. -fn cached(url: &str) -> Bytes { - let uri: http::Uri = url.parse().unwrap(); - - let file_name = Path::new(uri.path()).file_name().unwrap(); - let cache_dir = Path::new(env!("CARGO_TARGET_TMPDIR")).join(env!("CARGO_PKG_NAME")); - let cached_path = cache_dir.join(file_name); - - if cached_path.exists() { - return std::fs::read(&cached_path).unwrap().into(); - } - - let response = ureq::get(url).call().unwrap(); - assert_eq!( - response.status(), - 200, - "Unable to get \"{url}\": {} {}", - response.status(), - response.status_text() - ); - - let mut body = Vec::new(); - response.into_reader().read_to_end(&mut body).unwrap(); - - std::fs::create_dir_all(&cache_dir).unwrap(); - std::fs::write(&cached_path, &body).unwrap(); - - body.into() -} From cb0cea4653345a8ad9442b9c268ca4c421028e14 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Mar 2023 00:58:52 +0800 Subject: [PATCH 10/26] Got the WASI and WCGI tests passing --- lib/wasi/src/runners/container.rs | 3 +- lib/wasi/src/runners/wcgi/handler.rs | 51 +++++-------- lib/wasi/src/runners/wcgi/mod.rs | 2 +- lib/wasi/src/runners/wcgi/runner.rs | 59 ++++++++++----- lib/wasi/tests/runners.rs | 108 ++++++++++++++++++--------- 5 files changed, 134 insertions(+), 89 deletions(-) diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index 2eb01baef5c..a80a4351e24 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -97,7 +97,8 @@ impl WapmContainer { } } - pub(crate) fn volume_fs(&self, package_name: &str) -> Box { + /// Load a volume as a [`FileSystem`] node. + pub fn volume_fs(&self, package_name: &str) -> Box { match &self.repr { Repr::V1Mmap(mapped) => { Box::new(WebcFileSystem::init(Arc::clone(mapped), package_name)) diff --git a/lib/wasi/src/runners/wcgi/handler.rs b/lib/wasi/src/runners/wcgi/handler.rs index c0c419fe771..38444b52926 100644 --- a/lib/wasi/src/runners/wcgi/handler.rs +++ b/lib/wasi/src/runners/wcgi/handler.rs @@ -19,13 +19,15 @@ use wasmer_vfs::{FileSystem, PassthruFileSystem, RootFileSystemBuilder, TmpFileS use wcgi_host::CgiDialect; use crate::{ - http::HttpClientCapabilityV1, runners::wcgi::MappedDirectory, Capabilities, Pipe, - PluggableRuntimeImplementation, VirtualTaskManager, WasiEnv, + http::HttpClientCapabilityV1, + runners::wcgi::{Callbacks, MappedDirectory}, + Capabilities, Pipe, PluggableRuntimeImplementation, VirtualTaskManager, WasiEnv, }; /// The shared object that manages the instantiaion of WASI executables and /// communicating with them via the CGI protocol. -#[derive(Debug, Clone)] +#[derive(Clone, derivative::Derivative)] +#[derivative(Debug)] pub(crate) struct Handler { pub(crate) program: Arc, pub(crate) env: Arc>, @@ -34,6 +36,8 @@ pub(crate) struct Handler { pub(crate) task_manager: Arc, pub(crate) module: Module, pub(crate) dialect: CgiDialect, + #[derivative(Debug = "ignore")] + pub(crate) callbacks: Arc, } impl Handler { @@ -76,10 +80,14 @@ impl Handler { .and_then(|r| async { r.map_err(Error::from) }); let handle = self.task_manager.runtime().clone(); + let callbacks = Arc::clone(&self.callbacks); + + handle.spawn(async move { + consume_stderr(stderr_receiver, callbacks).await; + }); + self.task_manager.runtime().spawn(async move { - if let Err(e) = - drive_request_to_completion(&handle, done, body, req_body_sender, stderr_receiver) - .await + if let Err(e) = drive_request_to_completion(&handle, done, body, req_body_sender).await { tracing::error!( error = &*e as &dyn std::error::Error, @@ -152,7 +160,6 @@ async fn drive_request_to_completion( done: impl Future>, mut request_body: hyper::Body, mut instance_stdin: impl AsyncWrite + Send + Unpin + 'static, - instance_stderr: impl AsyncRead + Send + Unpin + 'static, ) -> Result<(), Error> { let request_body_send = handle .spawn(async move { @@ -173,10 +180,6 @@ async fn drive_request_to_completion( .map_err(Error::from) .and_then(|r| async { r }); - handle.spawn(async move { - consume_stderr(instance_stderr).await; - }); - futures::try_join!(done, request_body_send)?; Ok(()) @@ -185,12 +188,12 @@ async fn drive_request_to_completion( /// Read the instance's stderr, taking care to preserve output even when WASI /// pipe errors occur so users still have *something* they use for /// troubleshooting. -async fn consume_stderr(stderr: impl AsyncRead + Send + Unpin + 'static) { +async fn consume_stderr( + stderr: impl AsyncRead + Send + Unpin + 'static, + callbacks: Arc, +) { let mut stderr = tokio::io::BufReader::new(stderr); - // FIXME: this could lead to unbound memory usage - let mut buffer = Vec::new(); - // Note: we don't want to just read_to_end() because a reading error // would cause us to lose all of stderr. At least this way we'll be // able to show users the partial result. @@ -201,30 +204,16 @@ async fn consume_stderr(stderr: impl AsyncRead + Send + Unpin + 'static) { break; } Ok(chunk) => { - buffer.extend(chunk); + callbacks.on_stderr(chunk); let bytes_read = chunk.len(); stderr.consume(bytes_read); } Err(e) => { - tracing::error!( - error = &e as &dyn std::error::Error, - bytes_read = buffer.len(), - "Unable to read the complete stderr", - ); + callbacks.on_stderr_error(e); break; } } } - - let stderr = String::from_utf8(buffer).unwrap_or_else(|e| { - tracing::warn!( - error = &e as &dyn std::error::Error, - "Stdout wasn't valid UTF-8", - ); - String::from_utf8_lossy(e.as_bytes()).into_owned() - }); - - tracing::info!(%stderr); } impl Service> for Handler { diff --git a/lib/wasi/src/runners/wcgi/mod.rs b/lib/wasi/src/runners/wcgi/mod.rs index 4b3df172778..a160c3c87b3 100644 --- a/lib/wasi/src/runners/wcgi/mod.rs +++ b/lib/wasi/src/runners/wcgi/mod.rs @@ -3,7 +3,7 @@ mod runner; use std::path::PathBuf; -pub use self::runner::WcgiRunner; +pub use self::runner::{Config, WcgiRunner, Callbacks}; #[derive(Debug, Clone, PartialEq)] pub(crate) struct MappedDirectory { diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index e2277b147e2..b5da3c7cc66 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -1,7 +1,7 @@ use std::{collections::HashMap, convert::Infallible, net::SocketAddr, path::PathBuf, sync::Arc}; use anyhow::{Context, Error}; -use futures::channel::oneshot::Receiver; +use futures::future::AbortHandle; use wasmer::{Engine, Module, Store}; use wasmer_vfs::FileSystem; use wcgi_host::CgiDialect; @@ -43,26 +43,25 @@ impl WcgiRunner { let address = self.config.addr; tracing::info!(%address, "Starting the server"); - let abort = self.config.abort.take(); + let callbacks = Arc::clone(&self.config.callbacks); task_manager .block_on(async { - let server = hyper::Server::bind(&address).serve(make_service); - - match abort { - Some(abort) => { - server - .with_graceful_shutdown(async move { - let _ = abort.await; - }) - .await - } - None => server.await, - } + let (shutdown, abort_handle) = + futures::future::abortable(futures::future::pending::<()>()); + + callbacks.started(abort_handle); + + hyper::Server::bind(&address) + .serve(make_service) + .with_graceful_shutdown(async { + let _ = shutdown.await; + }) + .await }) .context("Unable to start the server")?; - todo!(); + Ok(()) } } @@ -128,6 +127,7 @@ impl WcgiRunner { .unwrap_or_else(|| Arc::new(TokioTaskManager::default())), module, dialect, + callbacks: Arc::clone(&self.config.callbacks), }; Ok(handler) @@ -200,7 +200,8 @@ impl crate::runners::Runner for WcgiRunner { } } -#[derive(Debug)] +#[derive(derivative::Derivative)] +#[derivative(Debug)] pub struct Config { task_manager: Option>, addr: SocketAddr, @@ -208,7 +209,8 @@ pub struct Config { env: HashMap, forward_host_env: bool, mapped_dirs: Vec, - abort: Option>, + #[derivative(Debug = "ignore")] + callbacks: Arc, } impl Config { @@ -274,8 +276,10 @@ impl Config { self } - pub fn abort_channel(&mut self, rx: Receiver<()>) -> &mut Self { - self.abort = Some(rx); + /// Set callbacks that will be triggered at various points in the runner's + /// lifecycle. + pub fn callbacks(&mut self, callbacks: impl Callbacks + Send + Sync + 'static) -> &mut Self { + self.callbacks = Arc::new(callbacks); self } } @@ -289,7 +293,22 @@ impl Default for Config { forward_host_env: false, mapped_dirs: Vec::new(), args: Vec::new(), - abort: None, + callbacks: Arc::new(NoopCallbacks), } } } + +/// Callbacks that are triggered at various points in the lifecycle of a runner +/// and any WebAssembly instances it may start. +pub trait Callbacks: Send + Sync + 'static { + /// A callback that is called whenever the server starts. + fn started(&self, _abort: AbortHandle) {} + /// Data was written to stderr by an instance. + fn on_stderr(&self, _stderr: &[u8]) {} + /// Reading from stderr failed. + fn on_stderr_error(&self, _error: std::io::Error) {} +} + +struct NoopCallbacks; + +impl Callbacks for NoopCallbacks {} diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs index ef252de6da3..dd7027088d9 100644 --- a/lib/wasi/tests/runners.rs +++ b/lib/wasi/tests/runners.rs @@ -1,7 +1,8 @@ #![cfg(feature = "webc_runner")] -use std::path::Path; +use std::{path::Path, time::Duration}; +use once_cell::sync::Lazy; use reqwest::Client; use wasmer_wasi::runners::{Runner, WapmContainer}; @@ -10,7 +11,7 @@ mod wasi { use tokio::runtime::Handle; use wasmer::Store; use wasmer_wasi::{ - runners::wasi::WasiRunner, runtime::task_manager::tokio::TokioTaskManager, WasiRuntimeError, + runners::wasi::WasiRunner, runtime::task_manager::tokio::TokioTaskManager, WasiError, }; use super::*; @@ -24,21 +25,30 @@ mod wasi { // Note: we don't have any way to intercept stdin or stdout, so blindly // assume that everything is fine if it runs successfully. - let err = WasiRunner::new(store) - .with_task_manager(tasks) - .run_cmd(&container, "wat2wasm") - .unwrap_err(); + let handle = std::thread::spawn(move || { + WasiRunner::new(store) + .with_task_manager(tasks) + .run_cmd(&container, "wat2wasm") + }); + let err = handle.join().unwrap().unwrap_err(); - let runtime_error: &WasiRuntimeError = err.downcast().unwrap(); - let exit_code = runtime_error.as_exit_code().unwrap(); + let runtime_error = err + .chain() + .find_map(|e| e.downcast_ref::()) + .unwrap(); + let exit_code = match runtime_error { + WasiError::Exit(code) => *code, + _ => unreachable!(), + }; assert_eq!(exit_code, 1); } } #[cfg(feature = "webc_runner_rt_wcgi")] mod wcgi { - use std::thread::JoinHandle; + use std::future::Future; + use futures::{channel::mpsc::Sender, future::AbortHandle, SinkExt, StreamExt}; use rand::Rng; use tokio::runtime::Handle; use wasmer_wasi::{runners::wcgi::WcgiRunner, runtime::task_manager::tokio::TokioTaskManager}; @@ -52,46 +62,64 @@ mod wcgi { let container = WapmContainer::from_bytes(webc).unwrap(); let mut runner = WcgiRunner::new("staticserver"); let port = rand::thread_rng().gen_range(10000_u16..65535_u16); - let port = 12345; - let (tx, rx) = futures::channel::oneshot::channel(); + let (cb, started) = callbacks(Handle::current()); runner .config() .addr(([127, 0, 0, 1], port).into()) .task_manager(tasks) - .abort_channel(rx); - // Note: the server blocks, so spin it up in a background thread and kill it - // after we've made our request. - let _guard = thread_spawn(move || { + .callbacks(cb); + + // The server blocks so we need to start it on a background thread. + std::thread::spawn(move || { runner.run_cmd(&container, "wcgi").unwrap(); }); - // The way we test this is by fetching "/" and checking it contains - // something we expect - let resp = reqwest::get(format!("http://localhost:{port}/index.html")) + // wait for the server to have started + let abort_handle = started.await; + + // Now the server is running, we can check that it is working by + // fetching "/" and checking for known content + let resp = client() + .get(format!("http://localhost:{port}/")) + .send() .await .unwrap(); let body = resp.error_for_status().unwrap().text().await.unwrap(); - assert!(body.contains("asdf"), "{}", body); + assert!(body.contains("Index of /"), "{}", body); + + // Make sure the server is shutdown afterwards + abort_handle.abort(); + } + + fn callbacks(handle: Handle) -> (Callbacks, impl Future) { + let (sender, mut rx) = futures::channel::mpsc::channel(1); - // Make sure we shut the server down afterwards - drop(tx); + let cb = Callbacks { sender, handle }; + let fut = async move { rx.next().await.unwrap() }; + + (cb, fut) + } + + struct Callbacks { + sender: Sender, + handle: Handle, } - fn thread_spawn(f: impl FnOnce() + Send + 'static) -> impl Drop { - struct JoinOnDrop(Option>); - impl Drop for JoinOnDrop { - fn drop(&mut self) { - if let Err(e) = self.0.take().unwrap().join() { - if !std::thread::panicking() { - std::panic::resume_unwind(e); - } - } - } + impl wasmer_wasi::runners::wcgi::Callbacks for Callbacks { + fn started(&self, abort: futures::stream::AbortHandle) { + let mut sender = self.sender.clone(); + self.handle.spawn(async move { + sender.send(abort).await.unwrap(); + }); } - let handle = std::thread::spawn(f); - JoinOnDrop(Some(handle)) + fn on_stderr(&self, stderr: &[u8]) { + panic!( + "Something was written to stderr: {}", + String::from_utf8_lossy(stderr) + ); + } } } @@ -106,9 +134,7 @@ async fn download_cached(url: &str) -> bytes::Bytes { return std::fs::read(&cached_path).unwrap().into(); } - let client = Client::new(); - - let response = client + let response = client() .get(url) .header("Accept", "application/webc") .send() @@ -129,3 +155,13 @@ async fn download_cached(url: &str) -> bytes::Bytes { body } + +fn client() -> Client { + static CLIENT: Lazy = Lazy::new(|| { + Client::builder() + .connect_timeout(Duration::from_secs(30)) + .build() + .unwrap() + }); + CLIENT.clone() +} From 649c808afea1f33cc8455f0507c86505fddd12ac Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Mar 2023 02:21:10 +0800 Subject: [PATCH 11/26] Miscellaneous refactoring and tidy-ups --- Cargo.lock | 1 - lib/c-api/Cargo.toml | 39 ++++------ lib/cli/Cargo.toml | 107 ++++++++++++++++------------ lib/cli/src/commands/run.rs | 2 +- lib/registry/Cargo.toml | 10 ++- lib/vfs/Cargo.toml | 12 +++- lib/wasi/src/runners/container.rs | 7 +- lib/wasi/src/runners/emscripten.rs | 21 +++--- lib/wasi/src/runners/wasi.rs | 35 ++++----- lib/wasi/src/runners/wcgi/runner.rs | 93 +++++++++++++++++------- 10 files changed, 190 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 091617d391b..e8a06b68be9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5215,7 +5215,6 @@ dependencies = [ "lazy_static", "libc", "pin-project-lite", - "serde", "slab", "thiserror", "tokio", diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index 3760dfbc2cf..a498e930dd3 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -16,20 +16,25 @@ edition = "2018" # `libwasmer.so`, `libwasmer.dylib`, `wasmer.dll` etc. But it creates # a conflict with the existing `wasmer` crate, see below. name = "wasmer" # ##lib.name## - # ^ DO NOT REMOVE, it's used the `Makefile`, see `build-docs-capi`. +# ^ DO NOT REMOVE, it's used the `Makefile`, see `build-docs-capi`. crate-type = ["staticlib", "cdylib"] #"cdylib", "rlib", "staticlib"] [dependencies] # We rename `wasmer` to `wasmer-api` to avoid the conflict with this # library name (see `[lib]`). -wasmer-api = { version = "=3.2.0-alpha.1", path = "../api", default-features = false, features = ["sys"], package = "wasmer" } +wasmer-api = { version = "=3.2.0-alpha.1", path = "../api", default-features = false, features = [ + "sys", +], package = "wasmer" } wasmer-compiler-cranelift = { version = "=3.2.0-alpha.1", path = "../compiler-cranelift", optional = true } wasmer-compiler-singlepass = { version = "=3.2.0-alpha.1", path = "../compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=3.2.0-alpha.1", path = "../compiler-llvm", optional = true } wasmer-emscripten = { version = "=3.2.0-alpha.1", path = "../emscripten", optional = true } wasmer-compiler = { version = "=3.2.0-alpha.1", path = "../compiler" } wasmer-middlewares = { version = "=3.2.0-alpha.1", path = "../middlewares", optional = true } -wasmer-wasi = { version = "=3.2.0-alpha.1", path = "../wasi", features = ["host-fs", "host-vnet"], optional = true } +wasmer-wasi = { version = "=3.2.0-alpha.1", path = "../wasi", features = [ + "host-fs", + "host-vnet", +], optional = true } wasmer-types = { version = "=3.2.0-alpha.1", path = "../types" } wasmer-vfs = { version = "=3.2.0-alpha.1", path = "../vfs", optional = true, default-features = false, features = ["static-fs"] } webc = { version = "5.0.0-rc.5", optional = true } @@ -51,19 +56,10 @@ wasmer-inline-c = "0.1.1" inline-c = "0.1.7" [features] -default = [ - "wat", - "cranelift", - "compiler", - "wasi", - "middlewares", -] +default = ["wat", "cranelift", "compiler", "wasi", "middlewares"] wat = ["wasmer-api/wat"] wasi = ["wasmer-wasi"] -middlewares = [ - "compiler", - "wasmer-middlewares", -] +middlewares = ["compiler", "wasmer-middlewares"] compiler = [ "wasmer-api/compiler", "wasmer-compiler/translator", @@ -76,18 +72,9 @@ compiler-headless = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", ] -singlepass = [ - "wasmer-compiler-singlepass", - "compiler", -] -cranelift = [ - "wasmer-compiler-cranelift", - "compiler", -] -llvm = [ - "wasmer-compiler-llvm", - "compiler", -] +singlepass = ["wasmer-compiler-singlepass", "compiler"] +cranelift = ["wasmer-compiler-cranelift", "compiler"] +llvm = ["wasmer-compiler-llvm", "compiler"] wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] static-artifact-load = ["wasmer-compiler/static-artifact-load"] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 2d72e3890ff..368b7ab36a6 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -26,22 +26,30 @@ required-features = ["headless"] [dependencies] wasmer = { version = "=3.2.0-alpha.1", path = "../api", default-features = false } -wasmer-compiler = { version = "=3.2.0-alpha.1", path = "../compiler", features = ["compiler", ] } +wasmer-compiler = { version = "=3.2.0-alpha.1", path = "../compiler", features = [ + "compiler", +] } wasmer-compiler-cranelift = { version = "=3.2.0-alpha.1", path = "../compiler-cranelift", optional = true } wasmer-compiler-singlepass = { version = "=3.2.0-alpha.1", path = "../compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=3.2.0-alpha.1", path = "../compiler-llvm", optional = true } wasmer-emscripten = { version = "=3.2.0-alpha.1", path = "../emscripten", optional = true } wasmer-vm = { version = "=3.2.0-alpha.1", path = "../vm" } wasmer-wasi = { version = "=3.2.0-alpha.1", path = "../wasi", optional = true } -wasmer-wasi-experimental-io-devices = { version = "=3.2.0-alpha.1", path = "../wasi-experimental-io-devices", optional = true, features = ["link_external_libs"] } +wasmer-wasi-experimental-io-devices = { version = "=3.2.0-alpha.1", path = "../wasi-experimental-io-devices", optional = true, features = [ + "link_external_libs", +] } wasmer-wasi-local-networking = { version = "=3.2.0-alpha.1", path = "../wasi-local-networking", optional = true } wasmer-wast = { version = "=3.2.0-alpha.1", path = "../../tests/lib/wast", optional = true } wasmer-cache = { version = "=3.2.0-alpha.1", path = "../cache", optional = true } -wasmer-types = { version = "=3.2.0-alpha.1", path = "../types", features = ["enable-serde"] } +wasmer-types = { version = "=3.2.0-alpha.1", path = "../types", features = [ + "enable-serde", +] } wasmer-registry = { version = "=4.0.0", path = "../registry" } wasmer-object = { version = "=3.2.0-alpha.1", path = "../object", optional = true } -wasmer-vfs = { version = "=3.2.0-alpha.1", path = "../vfs", default-features = false, features = ["host-fs"] } -wasmer-vnet = { version = "=3.2.0-alpha.1", path = "../vnet" } +wasmer-vfs = { version = "=3.2.0-alpha.1", path = "../vfs", default-features = false, features = [ + "host-fs", +] } +wasmer-vnet = { version = "=3.2.0-alpha.1", path = "../vnet" } wasmer-wasm-interface = { version = "3.2.0-alpha.1", path = "../wasm-interface" } wasmparser = "0.51.4" atty = "0.2" @@ -56,7 +64,7 @@ bytesize = "1.0" cfg-if = "1.0" # For debug feature fern = { version = "0.6", features = ["colored"], optional = true } -tempfile = "3.4.0" +tempfile = "3" http_req = { version="^0.8", default-features = false, features = ["rust-tls"] } reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json", "multipart"] } serde = { version = "1.0.147", features = ["derive"] } @@ -82,7 +90,11 @@ cargo_metadata = "0.15.2" rusqlite = { version = "0.28.0", features = ["bundled"] } tar = "0.4.38" thiserror = "1.0.37" -time = { version = "0.3.17", default-features = false, features = ["parsing", "std", "formatting"] } +time = { version = "0.3.17", default-features = false, features = [ + "parsing", + "std", + "formatting", +] } log = "0.4.17" minisign = "0.7.2" semver = "1.0.14" @@ -93,7 +105,10 @@ object = "0.30.0" wasm-coredump-builder = { version = "0.1.11" } [build-dependencies] -chrono = { version = "^0.4", default-features = false, features = [ "std", "clock" ] } +chrono = { version = "^0.4", default-features = false, features = [ + "std", + "clock", +] } [target.'cfg(target_os = "linux")'.dependencies] unix_mode = "0.1.3" @@ -118,51 +133,51 @@ wast = ["wasmer-wast"] wasi = ["wasmer-wasi", "wasmer-wasi-local-networking"] emscripten = ["wasmer-emscripten"] wat = ["wasmer/wat"] -webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "wasmer-wasi/webc_runner_rt_wcgi", "nuke-dir", "webc"] +webc_runner = [ + "wasi", + "wasmer-wasi/webc_runner", + "wasmer-wasi/webc_runner_rt_wasi", + "wasmer-wasi/webc_runner_rt_emscripten", + "wasmer-wasi/webc_runner_rt_wcgi", + "nuke-dir", + "webc", +] compiler = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", - "wasmer-wasi/compiler" + "wasmer-wasi/compiler", ] -wasmer-artifact-create = ["compiler", - "wasmer/wasmer-artifact-load", - "wasmer/wasmer-artifact-create", - "wasmer-compiler/wasmer-artifact-load", - "wasmer-compiler/wasmer-artifact-create", - "wasmer-object", - ] -static-artifact-create = ["compiler", - "wasmer/static-artifact-load", - "wasmer/static-artifact-create", - "wasmer-compiler/static-artifact-load", - "wasmer-compiler/static-artifact-create", - "wasmer-object", - ] -wasmer-artifact-load = ["compiler", - "wasmer/wasmer-artifact-load", - "wasmer-compiler/wasmer-artifact-load", - ] -static-artifact-load = ["compiler", - "wasmer/static-artifact-load", - "wasmer-compiler/static-artifact-load", - ] - -experimental-io-devices = [ - "wasmer-wasi-experimental-io-devices", - "wasi" +wasmer-artifact-create = [ + "compiler", + "wasmer/wasmer-artifact-load", + "wasmer/wasmer-artifact-create", + "wasmer-compiler/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-create", + "wasmer-object", ] -singlepass = [ - "wasmer-compiler-singlepass", +static-artifact-create = [ "compiler", + "wasmer/static-artifact-load", + "wasmer/static-artifact-create", + "wasmer-compiler/static-artifact-load", + "wasmer-compiler/static-artifact-create", + "wasmer-object", ] -cranelift = [ - "wasmer-compiler-cranelift", +wasmer-artifact-load = [ "compiler", + "wasmer/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-load", ] -llvm = [ - "wasmer-compiler-llvm", +static-artifact-load = [ "compiler", + "wasmer/static-artifact-load", + "wasmer-compiler/static-artifact-load", ] + +experimental-io-devices = ["wasmer-wasi-experimental-io-devices", "wasi"] +singlepass = ["wasmer-compiler-singlepass", "compiler"] +cranelift = ["wasmer-compiler-cranelift", "compiler"] +llvm = ["wasmer-compiler-llvm", "compiler"] debug = ["fern", "wasmer-wasi/logging"] disable-all-logging = ["wasmer-wasi/disable-all-logging", "log/release_max_level_off"] headless = [] @@ -170,10 +185,10 @@ headless-minimal = ["headless", "disable-all-logging", "wasi"] # Optional enable-serde = [ - "wasmer/enable-serde", - "wasmer-vm/enable-serde", - "wasmer-compiler/enable-serde", - "wasmer-wasi/enable-serde", + "wasmer/enable-serde", + "wasmer-vm/enable-serde", + "wasmer-compiler/enable-serde", + "wasmer-wasi/enable-serde", ] [target.'cfg(target_os = "windows")'.dependencies] diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index f918663bde7..165cc5df546 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -408,7 +408,7 @@ impl RunWithPathBuf { if runner.can_run_command(id, command).unwrap_or(false) { runner .run_cmd(&container, id) - .context("Emscripten runner failed")?; + .context("WCGI runner failed")?; } anyhow::bail!("No runner"); diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index bd2a32d0671..aee1820cd97 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -13,7 +13,13 @@ dirs = "4.0.0" graphql_client = "0.11.0" serde = { version = "1.0.145", features = ["derive"] } anyhow = "1.0.65" -reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json", "stream"] } +reqwest = { version = "0.11.12", default-features = false, features = [ + "rustls-tls", + "blocking", + "multipart", + "json", + "stream", +] } futures-util = "0.3.25" whoami = "1.2.3" serde_json = "1.0.85" @@ -25,7 +31,7 @@ tar = "0.4.38" flate2 = "1.0.24" semver = "1.0.14" lzma-rs = "0.2.0" -webc = { version ="5.0.0-rc.5", features = ["mmap"] } +webc = { version = "5.0.0-rc.5", features = ["mmap"] } hex = "0.4.3" tokio = "1.24.0" log = "0.4.17" diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 3b0df8c7b85..7789d645a72 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -11,7 +11,6 @@ libc = { version = "^0.2", default-features = false, optional = true } thiserror = "1" tracing = { version = "0.1" } typetag = { version = "0.1", optional = true } -serde = { version = "1.0", default-features = false, features = ["derive"], optional = true } webc = { version = "5.0.0-rc.5", optional = true } slab = { version = "0.4" } derivative = "2.2.0" @@ -21,12 +20,19 @@ lazy_static = "1.4" fs_extra = { version = "1.2.0", optional = true } filetime = { version = "0.2.18", optional = true } bytes = "1" -tokio = { version = "1", features = [ "io-util", "sync", "macros" ], default_features = false } +tokio = { version = "1", features = [ + "io-util", + "sync", + "macros", +], default_features = false } pin-project-lite = "0.2.9" indexmap = "1.9.2" [dev-dependencies] -tokio = { version = "1", features = [ "io-util", "rt" ], default_features = false } +tokio = { version = "1", features = [ + "io-util", + "rt", +], default_features = false } [features] default = ["host-fs", "webc-fs", "static-fs"] diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index a80a4351e24..b5ab831343f 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -8,7 +8,7 @@ use webc::{ Version, }; -/// Parsed WAPM file, memory-mapped to an on-disk path +/// A parsed WAPM package. #[derive(Debug, Clone)] pub struct WapmContainer { repr: Repr, @@ -89,8 +89,9 @@ impl WapmContainer { // HACK(Michael-F-Bryan): WapmContainer originally exposed its Arc // field, so every man and his dog accessed it directly instead of going // through the WapmContainer abstraction. This is an escape hatch to make - // that code w - pub fn v1(&self) -> &WebC<'_> { + // that code keep working for the time being. + // #[deprecated] + pub(crate) fn v1(&self) -> &WebC<'_> { match &self.repr { Repr::V1Mmap(mapped) => &*mapped, Repr::V1Owned(owned) => &*owned, diff --git a/lib/wasi/src/runners/emscripten.rs b/lib/wasi/src/runners/emscripten.rs index bd8a6e9777c..af2a5276656 100644 --- a/lib/wasi/src/runners/emscripten.rs +++ b/lib/wasi/src/runners/emscripten.rs @@ -1,14 +1,14 @@ //! WebC container support for running Emscripten modules use crate::runners::WapmContainer; -use anyhow::{anyhow, Error}; +use anyhow::{anyhow, Context, Error}; use serde::{Deserialize, Serialize}; use wasmer::{FunctionEnv, Instance, Module, Store}; use wasmer_emscripten::{ generate_emscripten_env, is_emscripten_module, run_emscripten_instance, EmEnv, EmscriptenGlobals, }; -use webc::metadata::Command; +use webc::metadata::{annotations::Emscripten, Command}; #[derive(Debug, PartialEq, Serialize, Deserialize)] pub struct EmscriptenRunner { @@ -56,17 +56,18 @@ impl crate::runners::Runner for EmscriptenRunner { fn run_command( &mut self, command_name: &str, - _command: &Command, + command: &Command, container: &WapmContainer, ) -> Result { - let container = container.v1(); - let atom_name = container - .get_atom_name_for_command("emscripten", command_name) - .map_err(Error::msg)?; - let main_args = container.get_main_args_for_command(command_name); + let Emscripten { + atom: atom_name, + main_args, + .. + } = command.get_annotation("emscripten")?.unwrap_or_default(); + let atom_name = atom_name.context("The atom name is required")?; let atom_bytes = container - .get_atom(&container.get_package_name(), &atom_name) - .map_err(Error::msg)?; + .get_atom(&atom_name) + .with_context(|| format!("Unable to read the \"{atom_name}\" atom"))?; let mut module = Module::new(&self.store, atom_bytes)?; module.set_name(&atom_name); diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 66c3b99d7e5..d337fad09a9 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -4,10 +4,10 @@ use std::sync::Arc; use crate::{runners::WapmContainer, PluggableRuntimeImplementation, VirtualTaskManager}; use crate::{WasiEnv, WasiEnvBuilder}; -use anyhow::Error; +use anyhow::{Context, Error}; use serde::{Deserialize, Serialize}; use wasmer::{Module, Store}; -use webc::metadata::Command; +use webc::metadata::{annotations::Wasi, Command}; #[derive(Debug, Serialize, Deserialize)] pub struct WasiRunner { @@ -61,20 +61,22 @@ impl crate::runners::Runner for WasiRunner { Ok(command.runner.starts_with("https://webc.org/runner/wasi")) } - #[allow(unreachable_code, unused_variables)] fn run_command( &mut self, - command_name: &str, - _command: &Command, + _command_name: &str, + command: &Command, container: &WapmContainer, ) -> Result { - let webc = container.v1(); - let atom_name = webc - .get_atom_name_for_command("wasi", command_name) - .map_err(Error::msg)?; - let atom_bytes = webc.get_atom(&webc.get_package_name(), &atom_name)?; - - let mut module = Module::new(&self.store, atom_bytes)?; + let Wasi { + atom: atom_name, .. + } = command + .get_annotation("wasi")? + .context("The command doesn't have any WASI annotations")?; + let atom = container + .get_atom(&atom_name) + .with_context(|| format!("Unable to get the \"{atom_name}\" atom"))?; + + let mut module = Module::new(&self.store, atom)?; module.set_name(&atom_name); let mut builder = prepare_webc_env(container, &atom_name, &self.args)?; @@ -84,14 +86,7 @@ impl crate::runners::Runner for WasiRunner { builder.set_runtime(Arc::new(rt)); } - let init = builder.build_init()?; - - let (instance, env) = WasiEnv::instantiate(init, module, &mut self.store)?; - - let _result = instance - .exports - .get_function("_start")? - .call(&mut self.store, &[])?; + builder.run(module)?; Ok(()) } diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index b5da3c7cc66..0a6abc21695 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -5,7 +5,10 @@ use futures::future::AbortHandle; use wasmer::{Engine, Module, Store}; use wasmer_vfs::FileSystem; use wcgi_host::CgiDialect; -use webc::metadata::{annotations::Wcgi, Command, Manifest}; +use webc::metadata::{ + annotations::{Wasi, Wcgi}, + Command, Manifest, +}; use crate::{ runners::{ @@ -30,9 +33,17 @@ impl WcgiRunner { #[tracing::instrument(skip(self, ctx))] fn run_(&mut self, command_name: &str, ctx: &RunnerContext<'_>) -> Result<(), Error> { - let module = self.load_module(ctx).context("Couldn't load the module")?; + let wasi: Wasi = ctx + .command() + .get_annotation("wasi") + .context("Unable to retrieve the WASI metadata")? + .context("The command doesn't have any WASI annotations")?; + + let module = self + .load_module(&wasi, ctx) + .context("Couldn't load the module")?; - let handler = self.create_handler(module, ctx)?; + let handler = self.create_handler(module, &wasi, ctx)?; let task_manager = Arc::clone(&handler.task_manager); let make_service = hyper::service::make_service_fn(move |_| { @@ -56,6 +67,7 @@ impl WcgiRunner { .serve(make_service) .with_graceful_shutdown(async { let _ = shutdown.await; + tracing::info!("Shutting down gracefully"); }) .await }) @@ -77,15 +89,7 @@ impl WcgiRunner { &mut self.config } - fn load_module(&self, ctx: &RunnerContext<'_>) -> Result { - let wasi: webc::metadata::annotations::Wasi = ctx - .command() - .annotations - .get("wasi") - .cloned() - .and_then(|v| serde_cbor::value::from_value(v).ok()) - .context("Unable to retrieve the WASI metadata")?; - + fn load_module(&self, wasi: &Wasi, ctx: &RunnerContext<'_>) -> Result { let atom_name = &wasi.atom; let atom = ctx .get_atom(&atom_name) @@ -96,19 +100,16 @@ impl WcgiRunner { Ok(module) } - fn create_handler(&self, module: Module, ctx: &RunnerContext<'_>) -> Result { - let mut env = HashMap::new(); - - if self.config.forward_host_env { - env.extend(std::env::vars()); - } - - env.extend(self.config.env.clone()); + fn create_handler( + &self, + module: Module, + wasi: &Wasi, + ctx: &RunnerContext<'_>, + ) -> Result { + let env = construct_env(wasi, self.config.forward_host_env, &self.config.env); + let args = construct_args(wasi, &self.config.args); - let Wcgi { dialect, .. } = match ctx.command().annotations.get("wcgi") { - Some(v) => serde_cbor::value::from_value(v.clone())?, - None => Wcgi::default(), - }; + let Wcgi { dialect, .. } = ctx.command().get_annotation("wcgi")?.unwrap_or_default(); let dialect = match dialect { Some(d) => d.parse().context("Unable to parse the CGI dialect")?, @@ -118,7 +119,7 @@ impl WcgiRunner { let handler = Handler { program: Arc::clone(&self.program_name), env: Arc::new(env), - args: self.config.args.clone().into(), + args, mapped_dirs: self.config.mapped_dirs.clone().into(), task_manager: self .config @@ -134,6 +135,48 @@ impl WcgiRunner { } } +fn construct_args(wasi: &Wasi, extras: &[String]) -> Arc<[String]> { + let mut args = Vec::new(); + + if let Some(main_args) = &wasi.main_args { + args.extend(main_args.iter().cloned()); + } + + args.extend(extras.iter().cloned()); + + args.into() +} + +fn construct_env( + wasi: &Wasi, + forward_host_env: bool, + overrides: &HashMap, +) -> HashMap { + let mut env: HashMap = HashMap::new(); + + for item in wasi.env.as_deref().unwrap_or_default() { + // TODO(Michael-F-Bryan): Convert "wasi.env" in the webc crate from an + // Option> to a HashMap so we avoid this + // string.split() business + match item.split_once('=') { + Some((k, v)) => { + env.insert(k.to_string(), v.to_string()); + } + None => { + env.insert(item.to_string(), String::new()); + } + } + } + + if forward_host_env { + env.extend(std::env::vars()); + } + + env.extend(overrides.clone()); + + env +} + // TODO(Michael-F-Bryan): Pass this to Runner::run() as "&dyn RunnerContext" // when we rewrite the "Runner" trait. struct RunnerContext<'a> { From b174d60a0fa669387564ce04a7a5ec7f0a40f48e Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Mar 2023 13:14:18 +0800 Subject: [PATCH 12/26] Fix the C API crate --- lib/c-api/Cargo.toml | 3 ++- lib/c-api/src/wasm_c_api/wasi/mod.rs | 4 ++-- lib/wasi/src/runners/container.rs | 3 ++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index a498e930dd3..b12216602b6 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -45,6 +45,7 @@ libc = { version = "^0.2", default-features = false } thiserror = "1" typetag = { version = "0.1", optional = true } paste = "1.0" +wasmer-vfs = { version = "3.2.0-alpha.1", path = "../vfs", optional = true } [dev-dependencies] field-offset = "0.3.3" @@ -79,7 +80,7 @@ wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] static-artifact-load = ["wasmer-compiler/static-artifact-load"] static-artifact-create = ["wasmer-compiler/static-artifact-create"] -webc_runner = ["wasmer-wasi/webc_runner", "webc"] +webc_runner = ["wasmer-wasi/webc_runner", "webc", "wasmer-vfs"] # Deprecated features. jit = ["compiler"] diff --git a/lib/c-api/src/wasm_c_api/wasi/mod.rs b/lib/c-api/src/wasm_c_api/wasi/mod.rs index 286b6f02595..4b8afc1bb2b 100644 --- a/lib/c-api/src/wasm_c_api/wasi/mod.rs +++ b/lib/c-api/src/wasm_c_api/wasi/mod.rs @@ -249,10 +249,10 @@ fn prepare_webc_env( package_name: &str, ) -> Option<(WasiFunctionEnv, Imports)> { use wasmer_vfs::static_fs::StaticFileSystem; - use webc::FsEntryType; + use webc::v1::{FsEntryType, WebC}; let slice = unsafe { std::slice::from_raw_parts(bytes, len) }; - let volumes = webc::WebC::parse_volumes_from_fileblock(slice).ok()?; + let volumes = WebC::parse_volumes_from_fileblock(slice).ok()?; let top_level_dirs = volumes .into_iter() .flat_map(|(_, volume)| { diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index b5ab831343f..029e5e27659 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -14,6 +14,7 @@ pub struct WapmContainer { repr: Repr, } +#[allow(dead_code)] // Some pub(crate) items are only used behind #[cfg] code impl WapmContainer { /// Parses a .webc container file. Since .webc files /// can be very large, only file paths are allowed. @@ -99,7 +100,7 @@ impl WapmContainer { } /// Load a volume as a [`FileSystem`] node. - pub fn volume_fs(&self, package_name: &str) -> Box { + pub(crate) fn volume_fs(&self, package_name: &str) -> Box { match &self.repr { Repr::V1Mmap(mapped) => { Box::new(WebcFileSystem::init(Arc::clone(mapped), package_name)) From ca52b2e4f7e37c4d85bd4eb4ee3c915440bf6e92 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Mar 2023 16:53:33 +0800 Subject: [PATCH 13/26] We can run a WCGI server locally! --- Cargo.lock | 7 ------- lib/c-api/Cargo.toml | 1 - lib/cli/Cargo.toml | 2 -- lib/cli/src/commands/run.rs | 25 +++++++++++-------------- lib/wasi/src/runners/emscripten.rs | 2 +- lib/wasi/src/runners/wasi.rs | 15 ++++++++------- lib/wasi/src/runners/wcgi/runner.rs | 18 ++++++++++++++---- 7 files changed, 34 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e8a06b68be9..60118dd307b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2379,12 +2379,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "nuke-dir" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6842d8099b88d19a64158a6cfdc3e9ad82c738c041dab98280ef7ba98d64fa" - [[package]] name = "num-integer" version = "0.1.45" @@ -4870,7 +4864,6 @@ dependencies = [ "libc", "log", "minisign", - "nuke-dir", "object 0.30.3", "pathdiff", "prettytable-rs", diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index b12216602b6..2520c9fd56f 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -45,7 +45,6 @@ libc = { version = "^0.2", default-features = false } thiserror = "1" typetag = { version = "0.1", optional = true } paste = "1.0" -wasmer-vfs = { version = "3.2.0-alpha.1", path = "../vfs", optional = true } [dev-dependencies] field-offset = "0.3.3" diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 368b7ab36a6..bb2f3eb50db 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -79,7 +79,6 @@ regex = "1.6.0" toml = "0.5.9" url = "2.3.1" libc = { version = "^0.2", default-features = false } -nuke-dir = { version = "0.1.0", optional = true } webc = { version = "5.0.0-rc.5", optional = true } isatty = "0.1.9" dialoguer = "0.10.2" @@ -139,7 +138,6 @@ webc_runner = [ "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "wasmer-wasi/webc_runner_rt_wcgi", - "nuke-dir", "webc", ] compiler = [ diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 165cc5df546..195bcd99b92 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -224,9 +224,7 @@ impl RunWithPathBuf { #[cfg(feature = "webc_runner")] { if let Ok(pf) = WapmContainer::from_path(self.path.clone()) { - return self - .run_container(pf, self.command_name.as_deref(), &self.args) - .map_err(|e| anyhow!("Could not run PiritaFile: {e}")); + return self.run_container(pf, self.command_name.as_deref(), &self.args); } } let (mut store, module) = self.get_store_module()?; @@ -378,7 +376,6 @@ impl RunWithPathBuf { let id = id .or_else(|| container.manifest().entrypoint.as_deref()) .context("No command specified")?; - let command = container .manifest() .commands @@ -389,29 +386,29 @@ impl RunWithPathBuf { let mut runner = wasmer_wasi::runners::wasi::WasiRunner::new(store); runner.set_args(args.to_vec()); if runner.can_run_command(id, command).unwrap_or(false) { - runner - .run_cmd(&container, id) - .context("WASI runner failed")?; + return runner.run_cmd(&container, id).context("WASI runner failed"); } let (store, _compiler_type) = self.store.get_store()?; let mut runner = wasmer_wasi::runners::emscripten::EmscriptenRunner::new(store); runner.set_args(args.to_vec()); if runner.can_run_command(id, command).unwrap_or(false) { - runner + return runner .run_cmd(&container, id) - .context("Emscripten runner failed")?; + .context("Emscripten runner failed"); } let mut runner = wasmer_wasi::runners::wcgi::WcgiRunner::new(id); - runner.config().args(args); + let (store, _compiler_type) = self.store.get_store()?; + runner.config().args(args).store(store); if runner.can_run_command(id, command).unwrap_or(false) { - runner - .run_cmd(&container, id) - .context("WCGI runner failed")?; + return runner.run_cmd(&container, id).context("WCGI runner failed"); } - anyhow::bail!("No runner"); + anyhow::bail!( + "Unable to find a runner that supports \"{}\"", + command.runner + ); } fn get_store_module(&self) -> Result<(Store, Module)> { diff --git a/lib/wasi/src/runners/emscripten.rs b/lib/wasi/src/runners/emscripten.rs index af2a5276656..f0b9cb4ee8e 100644 --- a/lib/wasi/src/runners/emscripten.rs +++ b/lib/wasi/src/runners/emscripten.rs @@ -49,7 +49,7 @@ impl crate::runners::Runner for EmscriptenRunner { fn can_run_command(&self, _: &str, command: &Command) -> Result { Ok(command .runner - .starts_with("https://webc.org/runner/emscripten")) + .starts_with(webc::metadata::annotations::EMSCRIPTEN_RUNNER_URI)) } #[allow(unreachable_code, unused_variables)] diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index d337fad09a9..f188bc394a3 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -58,20 +58,21 @@ impl crate::runners::Runner for WasiRunner { type Output = (); fn can_run_command(&self, _command_name: &str, command: &Command) -> Result { - Ok(command.runner.starts_with("https://webc.org/runner/wasi")) + Ok(command + .runner + .starts_with(webc::metadata::annotations::WASI_RUNNER_URI)) } fn run_command( &mut self, - _command_name: &str, + command_name: &str, command: &Command, container: &WapmContainer, ) -> Result { - let Wasi { - atom: atom_name, .. - } = command - .get_annotation("wasi")? - .context("The command doesn't have any WASI annotations")?; + let atom_name = match command.get_annotation("wasi")? { + Some(Wasi { atom, .. }) => atom, + None => command_name.to_string(), + }; let atom = container .get_atom(&atom_name) .with_context(|| format!("Unable to get the \"{atom_name}\" atom"))?; diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index 0a6abc21695..eb8cda9be69 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -28,7 +28,9 @@ pub struct WcgiRunner { // make the "Runner" trait contain just these two methods. impl WcgiRunner { fn supports(cmd: &Command) -> Result { - Ok(cmd.runner.starts_with("https://webc.org/runner/wasi")) + Ok(cmd + .runner + .starts_with(webc::metadata::annotations::WCGI_RUNNER_URI)) } #[tracing::instrument(skip(self, ctx))] @@ -37,7 +39,7 @@ impl WcgiRunner { .command() .get_annotation("wasi") .context("Unable to retrieve the WASI metadata")? - .context("The command doesn't have any WASI annotations")?; + .unwrap_or_else(|| Wasi::new(command_name)); let module = self .load_module(&wasi, ctx) @@ -183,7 +185,7 @@ struct RunnerContext<'a> { container: &'a WapmContainer, command: &'a Command, engine: Engine, - store: Store, + store: Arc, } #[allow(dead_code)] @@ -231,7 +233,8 @@ impl crate::runners::Runner for WcgiRunner { command: &Command, container: &WapmContainer, ) -> Result { - let store = Store::default(); + let store = self.config.store.clone().unwrap_or_default(); + let ctx = RunnerContext { container, command, @@ -254,6 +257,7 @@ pub struct Config { mapped_dirs: Vec, #[derivative(Debug = "ignore")] callbacks: Arc, + store: Option>, } impl Config { @@ -325,6 +329,11 @@ impl Config { self.callbacks = Arc::new(callbacks); self } + + pub fn store(&mut self, store: Store) -> &mut Self { + self.store = Some(Arc::new(store)); + self + } } impl Default for Config { @@ -337,6 +346,7 @@ impl Default for Config { mapped_dirs: Vec::new(), args: Vec::new(), callbacks: Arc::new(NoopCallbacks), + store: None, } } } From bd324f9fc0b5b2304e827ccd466ea9db384f9efd Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Mar 2023 22:13:58 +0800 Subject: [PATCH 14/26] Added WCGI-specific flags to the CLI --- lib/cli/src/commands/run.rs | 37 +++++++++++++++++++++++++++-- lib/wasi/src/runners/wcgi/runner.rs | 14 +++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 195bcd99b92..9ad2c108f49 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -8,12 +8,12 @@ use crate::suggestions::suggest_function_exports; use crate::warning; use anyhow::{anyhow, Context, Result}; use clap::Parser; -use std::fs::File; use std::io::Write; use std::ops::Deref; use std::path::PathBuf; #[cfg(feature = "cache")] use std::str::FromStr; +use std::{fs::File, net::SocketAddr}; #[cfg(feature = "emscripten")] use wasmer::FunctionEnv; use wasmer::*; @@ -91,6 +91,10 @@ pub struct RunWithoutFile { #[clap(long = "verbose")] pub(crate) verbose: Option, + #[cfg(feature = "webc_runner")] + #[clap(flatten)] + pub(crate) wcgi: WcgiOptions, + /// Enable coredump generation after a WebAssembly trap. #[clap(name = "COREDUMP PATH", long = "coredump-on-trap", parse(from_os_str))] coredump_on_trap: Option, @@ -223,6 +227,7 @@ impl RunWithPathBuf { fn inner_execute(&self) -> Result<()> { #[cfg(feature = "webc_runner")] { + dbg!(&self.path); if let Ok(pf) = WapmContainer::from_path(self.path.clone()) { return self.run_container(pf, self.command_name.as_deref(), &self.args); } @@ -400,7 +405,16 @@ impl RunWithPathBuf { let mut runner = wasmer_wasi::runners::wcgi::WcgiRunner::new(id); let (store, _compiler_type) = self.store.get_store()?; - runner.config().args(args).store(store); + runner + .config() + .args(args) + .store(store) + .addr(self.wcgi.addr) + .envs(self.wasi.env_vars.clone()) + .map_directories(self.wasi.mapped_dirs.iter().map(|(g, h)| (h, g))); + if self.wcgi.forward_host_env { + runner.config().forward_host_env(); + } if runner.can_run_command(id, command).unwrap_or(false) { return runner.run_cmd(&container, id).context("WCGI runner failed"); } @@ -689,3 +703,22 @@ fn generate_coredump( Ok(()) } + +#[derive(Debug, Clone, Parser)] +pub(crate) struct WcgiOptions { + /// The address to serve on. + #[clap(long, short, env, default_value_t = ([127, 0, 0, 1], 8000).into())] + pub(crate) addr: SocketAddr, + /// Forward all host env variables to the wcgi task. + #[clap(long)] + pub(crate) forward_host_env: bool, +} + +impl Default for WcgiOptions { + fn default() -> Self { + Self { + addr: ([127, 0, 0, 1], 8000).into(), + forward_host_env: false, + } + } +} diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index eb8cda9be69..bb2b33ccb23 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -323,6 +323,20 @@ impl Config { self } + pub fn map_directories(&mut self, mappings: I) -> &mut Self + where + I: IntoIterator, + H: Into, + G: Into, + { + let mappings = mappings.into_iter().map(|(h, g)| MappedDirectory { + host: h.into(), + guest: g.into(), + }); + self.mapped_dirs.extend(mappings); + self + } + /// Set callbacks that will be triggered at various points in the runner's /// lifecycle. pub fn callbacks(&mut self, callbacks: impl Callbacks + Send + Sync + 'static) -> &mut Self { From 3af48f7028c5ca8f8b569b356b6cd0447f991449 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Mar 2023 22:34:51 +0800 Subject: [PATCH 15/26] Hoisted shared state into its own struct --- lib/cli/src/commands/run.rs | 1 - lib/wasi/src/runners/wcgi/handler.rs | 42 +++++++++++++++++++--------- lib/wasi/src/runners/wcgi/runner.rs | 21 ++++++++------ lib/wasi/tests/runners.rs | 2 +- 4 files changed, 42 insertions(+), 24 deletions(-) diff --git a/lib/cli/src/commands/run.rs b/lib/cli/src/commands/run.rs index 9ad2c108f49..e5b1f4e78b7 100644 --- a/lib/cli/src/commands/run.rs +++ b/lib/cli/src/commands/run.rs @@ -227,7 +227,6 @@ impl RunWithPathBuf { fn inner_execute(&self) -> Result<()> { #[cfg(feature = "webc_runner")] { - dbg!(&self.path); if let Ok(pf) = WapmContainer::from_path(self.path.clone()) { return self.run_container(pf, self.command_name.as_deref(), &self.args); } diff --git a/lib/wasi/src/runners/wcgi/handler.rs b/lib/wasi/src/runners/wcgi/handler.rs index 38444b52926..970c50f8515 100644 --- a/lib/wasi/src/runners/wcgi/handler.rs +++ b/lib/wasi/src/runners/wcgi/handler.rs @@ -1,5 +1,6 @@ use std::{ collections::HashMap, + ops::Deref, path::{Path, PathBuf}, pin::Pin, sync::Arc, @@ -26,21 +27,14 @@ use crate::{ /// The shared object that manages the instantiaion of WASI executables and /// communicating with them via the CGI protocol. -#[derive(Clone, derivative::Derivative)] -#[derivative(Debug)] -pub(crate) struct Handler { - pub(crate) program: Arc, - pub(crate) env: Arc>, - pub(crate) args: Arc<[String]>, - pub(crate) mapped_dirs: Arc<[MappedDirectory]>, - pub(crate) task_manager: Arc, - pub(crate) module: Module, - pub(crate) dialect: CgiDialect, - #[derivative(Debug = "ignore")] - pub(crate) callbacks: Arc, -} +#[derive(Clone, Debug)] +pub(crate) struct Handler(Arc); impl Handler { + pub(crate) fn new(state: SharedState) -> Self { + Handler(Arc::new(state)) + } + pub(crate) async fn handle(&self, req: Request) -> Result, Error> { let (parts, body) = req.into_parts(); @@ -153,6 +147,14 @@ impl Handler { } } +impl Deref for Handler { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + /// Drive the request to completion by streaming the request body to the /// instance and waiting for it to exit. async fn drive_request_to_completion( @@ -216,6 +218,20 @@ async fn consume_stderr( } } +#[derive(Clone, derivative::Derivative)] +#[derivative(Debug)] +pub(crate) struct SharedState { + pub(crate) program: String, + pub(crate) env: HashMap, + pub(crate) args: Vec, + pub(crate) mapped_dirs: Vec, + pub(crate) module: Module, + pub(crate) dialect: CgiDialect, + pub(crate) task_manager: Arc, + #[derivative(Debug = "ignore")] + pub(crate) callbacks: Arc, +} + impl Service> for Handler { type Response = Response; type Error = Error; diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index bb2b33ccb23..28d32d9c856 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -12,7 +12,10 @@ use webc::metadata::{ use crate::{ runners::{ - wcgi::{handler::Handler, MappedDirectory}, + wcgi::{ + handler::{Handler, SharedState}, + MappedDirectory, + }, WapmContainer, }, runtime::task_manager::tokio::TokioTaskManager, @@ -20,7 +23,7 @@ use crate::{ }; pub struct WcgiRunner { - program_name: Arc, + program_name: String, config: Config, } @@ -80,7 +83,7 @@ impl WcgiRunner { } impl WcgiRunner { - pub fn new(program_name: impl Into>) -> Self { + pub fn new(program_name: impl Into) -> Self { WcgiRunner { program_name: program_name.into(), config: Config::default(), @@ -118,9 +121,9 @@ impl WcgiRunner { None => CgiDialect::Wcgi, }; - let handler = Handler { - program: Arc::clone(&self.program_name), - env: Arc::new(env), + let shared = SharedState { + program: self.program_name.clone(), + env, args, mapped_dirs: self.config.mapped_dirs.clone().into(), task_manager: self @@ -133,11 +136,11 @@ impl WcgiRunner { callbacks: Arc::clone(&self.config.callbacks), }; - Ok(handler) + Ok(Handler::new(shared)) } } -fn construct_args(wasi: &Wasi, extras: &[String]) -> Arc<[String]> { +fn construct_args(wasi: &Wasi, extras: &[String]) -> Vec { let mut args = Vec::new(); if let Some(main_args) = &wasi.main_args { @@ -146,7 +149,7 @@ fn construct_args(wasi: &Wasi, extras: &[String]) -> Arc<[String]> { args.extend(extras.iter().cloned()); - args.into() + args } fn construct_env( diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs index dd7027088d9..075721fda42 100644 --- a/lib/wasi/tests/runners.rs +++ b/lib/wasi/tests/runners.rs @@ -57,7 +57,7 @@ mod wcgi { #[tokio::test] async fn static_server() { - let webc = download_cached("https://wapm.dev/syrusakbary/staticserver").await; + let webc = download_cached("https://wapm.io/Michael-F-Bryan/staticserver").await; let tasks = TokioTaskManager::new(Handle::current()); let container = WapmContainer::from_bytes(webc).unwrap(); let mut runner = WcgiRunner::new("staticserver"); From 0d88c8faa2351ccc64e937c541d22a96b448f2d8 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 1 Mar 2023 23:12:53 +0800 Subject: [PATCH 16/26] Made "make lint" happy --- Cargo.lock | 1 + lib/vfs/Cargo.toml | 3 ++- lib/wasi/src/runners/container.rs | 4 ++-- lib/wasi/src/runners/wasi.rs | 2 +- lib/wasi/src/runners/wcgi/mod.rs | 2 +- lib/wasi/src/runners/wcgi/runner.rs | 4 ++-- 6 files changed, 9 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 60118dd307b..9910f809e0f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5208,6 +5208,7 @@ dependencies = [ "lazy_static", "libc", "pin-project-lite", + "serde", "slab", "thiserror", "tokio", diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 7789d645a72..b848c09f26e 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -27,6 +27,7 @@ tokio = { version = "1", features = [ ], default_features = false } pin-project-lite = "0.2.9" indexmap = "1.9.2" +serde = { version = "1", optional = true, features = ["derive"] } [dev-dependencies] tokio = { version = "1", features = [ @@ -39,5 +40,5 @@ default = ["host-fs", "webc-fs", "static-fs"] host-fs = ["libc", "fs_extra", "filetime", "tokio/fs", "tokio/io-std"] webc-fs = ["webc", "anyhow"] static-fs = ["webc", "anyhow"] -enable-serde = ["typetag"] +enable-serde = ["serde", "typetag"] no-time = [] diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index 029e5e27659..8849567559c 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -94,8 +94,8 @@ impl WapmContainer { // #[deprecated] pub(crate) fn v1(&self) -> &WebC<'_> { match &self.repr { - Repr::V1Mmap(mapped) => &*mapped, - Repr::V1Owned(owned) => &*owned, + Repr::V1Mmap(mapped) => mapped, + Repr::V1Owned(owned) => owned, } } diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index f188bc394a3..9f0522241c0 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -83,7 +83,7 @@ impl crate::runners::Runner for WasiRunner { let mut builder = prepare_webc_env(container, &atom_name, &self.args)?; if let Some(tasks) = &self.tasks { - let rt = PluggableRuntimeImplementation::new(Arc::clone(&tasks)); + let rt = PluggableRuntimeImplementation::new(Arc::clone(tasks)); builder.set_runtime(Arc::new(rt)); } diff --git a/lib/wasi/src/runners/wcgi/mod.rs b/lib/wasi/src/runners/wcgi/mod.rs index a160c3c87b3..1fe6f485c12 100644 --- a/lib/wasi/src/runners/wcgi/mod.rs +++ b/lib/wasi/src/runners/wcgi/mod.rs @@ -3,7 +3,7 @@ mod runner; use std::path::PathBuf; -pub use self::runner::{Config, WcgiRunner, Callbacks}; +pub use self::runner::{Callbacks, Config, WcgiRunner}; #[derive(Debug, Clone, PartialEq)] pub(crate) struct MappedDirectory { diff --git a/lib/wasi/src/runners/wcgi/runner.rs b/lib/wasi/src/runners/wcgi/runner.rs index 28d32d9c856..516e7bbe532 100644 --- a/lib/wasi/src/runners/wcgi/runner.rs +++ b/lib/wasi/src/runners/wcgi/runner.rs @@ -97,7 +97,7 @@ impl WcgiRunner { fn load_module(&self, wasi: &Wasi, ctx: &RunnerContext<'_>) -> Result { let atom_name = &wasi.atom; let atom = ctx - .get_atom(&atom_name) + .get_atom(atom_name) .with_context(|| format!("Unable to retrieve the \"{atom_name}\" atom"))?; let module = ctx.compile(atom).context("Unable to compile the atom")?; @@ -125,7 +125,7 @@ impl WcgiRunner { program: self.program_name.clone(), env, args, - mapped_dirs: self.config.mapped_dirs.clone().into(), + mapped_dirs: self.config.mapped_dirs.clone(), task_manager: self .config .task_manager From c2dab64686c087a84ca23892ff169f0c1e90a54f Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 2 Mar 2023 14:43:52 +0800 Subject: [PATCH 17/26] Added "can_run" tests --- lib/wasi/tests/runners.rs | 46 ++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 8 deletions(-) diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs index 075721fda42..aaad9b5e1bf 100644 --- a/lib/wasi/tests/runners.rs +++ b/lib/wasi/tests/runners.rs @@ -17,7 +17,18 @@ mod wasi { use super::*; #[tokio::test] - async fn wat_2_wasm() { + async fn can_run_wat2wasm() { + let webc = download_cached("https://wapm.io/wasmer/wabt").await; + let store = Store::default(); + let container = WapmContainer::from_bytes(webc).unwrap(); + let runner = WasiRunner::new(store); + let command = &container.manifest().commands["wat2wasm"]; + + assert!(runner.can_run_command("wat2wasm", command).unwrap()); + } + + #[tokio::test] + async fn wat2wasm() { let webc = download_cached("https://wapm.io/wasmer/wabt").await; let store = Store::default(); let tasks = TokioTaskManager::new(Handle::current()); @@ -38,7 +49,7 @@ mod wasi { .unwrap(); let exit_code = match runtime_error { WasiError::Exit(code) => *code, - _ => unreachable!(), + other => unreachable!("Something else went wrong: {:?}", other), }; assert_eq!(exit_code, 1); } @@ -56,7 +67,19 @@ mod wcgi { use super::*; #[tokio::test] - async fn static_server() { + async fn can_run_staticserver() { + let webc = download_cached("https://wapm.io/Michael-F-Bryan/staticserver").await; + let container = WapmContainer::from_bytes(webc).unwrap(); + let runner = WcgiRunner::new("staticserver"); + + let entrypoint = container.manifest().entrypoint.as_ref().unwrap(); + assert!(runner + .can_run_command(entrypoint, &container.manifest().commands[entrypoint]) + .unwrap()); + } + + #[tokio::test] + async fn staticserver() { let webc = download_cached("https://wapm.io/Michael-F-Bryan/staticserver").await; let tasks = TokioTaskManager::new(Handle::current()); let container = WapmContainer::from_bytes(webc).unwrap(); @@ -70,26 +93,33 @@ mod wcgi { .callbacks(cb); // The server blocks so we need to start it on a background thread. - std::thread::spawn(move || { - runner.run_cmd(&container, "wcgi").unwrap(); + let join_handle = std::thread::spawn(move || { + runner.run(&container).unwrap(); }); // wait for the server to have started let abort_handle = started.await; // Now the server is running, we can check that it is working by - // fetching "/" and checking for known content + // fetching "/" and checking for known content. We also want the server + // to close connections immediately so the graceful shutdown can kill + // the server immediately instead of waiting for the connection to time + // out. let resp = client() .get(format!("http://localhost:{port}/")) + .header("Connection", "close") .send() .await .unwrap(); let body = resp.error_for_status().unwrap().text().await.unwrap(); - assert!(body.contains("Index of /"), "{}", body); - // Make sure the server is shutdown afterwards + // Make sure the server is shutdown afterwards. Failing tests will leak + // the server thread, but that's fine. abort_handle.abort(); + if let Err(e) = join_handle.join() { + std::panic::resume_unwind(e); + } } fn callbacks(handle: Handle) -> (Callbacks, impl Future) { From 1732b40ee1e9d41a4451f4e206816f909d002f98 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 2 Mar 2023 15:04:12 +0800 Subject: [PATCH 18/26] Made "cargo deny" pass --- deny.toml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/deny.toml b/deny.toml index aad99e01720..e8382d45698 100644 --- a/deny.toml +++ b/deny.toml @@ -64,7 +64,7 @@ git-fetch-with-cli = true # * Medium - CVSS Score 4.0 - 6.9 # * High - CVSS Score 7.0 - 8.9 # * Critical - CVSS Score 9.0 - 10.0 -#severity-threshold = +#severity-threshold = # This section is considered when running `cargo deny check licenses` # More documentation for the licenses section can be found here: @@ -111,6 +111,7 @@ confidence-threshold = 0.8 exceptions = [ # Each entry is the crate and version constraint, and its specific allow # list + { name = "webc", allow = ["BUSL-1.1"] } ] @@ -189,8 +190,8 @@ skip = [ { name = "itoa", version = "=0.4.8" }, { name = "object", version = "=0.27.1" }, ] -# Similarly to `skip` allows you to skip certain crates during duplicate -# detection. Unlike skip, it also includes the entire tree of transitive +# Similarly to `skip` allows you to skip certain crates during duplicate +# detection. Unlike skip, it also includes the entire tree of transitive # dependencies starting at the specified crate, up to a certain depth, which is # by default infinite skip-tree = [ From 2970aad6ddcffa2e5d7366d3e181bc3f5890d88c Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 2 Mar 2023 20:08:04 +0800 Subject: [PATCH 19/26] Refactored prepare_webc_env() to match the original implementation --- lib/wasi/src/runners/wasi.rs | 47 ++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 9f0522241c0..63b83798bce 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -1,12 +1,13 @@ //! WebC container support for running WASI modules -use std::sync::Arc; +use std::{collections::VecDeque, path::PathBuf, sync::Arc}; use crate::{runners::WapmContainer, PluggableRuntimeImplementation, VirtualTaskManager}; use crate::{WasiEnv, WasiEnvBuilder}; use anyhow::{Context, Error}; use serde::{Deserialize, Serialize}; use wasmer::{Module, Store}; +use wasmer_vfs::{DirEntry, FileSystem}; use webc::metadata::{annotations::Wasi, Command}; #[derive(Debug, Serialize, Deserialize)] @@ -101,24 +102,40 @@ fn prepare_webc_env( ) -> Result { let filesystem = container.container_fs(); let mut builder = WasiEnv::builder(command).args(args); + let preopen_dirs = all_directories(&*filesystem); - if let Ok(dir) = filesystem.read_dir("/".as_ref()) { - let entries = dir.filter_map(|entry| entry.ok()).filter(|entry| { - if let Ok(file_type) = entry.file_type() { - file_type.dir - } else { - false - } - }); - - for entry in entries { - builder.add_preopen_build(|p| { - p.directory(&entry.path).read(true).write(true).create(true) - })?; - } + for entry in preopen_dirs { + builder + .add_preopen_build(|p| p.directory(&entry.path).read(true).write(true).create(true))?; } builder.set_fs(filesystem); Ok(builder) } + +fn all_directories(fs: &dyn FileSystem) -> Vec { + let mut dirs = Vec::new(); + + let mut to_check = VecDeque::new(); + to_check.push_back(PathBuf::from("/")); + + while let Some(path) = to_check.pop_front() { + let children = fs + .read_dir(&path) + .into_iter() + .flatten() + .flat_map(|result| result.ok()); + + for child in children { + if let Ok(file_type) = child.file_type() { + if file_type.is_dir() { + to_check.push_back(child.path()); + dirs.push(child); + } + } + } + } + + dirs +} From e258df4dc476b727d58190b4d4174e1f97c1961a Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Thu, 2 Mar 2023 20:26:40 +0800 Subject: [PATCH 20/26] Removed an unused import --- lib/cli/src/commands/create_exe.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 8c5379f061a..c13d06abb6f 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -15,9 +15,7 @@ use std::process::Stdio; use tar::Archive; use wasmer::*; use wasmer_object::{emit_serialized, get_object_for_target}; -use wasmer_types::{ - compilation::symbols::ModuleMetadataSymbolRegistry, ModuleInfo, SymbolRegistry, -}; +use wasmer_types::{compilation::symbols::ModuleMetadataSymbolRegistry, ModuleInfo}; use webc::v1::{ParseOptions, WebCMmap}; const LINK_SYSTEM_LIBRARIES_WINDOWS: &[&str] = &["userenv", "Ws2_32", "advapi32", "bcrypt"]; From a9a091d85f3dde0bc265585e27ba4427dbf8c529 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Sat, 4 Mar 2023 17:57:40 +0800 Subject: [PATCH 21/26] Add an integration test for running the Python container --- lib/wasi/tests/runners.rs | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs index aaad9b5e1bf..d9283019768 100644 --- a/lib/wasi/tests/runners.rs +++ b/lib/wasi/tests/runners.rs @@ -53,6 +53,33 @@ mod wasi { }; assert_eq!(exit_code, 1); } + + #[tokio::test] + async fn python() { + let webc = download_cached("https://wapm.io/python/python").await; + let store = Store::default(); + let tasks = TokioTaskManager::new(Handle::current()); + let container = WapmContainer::from_bytes(webc).unwrap(); + + // Note: we don't have any way to intercept stdin or stdout, so blindly + // assume that everything is fine if it runs successfully. + let handle = std::thread::spawn(move || { + WasiRunner::new(store) + .with_task_manager(tasks) + .run_cmd(&container, "python") + }); + let err = handle.join().unwrap().unwrap_err(); + + let runtime_error = err + .chain() + .find_map(|e| e.downcast_ref::()) + .unwrap(); + let exit_code = match runtime_error { + WasiError::Exit(code) => *code, + other => unreachable!("Something else went wrong: {:?}", other), + }; + assert_eq!(exit_code, 42); + } } #[cfg(feature = "webc_runner_rt_wcgi")] From 43116e905669ea21b698546e655d8ff50a87b706 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Mon, 6 Mar 2023 22:17:38 +0800 Subject: [PATCH 22/26] Discovered that WebcFileSystem::top_level_dirs() and read_dir() aren't equivalent --- lib/wasi/src/runners/container.rs | 17 ++++++++--- lib/wasi/src/runners/wasi.rs | 49 +++++++++---------------------- 2 files changed, 27 insertions(+), 39 deletions(-) diff --git a/lib/wasi/src/runners/container.rs b/lib/wasi/src/runners/container.rs index 8849567559c..31c58fec1a0 100644 --- a/lib/wasi/src/runners/container.rs +++ b/lib/wasi/src/runners/container.rs @@ -109,11 +109,20 @@ impl WapmContainer { } } - /// Get the entire container as a single filesystem. - pub(crate) fn container_fs(&self) -> Box { + /// Get the entire container as a single filesystem and a list of suggested + /// directories to preopen. + pub(crate) fn container_fs(&self) -> (Box, Vec) { match &self.repr { - Repr::V1Mmap(mapped) => Box::new(WebcFileSystem::init_all(Arc::clone(mapped))), - Repr::V1Owned(owned) => Box::new(WebcFileSystem::init_all(Arc::clone(owned))), + Repr::V1Mmap(mapped) => { + let fs = WebcFileSystem::init_all(Arc::clone(mapped)); + let top_level_dirs = fs.top_level_dirs().clone(); + (Box::new(fs), top_level_dirs) + } + Repr::V1Owned(owned) => { + let fs = WebcFileSystem::init_all(Arc::clone(owned)); + let top_level_dirs = fs.top_level_dirs().clone(); + (Box::new(fs), top_level_dirs) + } } } } diff --git a/lib/wasi/src/runners/wasi.rs b/lib/wasi/src/runners/wasi.rs index 63b83798bce..c78f3650eaa 100644 --- a/lib/wasi/src/runners/wasi.rs +++ b/lib/wasi/src/runners/wasi.rs @@ -1,13 +1,12 @@ //! WebC container support for running WASI modules -use std::{collections::VecDeque, path::PathBuf, sync::Arc}; +use std::sync::Arc; use crate::{runners::WapmContainer, PluggableRuntimeImplementation, VirtualTaskManager}; use crate::{WasiEnv, WasiEnvBuilder}; use anyhow::{Context, Error}; use serde::{Deserialize, Serialize}; use wasmer::{Module, Store}; -use wasmer_vfs::{DirEntry, FileSystem}; use webc::metadata::{annotations::Wasi, Command}; #[derive(Debug, Serialize, Deserialize)] @@ -35,14 +34,22 @@ impl WasiRunner { } /// Builder method to provide CLI args to the runner - pub fn with_args(mut self, args: Vec) -> Self { + pub fn with_args(mut self, args: A) -> Self + where + A: IntoIterator, + S: Into, + { self.set_args(args); self } /// Set the CLI args - pub fn set_args(&mut self, args: Vec) { - self.args = args; + pub fn set_args(&mut self, args: A) + where + A: IntoIterator, + S: Into, + { + self.args = args.into_iter().map(|s| s.into()).collect(); } pub fn with_task_manager(mut self, tasks: impl VirtualTaskManager) -> Self { @@ -100,42 +107,14 @@ fn prepare_webc_env( command: &str, args: &[String], ) -> Result { - let filesystem = container.container_fs(); + let (filesystem, preopen_dirs) = container.container_fs(); let mut builder = WasiEnv::builder(command).args(args); - let preopen_dirs = all_directories(&*filesystem); for entry in preopen_dirs { - builder - .add_preopen_build(|p| p.directory(&entry.path).read(true).write(true).create(true))?; + builder.add_preopen_build(|p| p.directory(&entry).read(true).write(true).create(true))?; } builder.set_fs(filesystem); Ok(builder) } - -fn all_directories(fs: &dyn FileSystem) -> Vec { - let mut dirs = Vec::new(); - - let mut to_check = VecDeque::new(); - to_check.push_back(PathBuf::from("/")); - - while let Some(path) = to_check.pop_front() { - let children = fs - .read_dir(&path) - .into_iter() - .flatten() - .flat_map(|result| result.ok()); - - for child in children { - if let Ok(file_type) = child.file_type() { - if file_type.is_dir() { - to_check.push_back(child.path()); - dirs.push(child); - } - } - } - } - - dirs -} From 20c10bcaf6ca79322182f8d5d0e96a2d2d2f2df0 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Mon, 6 Mar 2023 22:17:59 +0800 Subject: [PATCH 23/26] Updated the Python WASI runner test to check for a specific exit code --- lib/wasi/tests/runners.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/wasi/tests/runners.rs b/lib/wasi/tests/runners.rs index d9283019768..713ab862e16 100644 --- a/lib/wasi/tests/runners.rs +++ b/lib/wasi/tests/runners.rs @@ -61,11 +61,10 @@ mod wasi { let tasks = TokioTaskManager::new(Handle::current()); let container = WapmContainer::from_bytes(webc).unwrap(); - // Note: we don't have any way to intercept stdin or stdout, so blindly - // assume that everything is fine if it runs successfully. let handle = std::thread::spawn(move || { WasiRunner::new(store) .with_task_manager(tasks) + .with_args(["-c", "import sys; sys.exit(42)"]) .run_cmd(&container, "python") }); let err = handle.join().unwrap().unwrap_err(); From 7ea729148f33eeea0d3c95c4006b9c238d05ba30 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Mon, 6 Mar 2023 22:56:14 +0800 Subject: [PATCH 24/26] Reverted accidental Cargo.toml formatting --- Cargo.lock | 9 +++- lib/c-api/Cargo.toml | 41 +++++++++------ lib/cli/Cargo.toml | 107 ++++++++++++++++++---------------------- lib/registry/Cargo.toml | 8 +-- lib/vfs/Cargo.toml | 15 ++---- 5 files changed, 85 insertions(+), 95 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9910f809e0f..f927bfba610 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2379,6 +2379,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "nuke-dir" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6842d8099b88d19a64158a6cfdc3e9ad82c738c041dab98280ef7ba98d64fa" + [[package]] name = "num-integer" version = "0.1.45" @@ -4864,6 +4870,7 @@ dependencies = [ "libc", "log", "minisign", + "nuke-dir", "object 0.30.3", "pathdiff", "prettytable-rs", @@ -5204,11 +5211,9 @@ dependencies = [ "derivative", "filetime", "fs_extra", - "indexmap", "lazy_static", "libc", "pin-project-lite", - "serde", "slab", "thiserror", "tokio", diff --git a/lib/c-api/Cargo.toml b/lib/c-api/Cargo.toml index 2520c9fd56f..04413d379ac 100644 --- a/lib/c-api/Cargo.toml +++ b/lib/c-api/Cargo.toml @@ -16,25 +16,20 @@ edition = "2018" # `libwasmer.so`, `libwasmer.dylib`, `wasmer.dll` etc. But it creates # a conflict with the existing `wasmer` crate, see below. name = "wasmer" # ##lib.name## -# ^ DO NOT REMOVE, it's used the `Makefile`, see `build-docs-capi`. + # ^ DO NOT REMOVE, it's used the `Makefile`, see `build-docs-capi`. crate-type = ["staticlib", "cdylib"] #"cdylib", "rlib", "staticlib"] [dependencies] # We rename `wasmer` to `wasmer-api` to avoid the conflict with this # library name (see `[lib]`). -wasmer-api = { version = "=3.2.0-alpha.1", path = "../api", default-features = false, features = [ - "sys", -], package = "wasmer" } +wasmer-api = { version = "=3.2.0-alpha.1", path = "../api", default-features = false, features = ["sys"], package = "wasmer" } wasmer-compiler-cranelift = { version = "=3.2.0-alpha.1", path = "../compiler-cranelift", optional = true } wasmer-compiler-singlepass = { version = "=3.2.0-alpha.1", path = "../compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=3.2.0-alpha.1", path = "../compiler-llvm", optional = true } wasmer-emscripten = { version = "=3.2.0-alpha.1", path = "../emscripten", optional = true } wasmer-compiler = { version = "=3.2.0-alpha.1", path = "../compiler" } wasmer-middlewares = { version = "=3.2.0-alpha.1", path = "../middlewares", optional = true } -wasmer-wasi = { version = "=3.2.0-alpha.1", path = "../wasi", features = [ - "host-fs", - "host-vnet", -], optional = true } +wasmer-wasi = { version = "=3.2.0-alpha.1", path = "../wasi", features = ["host-fs", "host-vnet"], optional = true } wasmer-types = { version = "=3.2.0-alpha.1", path = "../types" } wasmer-vfs = { version = "=3.2.0-alpha.1", path = "../vfs", optional = true, default-features = false, features = ["static-fs"] } webc = { version = "5.0.0-rc.5", optional = true } @@ -56,10 +51,19 @@ wasmer-inline-c = "0.1.1" inline-c = "0.1.7" [features] -default = ["wat", "cranelift", "compiler", "wasi", "middlewares"] +default = [ + "wat", + "cranelift", + "compiler", + "wasi", + "middlewares", +] wat = ["wasmer-api/wat"] wasi = ["wasmer-wasi"] -middlewares = ["compiler", "wasmer-middlewares"] +middlewares = [ + "compiler", + "wasmer-middlewares", +] compiler = [ "wasmer-api/compiler", "wasmer-compiler/translator", @@ -72,14 +76,23 @@ compiler-headless = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", ] -singlepass = ["wasmer-compiler-singlepass", "compiler"] -cranelift = ["wasmer-compiler-cranelift", "compiler"] -llvm = ["wasmer-compiler-llvm", "compiler"] +singlepass = [ + "wasmer-compiler-singlepass", + "compiler", +] +cranelift = [ + "wasmer-compiler-cranelift", + "compiler", +] +llvm = [ + "wasmer-compiler-llvm", + "compiler", +] wasmer-artifact-load = ["wasmer-compiler/wasmer-artifact-load"] wasmer-artifact-create = ["wasmer-compiler/wasmer-artifact-create"] static-artifact-load = ["wasmer-compiler/static-artifact-load"] static-artifact-create = ["wasmer-compiler/static-artifact-create"] -webc_runner = ["wasmer-wasi/webc_runner", "webc", "wasmer-vfs"] +webc_runner = ["wasmer-wasi/webc_runner", "wasmer-vfs", "webc"] # Deprecated features. jit = ["compiler"] diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index bb2f3eb50db..4fe65e42d8d 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -26,30 +26,22 @@ required-features = ["headless"] [dependencies] wasmer = { version = "=3.2.0-alpha.1", path = "../api", default-features = false } -wasmer-compiler = { version = "=3.2.0-alpha.1", path = "../compiler", features = [ - "compiler", -] } +wasmer-compiler = { version = "=3.2.0-alpha.1", path = "../compiler", features = ["compiler", ] } wasmer-compiler-cranelift = { version = "=3.2.0-alpha.1", path = "../compiler-cranelift", optional = true } wasmer-compiler-singlepass = { version = "=3.2.0-alpha.1", path = "../compiler-singlepass", optional = true } wasmer-compiler-llvm = { version = "=3.2.0-alpha.1", path = "../compiler-llvm", optional = true } wasmer-emscripten = { version = "=3.2.0-alpha.1", path = "../emscripten", optional = true } wasmer-vm = { version = "=3.2.0-alpha.1", path = "../vm" } wasmer-wasi = { version = "=3.2.0-alpha.1", path = "../wasi", optional = true } -wasmer-wasi-experimental-io-devices = { version = "=3.2.0-alpha.1", path = "../wasi-experimental-io-devices", optional = true, features = [ - "link_external_libs", -] } +wasmer-wasi-experimental-io-devices = { version = "=3.2.0-alpha.1", path = "../wasi-experimental-io-devices", optional = true, features = ["link_external_libs"] } wasmer-wasi-local-networking = { version = "=3.2.0-alpha.1", path = "../wasi-local-networking", optional = true } wasmer-wast = { version = "=3.2.0-alpha.1", path = "../../tests/lib/wast", optional = true } wasmer-cache = { version = "=3.2.0-alpha.1", path = "../cache", optional = true } -wasmer-types = { version = "=3.2.0-alpha.1", path = "../types", features = [ - "enable-serde", -] } +wasmer-types = { version = "=3.2.0-alpha.1", path = "../types", features = ["enable-serde"] } wasmer-registry = { version = "=4.0.0", path = "../registry" } wasmer-object = { version = "=3.2.0-alpha.1", path = "../object", optional = true } -wasmer-vfs = { version = "=3.2.0-alpha.1", path = "../vfs", default-features = false, features = [ - "host-fs", -] } -wasmer-vnet = { version = "=3.2.0-alpha.1", path = "../vnet" } +wasmer-vfs = { version = "=3.2.0-alpha.1", path = "../vfs", default-features = false, features = ["host-fs"] } +wasmer-vnet = { version = "=3.2.0-alpha.1", path = "../vnet" } wasmer-wasm-interface = { version = "3.2.0-alpha.1", path = "../wasm-interface" } wasmparser = "0.51.4" atty = "0.2" @@ -64,7 +56,7 @@ bytesize = "1.0" cfg-if = "1.0" # For debug feature fern = { version = "0.6", features = ["colored"], optional = true } -tempfile = "3" +tempfile = "3.4.0" http_req = { version="^0.8", default-features = false, features = ["rust-tls"] } reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json", "multipart"] } serde = { version = "1.0.147", features = ["derive"] } @@ -79,6 +71,7 @@ regex = "1.6.0" toml = "0.5.9" url = "2.3.1" libc = { version = "^0.2", default-features = false } +nuke-dir = { version = "0.1.0", optional = true } webc = { version = "5.0.0-rc.5", optional = true } isatty = "0.1.9" dialoguer = "0.10.2" @@ -89,11 +82,7 @@ cargo_metadata = "0.15.2" rusqlite = { version = "0.28.0", features = ["bundled"] } tar = "0.4.38" thiserror = "1.0.37" -time = { version = "0.3.17", default-features = false, features = [ - "parsing", - "std", - "formatting", -] } +time = { version = "0.3.17", default-features = false, features = ["parsing", "std", "formatting"] } log = "0.4.17" minisign = "0.7.2" semver = "1.0.14" @@ -104,10 +93,7 @@ object = "0.30.0" wasm-coredump-builder = { version = "0.1.11" } [build-dependencies] -chrono = { version = "^0.4", default-features = false, features = [ - "std", - "clock", -] } +chrono = { version = "^0.4", default-features = false, features = [ "std", "clock" ] } [target.'cfg(target_os = "linux")'.dependencies] unix_mode = "0.1.3" @@ -132,50 +118,51 @@ wast = ["wasmer-wast"] wasi = ["wasmer-wasi", "wasmer-wasi-local-networking"] emscripten = ["wasmer-emscripten"] wat = ["wasmer/wat"] -webc_runner = [ - "wasi", - "wasmer-wasi/webc_runner", - "wasmer-wasi/webc_runner_rt_wasi", - "wasmer-wasi/webc_runner_rt_emscripten", - "wasmer-wasi/webc_runner_rt_wcgi", - "webc", -] +webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "nuke-dir", "webc"] compiler = [ "wasmer-compiler/translator", "wasmer-compiler/compiler", - "wasmer-wasi/compiler", + "wasmer-wasi/compiler" ] -wasmer-artifact-create = [ - "compiler", - "wasmer/wasmer-artifact-load", - "wasmer/wasmer-artifact-create", - "wasmer-compiler/wasmer-artifact-load", - "wasmer-compiler/wasmer-artifact-create", - "wasmer-object", +wasmer-artifact-create = ["compiler", + "wasmer/wasmer-artifact-load", + "wasmer/wasmer-artifact-create", + "wasmer-compiler/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-create", + "wasmer-object", + ] +static-artifact-create = ["compiler", + "wasmer/static-artifact-load", + "wasmer/static-artifact-create", + "wasmer-compiler/static-artifact-load", + "wasmer-compiler/static-artifact-create", + "wasmer-object", + ] +wasmer-artifact-load = ["compiler", + "wasmer/wasmer-artifact-load", + "wasmer-compiler/wasmer-artifact-load", + ] +static-artifact-load = ["compiler", + "wasmer/static-artifact-load", + "wasmer-compiler/static-artifact-load", + ] + +experimental-io-devices = [ + "wasmer-wasi-experimental-io-devices", + "wasi" ] -static-artifact-create = [ +singlepass = [ + "wasmer-compiler-singlepass", "compiler", - "wasmer/static-artifact-load", - "wasmer/static-artifact-create", - "wasmer-compiler/static-artifact-load", - "wasmer-compiler/static-artifact-create", - "wasmer-object", ] -wasmer-artifact-load = [ +cranelift = [ + "wasmer-compiler-cranelift", "compiler", - "wasmer/wasmer-artifact-load", - "wasmer-compiler/wasmer-artifact-load", ] -static-artifact-load = [ +llvm = [ + "wasmer-compiler-llvm", "compiler", - "wasmer/static-artifact-load", - "wasmer-compiler/static-artifact-load", ] - -experimental-io-devices = ["wasmer-wasi-experimental-io-devices", "wasi"] -singlepass = ["wasmer-compiler-singlepass", "compiler"] -cranelift = ["wasmer-compiler-cranelift", "compiler"] -llvm = ["wasmer-compiler-llvm", "compiler"] debug = ["fern", "wasmer-wasi/logging"] disable-all-logging = ["wasmer-wasi/disable-all-logging", "log/release_max_level_off"] headless = [] @@ -183,10 +170,10 @@ headless-minimal = ["headless", "disable-all-logging", "wasi"] # Optional enable-serde = [ - "wasmer/enable-serde", - "wasmer-vm/enable-serde", - "wasmer-compiler/enable-serde", - "wasmer-wasi/enable-serde", + "wasmer/enable-serde", + "wasmer-vm/enable-serde", + "wasmer-compiler/enable-serde", + "wasmer-wasi/enable-serde", ] [target.'cfg(target_os = "windows")'.dependencies] diff --git a/lib/registry/Cargo.toml b/lib/registry/Cargo.toml index aee1820cd97..217b6609686 100644 --- a/lib/registry/Cargo.toml +++ b/lib/registry/Cargo.toml @@ -13,13 +13,7 @@ dirs = "4.0.0" graphql_client = "0.11.0" serde = { version = "1.0.145", features = ["derive"] } anyhow = "1.0.65" -reqwest = { version = "0.11.12", default-features = false, features = [ - "rustls-tls", - "blocking", - "multipart", - "json", - "stream", -] } +reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json", "stream"] } futures-util = "0.3.25" whoami = "1.2.3" serde_json = "1.0.85" diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index b848c09f26e..5abf8dd3023 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -20,25 +20,16 @@ lazy_static = "1.4" fs_extra = { version = "1.2.0", optional = true } filetime = { version = "0.2.18", optional = true } bytes = "1" -tokio = { version = "1", features = [ - "io-util", - "sync", - "macros", -], default_features = false } +tokio = { version = "1", features = [ "io-util", "sync", "macros" ], default_features = false } pin-project-lite = "0.2.9" -indexmap = "1.9.2" -serde = { version = "1", optional = true, features = ["derive"] } [dev-dependencies] -tokio = { version = "1", features = [ - "io-util", - "rt", -], default_features = false } +tokio = { version = "1", features = [ "io-util", "rt" ], default_features = false } [features] default = ["host-fs", "webc-fs", "static-fs"] host-fs = ["libc", "fs_extra", "filetime", "tokio/fs", "tokio/io-std"] webc-fs = ["webc", "anyhow"] static-fs = ["webc", "anyhow"] -enable-serde = ["serde", "typetag"] +enable-serde = ["typetag"] no-time = [] From 114eeb357e648f5dc6cb1c056125716edad67410 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Mon, 6 Mar 2023 23:16:13 +0800 Subject: [PATCH 25/26] We lost wasmer-vfs's dependency on indexmap during a rebase --- Cargo.lock | 1 + lib/vfs/Cargo.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f927bfba610..e8a06b68be9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5211,6 +5211,7 @@ dependencies = [ "derivative", "filetime", "fs_extra", + "indexmap", "lazy_static", "libc", "pin-project-lite", diff --git a/lib/vfs/Cargo.toml b/lib/vfs/Cargo.toml index 5abf8dd3023..6aa5bdd6183 100644 --- a/lib/vfs/Cargo.toml +++ b/lib/vfs/Cargo.toml @@ -22,6 +22,7 @@ filetime = { version = "0.2.18", optional = true } bytes = "1" tokio = { version = "1", features = [ "io-util", "sync", "macros" ], default_features = false } pin-project-lite = "0.2.9" +indexmap = "1.9.2" [dev-dependencies] tokio = { version = "1", features = [ "io-util", "rt" ], default_features = false } From e84fdf56a72360972b4d2ea20d5d812df821ae65 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Mon, 6 Mar 2023 23:37:15 +0800 Subject: [PATCH 26/26] Make sure we include the WCGI feature flag --- lib/cli/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 4fe65e42d8d..bd9555064a7 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -118,7 +118,7 @@ wast = ["wasmer-wast"] wasi = ["wasmer-wasi", "wasmer-wasi-local-networking"] emscripten = ["wasmer-emscripten"] wat = ["wasmer/wat"] -webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "nuke-dir", "webc"] +webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_wcgi", "wasmer-wasi/webc_runner_rt_emscripten", "nuke-dir", "webc"] compiler = [ "wasmer-compiler/translator", "wasmer-compiler/compiler",