Merge pull request #3599 from wasmerio/wcgi-runner

Move the `wcgi-runner` into Wasmer

Cargo.toml

@@ -11,7 +11,9 @@ autoexamples = false
 
 [dependencies]
 wasmer = { version = "=3.2.0-alpha.1", path = "lib/api", default-features = false }
-wasmer-compiler = { version = "=3.2.0-alpha.1", path = "lib/compiler", features = ["compiler"] }
+wasmer-compiler = { version = "=3.2.0-alpha.1", path = "lib/compiler", features = [
+    "compiler",
+] }
 wasmer-compiler-cranelift = { version = "=3.2.0-alpha.1", path = "lib/compiler-cranelift", optional = true }
 wasmer-compiler-singlepass = { version = "=3.2.0-alpha.1", path = "lib/compiler-singlepass", optional = true }
 wasmer-compiler-llvm = { version = "=3.2.0-alpha.1", path = "lib/compiler-llvm", optional = true }
@@ -71,7 +73,9 @@ glob = "0.3"
 rustc_version = "0.4"
 
 [dev-dependencies]
-wasmer = { version = "=3.2.0-alpha.1", path = "lib/api", default-features = false, features = ["cranelift"] }
+wasmer = { version = "=3.2.0-alpha.1", path = "lib/api", default-features = false, features = [
+    "cranelift",
+] }
 anyhow = "1.0"
 criterion = "0.3"
 lazy_static = "1.4"
@@ -105,10 +109,7 @@ wast = ["wasmer-wast"]
 wasi = ["wasmer-wasi"]
 emscripten = ["wasmer-emscripten"]
 wat = ["wasmer/wat"]
-compiler = [
-    "wasmer/compiler",
-    "wasmer-compiler/translator",
-]
+compiler = ["wasmer/compiler", "wasmer-compiler/translator"]
 singlepass = ["wasmer-compiler-singlepass", "compiler"]
 cranelift = ["wasmer-compiler-cranelift", "compiler"]
 llvm = ["wasmer-compiler-llvm", "compiler"]
@@ -123,9 +124,7 @@ test-singlepass = ["singlepass"]
 test-cranelift = ["cranelift"]
 test-llvm = ["llvm"]
 
-test-universal = [
-    "test-generator/test-universal",
-]
+test-universal = ["test-generator/test-universal"]
 
 # Specifies that we're running in coverage testing mode. This disables tests
 # that raise signals because that interferes with tarpaulin.

deny.toml

@@ -64,7 +64,7 @@ git-fetch-with-cli = true
 # * Medium - CVSS Score 4.0 - 6.9
 # * High - CVSS Score 7.0 - 8.9
 # * Critical - CVSS Score 9.0 - 10.0
-#severity-threshold = 
+#severity-threshold =
 
 # This section is considered when running `cargo deny check licenses`
 # More documentation for the licenses section can be found here:
@@ -111,6 +111,7 @@ confidence-threshold = 0.8
 exceptions = [
     # Each entry is the crate and version constraint, and its specific allow
     # list
+    { name = "webc", allow = ["BUSL-1.1"] }
 ]
 
 
@@ -189,8 +190,8 @@ skip = [
     { name = "itoa", version = "=0.4.8" },
     { name = "object", version = "=0.27.1" },
 ]
-# Similarly to `skip` allows you to skip certain crates during duplicate 
-# detection. Unlike skip, it also includes the entire tree of transitive 
+# Similarly to `skip` allows you to skip certain crates during duplicate
+# detection. Unlike skip, it also includes the entire tree of transitive
 # dependencies starting at the specified crate, up to a certain depth, which is
 # by default infinite
 skip-tree = [

lib/c-api/Cargo.toml

@@ -32,7 +32,7 @@ wasmer-middlewares = { version = "=3.2.0-alpha.1", path = "../middlewares", opti
 wasmer-wasi = { version = "=3.2.0-alpha.1", path = "../wasi", features = ["host-fs", "host-vnet"], optional = true }
 wasmer-types = { version = "=3.2.0-alpha.1", path = "../types" }
 wasmer-vfs = { version = "=3.2.0-alpha.1", path = "../vfs", optional = true, default-features = false, features = ["static-fs"] }
-webc = { version = "4.0.0", optional = true }
+webc = { version = "5.0.0-rc.5", optional = true }
 enumset = "1.0.2"
 cfg-if = "1.0"
 lazy_static = "1.4"

lib/c-api/src/wasm_c_api/wasi/mod.rs

@@ -249,10 +249,10 @@ fn prepare_webc_env(
     package_name: &str,
 ) -> Option<(WasiFunctionEnv, Imports)> {
     use wasmer_vfs::static_fs::StaticFileSystem;
-    use webc::FsEntryType;
+    use webc::v1::{FsEntryType, WebC};
 
     let slice = unsafe { std::slice::from_raw_parts(bytes, len) };
-    let volumes = webc::WebC::parse_volumes_from_fileblock(slice).ok()?;
+    let volumes = WebC::parse_volumes_from_fileblock(slice).ok()?;
     let top_level_dirs = volumes
         .into_iter()
         .flat_map(|(_, volume)| {

lib/cli/Cargo.toml

@@ -72,7 +72,7 @@ toml = "0.5.9"
 url = "2.3.1"
 libc = { version = "^0.2", default-features = false }
 nuke-dir = { version = "0.1.0", optional = true }
-webc = { version = "4.0.0", optional = true }
+webc = { version = "5.0.0-rc.5", optional = true }
 isatty = "0.1.9"
 dialoguer = "0.10.2"
 tldextract = "0.6.0"
@@ -118,7 +118,7 @@ wast = ["wasmer-wast"]
 wasi = ["wasmer-wasi", "wasmer-wasi-local-networking"]
 emscripten = ["wasmer-emscripten"]
 wat = ["wasmer/wat"]
-webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "nuke-dir", "webc"]
+webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_wcgi", "wasmer-wasi/webc_runner_rt_emscripten", "nuke-dir", "webc"]
 compiler = [
     "wasmer-compiler/translator",
     "wasmer-compiler/compiler",

lib/cli/src/commands/create_exe.rs

@@ -15,9 +15,8 @@ use std::process::Stdio;
 use tar::Archive;
 use wasmer::*;
 use wasmer_object::{emit_serialized, get_object_for_target};
-use wasmer_types::compilation::symbols::ModuleMetadataSymbolRegistry;
-use wasmer_types::ModuleInfo;
-use webc::{ParseOptions, WebCMmap};
+use wasmer_types::{compilation::symbols::ModuleMetadataSymbolRegistry, ModuleInfo};
+use webc::v1::{ParseOptions, WebCMmap};
 
 const LINK_SYSTEM_LIBRARIES_WINDOWS: &[&str] = &["userenv", "Ws2_32", "advapi32", "bcrypt"];
 
@@ -520,7 +519,7 @@ impl PrefixMapCompilation {
         // if prefixes are specified, have to match the atom names exactly
         if prefixes.len() != atoms.len() {
             println!(
-                "WARNING: invalid mapping of prefix and atoms: expected prefixes for {} atoms, got {} prefixes", 
+                "WARNING: invalid mapping of prefix and atoms: expected prefixes for {} atoms, got {} prefixes",
                 atoms.len(), prefixes.len()
             );
         }

lib/cli/src/commands/create_obj.rs

@@ -85,7 +85,7 @@ impl CreateObj {
         println!("Target: {}", target.triple());
 
         let atoms = if let Ok(pirita) =
-            webc::WebCMmap::parse(input_path.clone(), &webc::ParseOptions::default())
+            webc::v1::WebCMmap::parse(input_path.clone(), &webc::v1::ParseOptions::default())
         {
             crate::commands::create_exe::compile_pirita_into_directory(
                 &pirita,

lib/cli/src/commands/gen_c_header.rs

@@ -5,7 +5,7 @@ use std::path::PathBuf;
 use wasmer_compiler::Artifact;
 use wasmer_types::compilation::symbols::ModuleMetadataSymbolRegistry;
 use wasmer_types::{CpuFeature, MetadataHeader, Triple};
-use webc::WebC;
+use webc::v1::WebC;
 
 #[derive(Debug, Parser)]
 /// The options for the `wasmer gen-c-header` subcommand
@@ -57,7 +57,7 @@ impl GenCHeader {
             None => crate::commands::PrefixMapCompilation::hash_for_bytes(&file),
         };
 
-        if let Ok(pirita) = WebC::parse(&file, &webc::ParseOptions::default()) {
+        if let Ok(pirita) = WebC::parse(&file, &webc::v1::ParseOptions::default()) {
             let atoms = pirita
                 .manifest
                 .atoms

lib/cli/src/commands/run.rs

@@ -8,12 +8,12 @@ use crate::suggestions::suggest_function_exports;
 use crate::warning;
 use anyhow::{anyhow, Context, Result};
 use clap::Parser;
-use std::fs::File;
 use std::io::Write;
 use std::ops::Deref;
 use std::path::PathBuf;
 #[cfg(feature = "cache")]
 use std::str::FromStr;
+use std::{fs::File, net::SocketAddr};
 #[cfg(feature = "emscripten")]
 use wasmer::FunctionEnv;
 use wasmer::*;
@@ -91,6 +91,10 @@ pub struct RunWithoutFile {
     #[clap(long = "verbose")]
     pub(crate) verbose: Option<u8>,
 
+    #[cfg(feature = "webc_runner")]
+    #[clap(flatten)]
+    pub(crate) wcgi: WcgiOptions,
+
     /// Enable coredump generation after a WebAssembly trap.
     #[clap(name = "COREDUMP PATH", long = "coredump-on-trap", parse(from_os_str))]
     coredump_on_trap: Option<PathBuf>,
@@ -223,14 +227,8 @@ impl RunWithPathBuf {
     fn inner_execute(&self) -> Result<()> {
         #[cfg(feature = "webc_runner")]
         {
-            if let Ok(pf) = WapmContainer::new(self.path.clone()) {
-                return self
-                    .run_container(
-                        pf,
-                        &self.command_name.clone().unwrap_or_default(),
-                        &self.args,
-                    )
-                    .map_err(|e| anyhow!("Could not run PiritaFile: {e}"));
+            if let Ok(pf) = WapmContainer::from_path(self.path.clone()) {
+                return self.run_container(pf, self.command_name.as_deref(), &self.args);
             }
         }
         let (mut store, module) = self.get_store_module()?;
@@ -376,48 +374,54 @@ impl RunWithPathBuf {
     fn run_container(
         &self,
         container: WapmContainer,
-        id: &str,
+        id: Option<&str>,
         args: &[String],
     ) -> Result<(), anyhow::Error> {
-        let mut result = None;
-
-        #[cfg(feature = "wasi")]
-        {
-            if let Some(r) = result {
-                return r;
-            }
-
-            let (store, _compiler_type) = self.store.get_store()?;
-            let mut runner = wasmer_wasi::runners::wasi::WasiRunner::new(store);
-            runner.set_args(args.to_vec());
-            result = Some(if id.is_empty() {
-                runner.run(&container).map_err(|e| anyhow::anyhow!("{e}"))
-            } else {
-                runner
-                    .run_cmd(&container, id)
-                    .map_err(|e| anyhow::anyhow!("{e}"))
-            });
+        let id = id
+            .or_else(|| container.manifest().entrypoint.as_deref())
+            .context("No command specified")?;
+        let command = container
+            .manifest()
+            .commands
+            .get(id)
+            .with_context(|| format!("No metadata found for the command, \"{id}\""))?;
+
+        let (store, _compiler_type) = self.store.get_store()?;
+        let mut runner = wasmer_wasi::runners::wasi::WasiRunner::new(store);
+        runner.set_args(args.to_vec());
+        if runner.can_run_command(id, command).unwrap_or(false) {
+            return runner.run_cmd(&container, id).context("WASI runner failed");
         }
 
-        #[cfg(feature = "emscripten")]
-        {
-            if let Some(r) = result {
-                return r;
-            }
+        let (store, _compiler_type) = self.store.get_store()?;
+        let mut runner = wasmer_wasi::runners::emscripten::EmscriptenRunner::new(store);
+        runner.set_args(args.to_vec());
+        if runner.can_run_command(id, command).unwrap_or(false) {
+            return runner
+                .run_cmd(&container, id)
+                .context("Emscripten runner failed");
+        }
 
-            let (store, _compiler_type) = self.store.get_store()?;
-            let mut runner = wasmer_wasi::runners::emscripten::EmscriptenRunner::new(store);
-            runner.set_args(args.to_vec());
-            result = Some(if id.is_empty() {
-                runner.run(&container).map_err(|e| anyhow::anyhow!("{e}"))
-            } else {
-                runner
-                    .run_cmd(&container, id)
-                    .map_err(|e| anyhow::anyhow!("{e}"))
-            });
+        let mut runner = wasmer_wasi::runners::wcgi::WcgiRunner::new(id);
+        let (store, _compiler_type) = self.store.get_store()?;
+        runner
+            .config()
+            .args(args)
+            .store(store)
+            .addr(self.wcgi.addr)
+            .envs(self.wasi.env_vars.clone())
+            .map_directories(self.wasi.mapped_dirs.iter().map(|(g, h)| (h, g)));
+        if self.wcgi.forward_host_env {
+            runner.config().forward_host_env();
+        }
+        if runner.can_run_command(id, command).unwrap_or(false) {
+            return runner.run_cmd(&container, id).context("WCGI runner failed");
         }
 
-        result.unwrap_or_else(|| Err(anyhow::anyhow!("neither emscripten or wasi file")))
+        anyhow::bail!(
+            "Unable to find a runner that supports \"{}\"",
+            command.runner
+        );
     }
 
     fn get_store_module(&self) -> Result<(Store, Module)> {
@@ -698,3 +702,22 @@ fn generate_coredump(
 
     Ok(())
 }
+
+#[derive(Debug, Clone, Parser)]
+pub(crate) struct WcgiOptions {
+    /// The address to serve on.
+    #[clap(long, short, env, default_value_t = ([127, 0, 0, 1], 8000).into())]
+    pub(crate) addr: SocketAddr,
+    /// Forward all host env variables to the wcgi task.
+    #[clap(long)]
+    pub(crate) forward_host_env: bool,
+}
+
+impl Default for WcgiOptions {
+    fn default() -> Self {
+        Self {
+            addr: ([127, 0, 0, 1], 8000).into(),
+            forward_host_env: false,
+        }
+    }
+}

lib/registry/Cargo.toml

@@ -12,10 +12,10 @@ rand = "0.8.5"
 dirs = "4.0.0"
 graphql_client = "0.11.0"
 serde = { version = "1.0.145", features = ["derive"] }
-anyhow = "1.0.65" 
+anyhow = "1.0.65"
 reqwest = { version = "0.11.12", default-features = false, features = ["rustls-tls", "blocking", "multipart", "json", "stream"] }
 futures-util = "0.3.25"
-whoami = "1.2.3" 
+whoami = "1.2.3"
 serde_json = "1.0.85"
 url = "2.3.1"
 thiserror = "1.0.37"
@@ -25,7 +25,7 @@ tar = "0.4.38"
 flate2 = "1.0.24"
 semver = "1.0.14"
 lzma-rs = "0.2.0"
-webc = { version ="4.0.0", features = ["mmap"] }
+webc = { version = "5.0.0-rc.5", features = ["mmap"] }
 hex = "0.4.3"
 tokio = "1.24.0"
 log = "0.4.17"

lib/registry/src/lib.rs

@@ -690,7 +690,7 @@ pub fn get_all_available_registries(wasmer_dir: &Path) -> Result<Vec<String>, St
 #[derive(Debug, PartialEq, Clone)]
 pub struct RemoteWebcInfo {
     pub checksum: String,
-    pub manifest: webc::Manifest,
+    pub manifest: webc::metadata::Manifest,
 }
 
 pub fn install_webc_package(
@@ -772,9 +772,9 @@ fn get_all_installed_webc_packages_inner(wasmer_dir: &Path) -> Vec<RemoteWebcInf
     read_dir
         .filter_map(|r| Some(r.ok()?.path()))
         .filter_map(|path| {
-            webc::WebCMmap::parse(
+            webc::v1::WebCMmap::parse(
                 path,
-                &webc::ParseOptions {
+                &webc::v1::ParseOptions {
                     parse_atoms: false,
                     parse_volumes: false,
                     ..Default::default()
@@ -812,11 +812,11 @@ pub fn get_checksum_hash(bytes: &[u8]) -> String {
 /// Returns the checksum of the .webc file, so that we can check whether the
 /// file is already installed before downloading it
 pub fn get_remote_webc_checksum(url: &Url) -> Result<String, anyhow::Error> {
-    let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
+    let request_max_bytes = webc::v1::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
     let data = get_webc_bytes(url, Some(0..request_max_bytes), None)
         .with_context(|| anyhow::anyhow!("note: use --registry to change the registry URL"))?
         .unwrap();
-    let checksum = webc::WebC::get_checksum_bytes(&data)
+    let checksum = webc::v1::WebC::get_checksum_bytes(&data)
         .map_err(|e| anyhow::anyhow!("{e}"))?
         .to_vec();
     Ok(get_checksum_hash(&checksum))
@@ -826,20 +826,20 @@ pub fn get_remote_webc_checksum(url: &Url) -> Result<String, anyhow::Error> {
 /// so we can see if the package has already been installed
 pub fn get_remote_webc_manifest(url: &Url) -> Result<RemoteWebcInfo, anyhow::Error> {
     // Request up unti manifest size / manifest len
-    let request_max_bytes = webc::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
+    let request_max_bytes = webc::v1::WebC::get_signature_offset_start() + 4 + 1024 + 8 + 8;
     let data = get_webc_bytes(url, Some(0..request_max_bytes), None)?.unwrap();
-    let checksum = webc::WebC::get_checksum_bytes(&data)
+    let checksum = webc::v1::WebC::get_checksum_bytes(&data)
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("WebC::get_checksum_bytes failed")?
         .to_vec();
     let hex_string = get_checksum_hash(&checksum);
 
-    let (manifest_start, manifest_len) = webc::WebC::get_manifest_offset_size(&data)
+    let (manifest_start, manifest_len) = webc::v1::WebC::get_manifest_offset_size(&data)
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("WebC::get_manifest_offset_size failed")?;
     let data_with_manifest =
         get_webc_bytes(url, Some(0..manifest_start + manifest_len), None)?.unwrap();
-    let manifest = webc::WebC::get_manifest(&data_with_manifest)
+    let manifest = webc::v1::WebC::get_manifest(&data_with_manifest)
         .map_err(|e| anyhow::anyhow!("{e}"))
         .context("WebC::get_manifest failed")?;
     Ok(RemoteWebcInfo {