Merge #1693

1693: Add `wasmer create-exe` r=MarkMcCaskey a=MarkMcCaskey

This adds the `wasmer create-exe` subcommand.  This subcommand is a combination of `wasmer compile --object-file` and linking that compiled Wasm with a main function and libwasmer.  Put more plainly: it lets us transform Wasm modules into native executables in one step.

In order for this to work we need:
- [x] Ship wasm.h with Wasmer or use different mechanism to find it
- [x] Ship wasmer_wasm.h with Wasmer
- [x] Requires up to date libwasmer... had to build one and copy it over, may fail in CI because of this... will be fixed with next release though
- [x] More gracefully handle wasmer installed without WASMER_DIR, etc (even if just error messages, should be tested)
- [x] Add tests

# Review

- [ ] Add a short description of the the change to the CHANGELOG.md file


Co-authored-by: Mark McCaskey <[email protected]>
Co-authored-by: Mark McCaskey <[email protected]>

.github/workflows/main.yaml

@@ -150,19 +150,25 @@ jobs:
         run: |
           make build-wapm
         if: needs.setup.outputs.DOING_RELEASE == '1'
+      - name: Package Wasmer for integration tests
+        run: make package-without-wapm-for-integration-tests
+        if: needs.setup.outputs.DOING_RELEASE != '1'
+      - name: Package Wasmer
+        run: |
+          make package
+        if: needs.setup.outputs.DOING_RELEASE == '1'
       - name: Run integration tests (Windows)
         shell: cmd
         run: |
           call refreshenv
+          set WASMER_DIR=%CD%\package
           make test-integration
         if: matrix.run_integration_tests && matrix.os == 'windows-latest'
       - name: Run integration tests (Unix)
-        run: make test-integration
-        if: matrix.run_integration_tests && matrix.os != 'windows-latest'
-      - name: Package Wasmer
         run: |
-          make package
-        if: needs.setup.outputs.DOING_RELEASE == '1'
+          export WASMER_DIR=`pwd`/package
+          make test-integration
+        if: matrix.run_integration_tests && matrix.os != 'windows-latest'
       - name: Upload Artifacts
         uses: actions/upload-artifact@v2
         if: needs.setup.outputs.DOING_RELEASE == '1'

Makefile

@@ -185,6 +185,7 @@ package-capi:
 	mkdir -p "package/lib"
 	cp lib/c-api/wasmer.h* package/include
 	cp lib/c-api/wasmer_wasm.h* package/include
+	cp lib/c-api/wasm.h* package/include
 	cp lib/c-api/doc/deprecated/index.md package/include/README.md
 ifeq ($(OS), Windows_NT)
 	cp target/release/wasmer_c_api.dll package/lib
@@ -223,6 +224,9 @@ else
 	cp ./wasmer.tar.gz ./dist/$(shell ./scripts/binary-name.sh)
 endif
 
+# command for simulating installing Wasmer without wapm.
+package-without-wapm-for-integration-tests: package-wasmer package-capi
+
 #################
 # Miscellaneous #
 #################

lib/c-api/build.rs

@@ -361,6 +361,8 @@ fn exclude_items_from_wasm_c_api(builder: Builder) -> Builder {
     builder
         .exclude_item("wasi_config_arg")
         .exclude_item("wasi_config_env")
+        .exclude_item("wasi_config_mapdir")
+        .exclude_item("wasi_config_preopen_dir")
         .exclude_item("wasi_config_inherit_stderr")
         .exclude_item("wasi_config_inherit_stdin")
         .exclude_item("wasi_config_inherit_stdout")

lib/c-api/src/wasm_c_api/wasi/mod.rs

@@ -75,6 +75,63 @@ pub unsafe extern "C" fn wasi_config_arg(config: &mut wasi_config_t, arg: *const
     config.state_builder.arg(arg_bytes);
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn wasi_config_preopen_dir(
+    config: &mut wasi_config_t,
+    dir: *const c_char,
+) -> bool {
+    let dir_cstr = CStr::from_ptr(dir);
+    let dir_bytes = dir_cstr.to_bytes();
+    let dir_str = match std::str::from_utf8(dir_bytes) {
+        Ok(dir_str) => dir_str,
+        Err(e) => {
+            update_last_error(e);
+            return false;
+        }
+    };
+
+    if let Err(e) = config.state_builder.preopen_dir(dir_str) {
+        update_last_error(e);
+        return false;
+    }
+
+    true
+}
+
+#[no_mangle]
+pub unsafe extern "C" fn wasi_config_mapdir(
+    config: &mut wasi_config_t,
+    alias: *const c_char,
+    dir: *const c_char,
+) -> bool {
+    let alias_cstr = CStr::from_ptr(alias);
+    let alias_bytes = alias_cstr.to_bytes();
+    let alias_str = match std::str::from_utf8(alias_bytes) {
+        Ok(alias_str) => alias_str,
+        Err(e) => {
+            update_last_error(e);
+            return false;
+        }
+    };
+
+    let dir_cstr = CStr::from_ptr(dir);
+    let dir_bytes = dir_cstr.to_bytes();
+    let dir_str = match std::str::from_utf8(dir_bytes) {
+        Ok(dir_str) => dir_str,
+        Err(e) => {
+            update_last_error(e);
+            return false;
+        }
+    };
+
+    if let Err(e) = config.state_builder.map_dir(alias_str, dir_str) {
+        update_last_error(e);
+        return false;
+    }
+
+    true
+}
+
 #[no_mangle]
 pub extern "C" fn wasi_config_inherit_stdout(config: &mut wasi_config_t) {
     config.inherit_stdout = true;

lib/c-api/wasmer_wasm.h

@@ -101,10 +101,18 @@ void wasi_config_inherit_stdin(wasi_config_t *config);
 void wasi_config_inherit_stdout(wasi_config_t *config);
 #endif
 
+#if defined(WASMER_WASI_ENABLED)
+bool wasi_config_mapdir(wasi_config_t *config, const char *alias, const char *dir);
+#endif
+
 #if defined(WASMER_WASI_ENABLED)
 wasi_config_t *wasi_config_new(const char *program_name);
 #endif
 
+#if defined(WASMER_WASI_ENABLED)
+bool wasi_config_preopen_dir(wasi_config_t *config, const char *dir);
+#endif
+
 #if defined(WASMER_WASI_ENABLED)
 void wasi_env_delete(wasi_env_t *_state);
 #endif

lib/cli/Cargo.toml

@@ -45,6 +45,7 @@ cfg-if = "0.1"
 # For debug feature
 fern = { version = "0.6", features = ["colored"], optional = true }
 log = { version = "0.4", optional = true }
+tempfile = "3"
 
 [features]
 # Don't add the compiler features in default, please add them on the Makefile

lib/cli/src/bin/wasmer.rs

@@ -1,4 +1,6 @@
 use anyhow::Result;
+#[cfg(all(feature = "object-file", feature = "compiler"))]
+use wasmer_cli::commands::CreateExe;
 #[cfg(feature = "wast")]
 use wasmer_cli::commands::Wast;
 use wasmer_cli::commands::{Cache, Compile, Config, Inspect, Run, SelfUpdate, Validate};
@@ -26,6 +28,11 @@ enum WasmerCLIOptions {
     #[structopt(name = "compile")]
     Compile(Compile),
 
+    /// Compile a WebAssembly binary into a native executable
+    #[cfg(all(feature = "object-file", feature = "compiler"))]
+    #[structopt(name = "create-exe")]
+    CreateExe(CreateExe),
+
     /// Get various configuration information needed
     /// to compile programs which use Wasmer
     #[structopt(name = "config")]
@@ -53,6 +60,8 @@ impl WasmerCLIOptions {
             Self::Cache(cache) => cache.execute(),
             Self::Validate(validate) => validate.execute(),
             Self::Compile(compile) => compile.execute(),
+            #[cfg(all(feature = "object-file", feature = "compiler"))]
+            Self::CreateExe(create_exe) => create_exe.execute(),
             Self::Config(config) => config.execute(),
             Self::Inspect(inspect) => inspect.execute(),
             #[cfg(feature = "wast")]

lib/cli/src/commands.rs

@@ -2,13 +2,17 @@
 mod cache;
 mod compile;
 mod config;
+#[cfg(all(feature = "object-file", feature = "compiler"))]
+mod create_exe;
 mod inspect;
 mod run;
 mod self_update;
 mod validate;
 #[cfg(feature = "wast")]
 mod wast;
 
+#[cfg(all(feature = "object-file", feature = "compiler"))]
+pub use create_exe::*;
 #[cfg(feature = "wast")]
 pub use wast::*;
 pub use {cache::*, compile::*, config::*, inspect::*, run::*, self_update::*, validate::*};

lib/cli/src/commands/compile.rs

@@ -38,8 +38,7 @@ impl Compile {
             .context(format!("failed to compile `{}`", self.path.display()))
     }
 
-    fn get_recommend_extension(
-        &self,
+    pub(crate) fn get_recommend_extension(
         engine_type: &EngineType,
         target_triple: &Triple,
     ) -> &'static str {
@@ -82,7 +81,7 @@ impl Compile {
             .file_stem()
             .map(|osstr| osstr.to_string_lossy().to_string())
             .unwrap_or_default();
-        let recommended_extension = self.get_recommend_extension(&engine_type, target.triple());
+        let recommended_extension = Self::get_recommend_extension(&engine_type, target.triple());
         match self.output.extension() {
             Some(ext) => {
                 if ext != recommended_extension {