Fix create-exe to be able to cross-compile on Windows

lib/cli/Cargo.toml

@@ -55,11 +55,11 @@ fern = { version = "0.6", features = ["colored"], optional = true }
 log = { version = "0.4", optional = true }
 tempfile = "3"
 tempdir = "0.3.7"
-http_req  = { version="^0.8", default-features = false, features = ["rust-tls"], optional = true }
-reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json", "multipart"], optional = true }
-serde = { version = "1.0.147", features = ["derive"], optional = true }
-dirs = { version = "4.0", optional = true }
-serde_json = { version = "1.0", optional = true }
+http_req  = { version="^0.8", default-features = false, features = ["rust-tls"] }
+reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json", "multipart"] }
+serde = { version = "1.0.147", features = ["derive"] }
+dirs = { version = "4.0" }
+serde_json = { version = "1.0" }
 target-lexicon = { version = "0.12", features = ["std"] }
 prettytable-rs = "0.9.0"
 wapm-toml = "0.2.0"
@@ -86,7 +86,6 @@ unix_mode = "0.1.3"
 default = [
     "wat",
     "wast",
-    "http",
     "cache",
     "wasi",
     "emscripten",
@@ -158,14 +157,6 @@ enable-serde = [
   "wasmer-wasi/enable-serde",
 ]
 
-http = [
-  "http_req",
-  "reqwest",
-  "dirs",
-  "serde_json",
-  "serde",
-]
-
 [target.'cfg(target_os = "windows")'.dependencies]
 colored = "2.0.0"
 

lib/cli/src/commands/create_exe.rs

@@ -402,52 +402,34 @@ impl CreateExe {
             let library = if let Some(v) = cross_subc.library_path.clone() {
                 v.canonicalize().unwrap_or(v)
             } else {
-                {
-                    let libwasmer_path = "lib/libwasmer.a";
-                    let tarball_dir;
-                    let filename = if let Some(local_tarball) = cross_subc.tarball.as_ref() {
-                        let target_file_path = local_tarball
-                            .parent()
-                            .and_then(|parent| Some(parent.join(local_tarball.file_stem()?)))
-                            .unwrap_or_else(|| local_tarball.clone());
-
-                        let target_file_path = target_file_path
-                            .parent()
-                            .and_then(|parent| Some(parent.join(target_file_path.file_stem()?)))
-                            .unwrap_or_else(|| target_file_path.clone());
-
-                        let _ = std::fs::create_dir_all(&target_file_path);
-                        let files = untar(local_tarball.clone(), target_file_path.clone())?;
-                        tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path);
-                        files.iter().find(|f| f.contains(libwasmer_path)).cloned().ok_or_else(|| {
-                            anyhow!("Could not find libwasmer for {} target in the provided tarball path (files = {files:#?}, libwasmer_path = {libwasmer_path:?})", target)})?
+                let (filename, tarball_dir) =
+                    if let Some(local_tarball) = cross_subc.tarball.as_ref() {
+                        Self::find_filename(local_tarball, &target)
                     } else {
-                        #[cfg(feature = "http")]
-                        {
+                        // check if the tarball for the target already exists locally
+                        let local_tarball = std::fs::read_dir(get_libwasmer_cache_path()?)?
+                            .filter_map(|e| e.ok())
+                            .filter_map(|e| {
+                                let path = format!("{}", e.path().display());
+                                if path.ends_with(".tar.gz") {
+                                    Some(e.path())
+                                } else {
+                                    None
+                                }
+                            })
+                            .filter_map(|p| Self::filter_tarballs(&p, &target))
+                            .next();
+
+                        if let Some(local_tarball) = local_tarball.as_ref() {
+                            Self::find_filename(local_tarball, &target)
+                        } else {
                             let release = http_fetch::get_latest_release()?;
                             let tarball = http_fetch::download_release(release, target.clone())?;
-                            let target_file_path = tarball
-                                .parent()
-                                .and_then(|parent| Some(parent.join(tarball.file_stem()?)))
-                                .unwrap_or_else(|| tarball.clone());
-
-                            let target_file_path = target_file_path
-                                .parent()
-                                .and_then(|parent| Some(parent.join(target_file_path.file_stem()?)))
-                                .unwrap_or_else(|| target_file_path.clone());
-
-                            tarball_dir = target_file_path
-                                .canonicalize()
-                                .unwrap_or_else(|_| target_file_path.clone());
-                            let files = untar(tarball, target_file_path)?;
-                            files.into_iter().find(|f| f.contains(libwasmer_path)).ok_or_else(|| {
-                                anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag.", target)})?
+                            Self::find_filename(&tarball, &target)
                         }
-                        #[cfg(not(feature = "http"))]
-                        return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag."));
-                    };
-                    tarball_dir.join(&filename)
-                }
+                    }?;
+
+                tarball_dir.join(&filename)
             };
             let ccs = CrossCompileSetup {
                 target,
@@ -460,6 +442,70 @@ impl CreateExe {
         }
     }
 
+    fn find_filename(
+        local_tarball: &Path,
+        target: &Triple,
+    ) -> Result<(String, PathBuf), anyhow::Error> {
+        let target_file_path = local_tarball
+            .parent()
+            .and_then(|parent| Some(parent.join(local_tarball.file_stem()?)))
+            .unwrap_or_else(|| local_tarball.to_path_buf());
+
+        let target_file_path = target_file_path
+            .parent()
+            .and_then(|parent| Some(parent.join(target_file_path.file_stem()?)))
+            .unwrap_or_else(|| target_file_path.clone());
+
+        let _ = std::fs::create_dir_all(&target_file_path);
+        let files = untar(local_tarball.to_path_buf(), target_file_path.clone())?;
+        let tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path);
+
+        let file = files
+        .iter()
+        .find(|f| f.ends_with("libwasmer.a")).cloned()
+        .ok_or_else(|| {
+            anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)
+        })?;
+
+        Ok((file, tarball_dir))
+    }
+
+    fn filter_tarballs(p: &Path, target: &Triple) -> Option<PathBuf> {
+        if let Architecture::Aarch64(_) = target.architecture {
+            if !p.file_name()?.to_str()?.contains("aarch64") {
+                return None;
+            }
+        }
+
+        if let Architecture::X86_64 = target.architecture {
+            if !p.file_name()?.to_str()?.contains("x86_64") {
+                return None;
+            }
+        }
+
+        if let OperatingSystem::Windows = target.operating_system {
+            if !p.file_name()?.to_str()?.contains("windows") {
+                return None;
+            }
+        }
+
+        if let OperatingSystem::Darwin = target.operating_system {
+            if !(p.file_name()?.to_str()?.contains("apple")
+                || p.file_name()?.to_str()?.contains("darwin"))
+            {
+                return None;
+            }
+        }
+
+        if let OperatingSystem::Linux = target.operating_system {
+            if !p.file_name()?.to_str()?.contains("linux") {
+                return None;
+            }
+        }
+
+        Some(p.to_path_buf())
+    }
+
     fn compile_c(
         &self,
         wasm_object_path: PathBuf,
@@ -572,10 +618,6 @@ impl CreateExe {
                 cmd.arg("-lc");
             }
             cmd.arg("-lunwind");
-            cmd.arg("-OReleaseSafe");
-            cmd.arg("-fstrip");
-            cmd.arg("-dead_strip");
-            cmd.arg("-dead_strip_dylibs");
             cmd.arg("-fno-compiler-rt");
             cmd.arg(&format!("-femit-bin={}", output_path.display()));
 
@@ -1329,7 +1371,6 @@ impl LinkCode {
     }
 }
 
-#[cfg(feature = "http")]
 mod http_fetch {
     use anyhow::{anyhow, Context, Result};
     use http_req::{request::Request, response::StatusCode, uri::Uri};
@@ -1379,7 +1420,7 @@ mod http_fetch {
         }
 
         Err(anyhow!(
-            "Could not get expected Github API response.\n\nReason: response format is not recognized:\n{:#?}", ""
+            "Could not get expected Github API response.\n\nReason: response format is not recognized:\n{response:#?}",
         ))
     }
 

lib/compiler-singlepass/src/machine_x64.rs

@@ -45,7 +45,7 @@ impl AssemblerX64 {
                 Some(CpuFeature::SSE42)
             } else {
                 return Err(CompileError::UnsupportedTarget(
-                    "x86_64 without AVX or SSE 4.2".to_string(),
+                    "x86_64 without AVX or SSE 4.2, use -m avx to enable".to_string(),
                 ));
             }
         };

tests/integration/cli/tests/run.rs

@@ -21,7 +21,6 @@ fn test_no_start_wat_path() -> PathBuf {
     Path::new(ASSET_PATH).join("no_start.wat")
 }
 
-#[cfg(any(target_os = "linux", target_os = "macos"))]
 #[test]
 fn test_cross_compile_python_windows() -> anyhow::Result<()> {
     let temp_dir = tempfile::TempDir::new()?;
@@ -34,41 +33,70 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> {
         "x86_64-windows-gnu",
     ];
 
-    for t in targets {
-        let python_wasmer_path = temp_dir.path().join(format!("{t}-python"));
-
-        let mut output = Command::new(get_wasmer_path());
-
-        output.arg("create-exe");
-        output.arg(wasi_test_python_path());
-        output.arg("--target");
-        output.arg(t);
-        output.arg("-o");
-        output.arg(python_wasmer_path.clone());
-        let output = output.output()?;
-
-        let stdout = std::str::from_utf8(&output.stdout)
-            .expect("stdout is not utf8! need to handle arbitrary bytes");
-
-        let stderr = std::str::from_utf8(&output.stderr)
-            .expect("stderr is not utf8! need to handle arbitrary bytes");
-
-        if !output.status.success() {
-            bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}");
-        }
+    // MUSL has no support for LLVM in C-API
+    #[cfg(target_env = "musl")]
+    let compilers = &["cranelift", "singlepass"];
+    #[cfg(not(target_env = "musl"))]
+    let compilers = &["cranelift", "singlepass", "llvm"];
+
+    // llvm-objdump  --disassemble-all --demangle ./objects/wasmer_vm-50cb118b098c15db.wasmer_vm.60425a0a-cgu.12.rcgu.o
+    // llvm-objdump --macho --exports-trie ~/.wasmer/cache/wasmer-darwin-arm64/lib/libwasmer.dylib
+    let excluded_combinations = &[
+        ("aarch64-darwin", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported
+        ("aarch64-linux-gnu", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported
+        // https://github.com/ziglang/zig/issues/13729
+        ("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' kind Unknown
+        ("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown
+    ];
 
-        println!("stdout: {stdout}");
-        println!("stderr: {stderr}");
-
-        if !python_wasmer_path.exists() {
-            let p = std::fs::read_dir(temp_dir.path())
-                .unwrap()
-                .filter_map(|e| Some(e.ok()?.path()))
-                .collect::<Vec<_>>();
-            panic!(
-                "target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}",
-                p
-            );
+    for t in targets {
+        for c in compilers {
+            if excluded_combinations.contains(&(t, c)) {
+                continue;
+            }
+            println!("{t} target {c}");
+            let python_wasmer_path = temp_dir.path().join(format!("{t}-python"));
+
+            let mut output = Command::new(get_wasmer_path());
+
+            output.arg("create-exe");
+            output.arg(wasi_test_python_path());
+            output.arg("--target");
+            output.arg(t);
+            output.arg("-o");
+            output.arg(python_wasmer_path.clone());
+            output.arg(format!("--{c}"));
+
+            if t.contains("x86_64") && *c == "singlepass" {
+                output.arg("-m");
+                output.arg("avx");
+            }
+
+            let output = output.output()?;
+
+            let stdout = std::str::from_utf8(&output.stdout)
+                .expect("stdout is not utf8! need to handle arbitrary bytes");
+
+            let stderr = std::str::from_utf8(&output.stderr)
+                .expect("stderr is not utf8! need to handle arbitrary bytes");
+
+            if !output.status.success() {
+                bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}");
+            }
+
+            println!("stdout: {stdout}");
+            println!("stderr: {stderr}");
+
+            if !python_wasmer_path.exists() {
+                let p = std::fs::read_dir(temp_dir.path())
+                    .unwrap()
+                    .filter_map(|e| Some(e.ok()?.path()))
+                    .collect::<Vec<_>>();
+                panic!(
+                    "target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}",
+                    p
+                );
+            }
         }
     }