Fix "create-exe" for windows-x86_64 target

.github/workflows/test-sys.yaml

@@ -103,10 +103,11 @@ jobs:
           sudo apt-get update -y
           sudo apt-get install -y --allow-downgrades libstdc++6=8.4.0-1ubuntu1~18.04
           sudo apt-get install --reinstall g++-8
+          sudo apt-get install -y gcc-multilib
       - name: Set up base deps on musl
         if: matrix.build == 'linux-musl-x64'
         run: |
-            apk add build-base bash musl-dev curl make libtool libffi-dev gcc automake autoconf git openssl-dev g++
+            apk add build-base bash musl-dev curl make libtool libffi-dev gcc build-base automake autoconf git openssl-dev g++
       - name: Install Rust
         uses: dtolnay/rust-toolchain@stable
         with:

lib/cli/Cargo.toml

@@ -68,9 +68,10 @@ regex = "1.6.0"
 toml = "0.5.9"
 url = "2.3.1"
 libc = { version = "^0.2", default-features = false }
-nuke-dir = { version = "0.1.0", optional = true }
+nuke-dir = "0.1.0"
 webc = { version = "3.0.1", optional = true }
 isatty = "0.1.9"
+tar = "0.4"
 
 [build-dependencies]
 chrono = { version = "^0.4", default-features = false, features = [ "std", "clock" ] }
@@ -98,7 +99,7 @@ wast = ["wasmer-wast"]
 wasi = ["wasmer-wasi"]
 emscripten = ["wasmer-emscripten"]
 wat = ["wasmer/wat"]
-webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "nuke-dir", "webc"]
+webc_runner = ["wasi", "wasmer-wasi/webc_runner", "wasmer-wasi/webc_runner_rt_wasi", "wasmer-wasi/webc_runner_rt_emscripten", "webc"]
 compiler = [
     "wasmer-compiler/translator",
     "wasmer-compiler/compiler",

lib/cli/src/commands/create_exe.rs

@@ -8,6 +8,7 @@ use clap::Parser;
 use serde::{Deserialize, Serialize};
 #[cfg(feature = "http")]
 use std::collections::BTreeMap;
+use std::default;
 use std::env;
 use std::fs;
 use std::fs::File;
@@ -258,6 +259,7 @@ impl CreateExe {
                 )?;
             }
         } else {
+            println!("compiling non-pirita with object format {object_format:?}");
             match object_format {
                 ObjectFormat::Serialized => {
                     let module = Module::from_file(&store, &wasm_module_path)
@@ -562,26 +564,71 @@ impl CreateExe {
             include_dir.pop();
             include_dir.push("include");
 
+            let mut default_include_path = None;
+
+            #[cfg(target_os = "macos")]
+            {
+                if let Ok(output) = Command::new("xcrun").arg("--show-sdk-path").output() {
+                    default_include_path =
+                        Some(String::from_utf8_lossy(&output.stdout).to_string());
+                }
+            }
+
+            #[cfg(target_os = "linux")]
+            {
+                default_include_path = Some("/usr/include".to_string());
+            }
+
+            #[cfg(target_os = "windows")]
+            {
+                // TODO: discover include path?
+                default_include_path = Some(
+                    "C:\\Program Files (x86)\\Windows Kits\\10\\Include\\10.0.17134.0\\ucrt"
+                        .to_string(),
+                );
+            }
+
+            let default_include_path = match default_include_path {
+                Some(s) => Some(s),
+                None => {
+                    if zig_triple.contains("windows") {
+                        return Err(anyhow::anyhow!(
+                            "no default include path for target windows-x64"
+                        ));
+                    } else {
+                        None
+                    }
+                }
+            };
+
             let mut cmd = Command::new(zig_binary_path);
-            let mut cmd_mut: &mut Command = cmd
-                .arg("cc")
-                .arg("-target")
-                .arg(&zig_triple)
-                .arg(&format!("-L{}", libwasmer_path.display()))
-                .arg(&format!("-l:{}", lib_filename))
-                .arg(&format!("-I{}", include_dir.display()))
-                .arg(&format!("-I{}", header_code_path.display()));
+            cmd.arg("cc");
+            cmd.arg("-target");
+            cmd.arg(&zig_triple);
+            cmd.arg(&format!("-L{}", libwasmer_path.display()));
+            cmd.arg(&format!("-l:{}", lib_filename));
+
+            if let Some(default_include) = default_include_path {
+                cmd.arg(format!("-I{default_include}"));
+            }
+
+            cmd.arg(&format!("-I{}", include_dir.display()));
+            cmd.arg(&format!("-I{}", header_code_path.display()));
+
             if !zig_triple.contains("windows") {
-                cmd_mut = cmd_mut.arg("-lunwind");
+                cmd.arg("-lunwind");
             }
-            cmd_mut = cmd_mut.arg(&object_path).arg(&c_src_path);
+
+            cmd.arg(&object_path);
+            cmd.arg(&c_src_path);
 
             if let Some(volume_obj) = pirita_volume_path.as_ref() {
-                cmd_mut = cmd_mut.arg(volume_obj.clone());
+                cmd.arg(volume_obj.clone());
             }
 
-            cmd_mut
-                .arg("-o")
+            println!("cmd (zig cc): {:?}", cmd);
+
+            cmd.arg("-o")
                 .arg(&output_path)
                 .output()
                 .context("Could not execute `zig`")?
@@ -1357,6 +1404,8 @@ mod http_fetch {
     };
     use std::convert::TryFrom;
 
+    use crate::commands::cache;
+
     pub fn get_latest_release() -> Result<serde_json::Value> {
         let mut writer = Vec::new();
         let uri = Uri::try_from("https://api.github.com/repos/wasmerio/wasmer/releases").unwrap();
@@ -1398,6 +1447,7 @@ mod http_fetch {
         mut release: serde_json::Value,
         target_triple: wasmer::Triple,
     ) -> Result<std::path::PathBuf> {
+        use std::path::Path;
         let check_arch = |name: &str| -> bool {
             match target_triple.architecture {
                 wasmer_types::Architecture::X86_64 => {
@@ -1437,18 +1487,28 @@ mod http_fetch {
         };
 
         if let Ok(mut cache_path) = super::get_libwasmer_cache_path() {
-            match std::fs::read_dir(&cache_path).and_then(|r| {
+            println!("cache path: {}", cache_path.display());
+            let paths = std::fs::read_dir(&cache_path).and_then(|r| {
                 r.map(|res| res.map(|e| e.path()))
                     .collect::<Result<Vec<_>, std::io::Error>>()
-            }) {
+            });
+
+            println!("paths: {:#?}", paths);
+            match paths {
                 Ok(mut entries) => {
                     entries.retain(|p| p.to_str().map(|p| check_arch(p)).unwrap_or(true));
                     entries.retain(|p| p.to_str().map(|p| check_vendor(p)).unwrap_or(true));
                     entries.retain(|p| p.to_str().map(|p| check_os(p)).unwrap_or(true));
                     entries.retain(|p| p.to_str().map(|p| check_env(p)).unwrap_or(true));
-                    if entries.len() == 1 {
+                    if entries.len() > 0 {
                         cache_path.push(&entries[0]);
+                        println!("testing for cache path 2: {}", cache_path.display());
                         if cache_path.exists() {
+                            let target = Path::new(
+                                &format!("{}", cache_path.display()).replace(".tar.gz", ""),
+                            )
+                            .to_path_buf();
+                            super::untar(cache_path.clone(), target)?;
                             eprintln!(
                                 "Using cached tarball to cache path `{}`.",
                                 cache_path.display()
@@ -1504,39 +1564,61 @@ mod http_fetch {
                     .last()
                     .unwrap_or("output")
                     .to_string();
-                let mut file = std::fs::File::create(&filename)?;
-                println!("Downloading {} to {}", browser_download_url, &filename);
-                let download_thread: std::thread::JoinHandle<Result<Response, anyhow::Error>> =
-                    std::thread::spawn(move || {
-                        let uri = Uri::try_from(browser_download_url.as_str())?;
-                        let mut response = Request::new(&uri)
-                            .header("User-Agent", "wasmer")
-                            .send(&mut file)
-                            .map_err(anyhow::Error::new)
-                            .context("Could not lookup wasmer artifact on Github.")?;
-                        if response.status_code() == StatusCode::new(302) {
-                            let redirect_uri =
-                                Uri::try_from(response.headers().get("Location").unwrap().as_str())
-                                    .unwrap();
-                            response = Request::new(&redirect_uri)
-                                .header("User-Agent", "wasmer")
-                                .send(&mut file)
-                                .map_err(anyhow::Error::new)
-                                .context("Could not lookup wasmer artifact on Github.")?;
-                        }
-                        Ok(response)
-                    });
+                let download_tempdir = tempdir::TempDir::new("wasmer-download")?;
+                let download_path = download_tempdir.path().to_path_buf().join(&filename);
+                let mut file = std::fs::File::create(&download_path)?;
+                println!(
+                    "Downloading {} to {}",
+                    browser_download_url,
+                    download_path.display()
+                );
+                let uri = Uri::try_from(browser_download_url.as_str())?;
+                let mut response = Request::new(&uri)
+                    .header("User-Agent", "wasmer")
+                    .send(&mut file)
+                    .map_err(anyhow::Error::new)
+                    .context("Could not lookup wasmer artifact on Github.")?;
+                if response.status_code() == StatusCode::new(302) {
+                    let redirect_uri =
+                        Uri::try_from(response.headers().get("Location").unwrap().as_str())
+                            .unwrap();
+                    response = Request::new(&redirect_uri)
+                        .header("User-Agent", "wasmer")
+                        .send(&mut file)
+                        .map_err(anyhow::Error::new)
+                        .context("Could not lookup wasmer artifact on Github.")?;
+                }
                 match super::get_libwasmer_cache_path() {
                     Ok(mut cache_path) => {
                         cache_path.push(&filename);
                         if !cache_path.exists() {
-                            if let Err(err) = std::fs::copy(&filename, &cache_path) {
+                            eprintln!(
+                                "copying from {} to {}",
+                                download_path.display(),
+                                cache_path.display()
+                            );
+                            if let Err(err) = std::fs::copy(&download_path, &cache_path) {
                                 eprintln!(
                                     "Could not store tarball to cache path `{}`: {}",
                                     cache_path.display(),
                                     err
                                 );
                             } else {
+                                eprintln!("copying to /Development");
+                                let _ = std::fs::File::create(
+                                    "/Users/fs/Development/wasmer-windows.tar.gz",
+                                );
+                                eprintln!("source exists: {}", cache_path.exists());
+                                eprintln!(
+                                    "target exists: {}",
+                                    Path::new("/Users/fs/Development/wasmer-windows.tar.gz")
+                                        .exists()
+                                );
+                                std::fs::copy(
+                                    &cache_path,
+                                    "/Users/fs/Development/wasmer-windows.tar.gz",
+                                )
+                                .unwrap();
                                 eprintln!(
                                     "Cached tarball to cache path `{}`.",
                                     cache_path.display()
@@ -1551,12 +1633,14 @@ mod http_fetch {
                         );
                     }
                 }
-                let _response = download_thread
-                    .join()
-                    .expect("Could not join downloading thread");
                 match super::get_libwasmer_cache_path() {
                     Ok(mut cache_path) => {
                         cache_path.push(&filename);
+                        println!(
+                            "testing for cache path {}: {:?}",
+                            cache_path.display(),
+                            cache_path.exists()
+                        );
                         if !cache_path.exists() {
                             if let Err(err) = std::fs::copy(&filename, &cache_path) {
                                 eprintln!(
@@ -1569,6 +1653,9 @@ mod http_fetch {
                                     "Cached tarball to cache path `{}`.",
                                     cache_path.display()
                                 );
+                                eprintln!("copying to /Development");
+                                std::fs::copy(&cache_path, "~/Development/wasmer-windows.tar.gz")
+                                    .unwrap();
                             }
                         }
                     }
@@ -1587,29 +1674,34 @@ mod http_fetch {
 }
 
 fn untar(tarball: std::path::PathBuf, target: std::path::PathBuf) -> Result<Vec<String>> {
-    let files = std::process::Command::new("tar")
-        .arg("-tf")
-        .arg(&tarball)
-        .output()
-        .expect("failed to execute process")
-        .stdout;
+    println!("untar: {} -> {}", tarball.display(), target.display());
+
+    let files = {
+        let file = File::open(&tarball)?;
+        let mut a = tar::Archive::new(file);
+        a.entries_with_seek()?
+            .filter_map(|entry| Some(entry.ok()?.path().ok()?.to_path_buf()))
+            .map(|p| format!("{}", p.display()))
+            .filter(|p| !p.ends_with('/'))
+            .collect::<Vec<_>>()
+    };
 
-    let files_s = String::from_utf8(files)?;
+    let _ = nuke_dir::nuke_dir(&target);
+    let _ = std::fs::remove_dir(&target);
 
-    let files = files_s
-        .lines()
-        .filter(|p| !p.ends_with('/'))
-        .map(|s| s.to_string())
-        .collect::<Vec<String>>();
+    if files.is_empty() {
+        let _ = std::fs::remove_file(&tarball);
+        return Ok(files);
+    }
 
     let _ = std::fs::create_dir_all(&target);
-    let _output = std::process::Command::new("tar")
-        .arg("-xf")
-        .arg(&tarball)
-        .arg("-C")
-        .arg(&target)
-        .output()
-        .expect("failed to execute process");
+
+    println!("files ok!");
+
+    let file = File::open(&tarball)?;
+    let mut a = tar::Archive::new(file);
+    a.unpack(target)?;
+    println!("untar ok! {:#?}", files);
     Ok(files)
 }
 

tests/integration/cli/tests/run.rs

@@ -21,6 +21,33 @@ fn test_no_start_wat_path() -> String {
     format!("{}/{}", ASSET_PATH, "no_start.wat")
 }
 
+#[test]
+fn test_cross_compile_python_windows() -> anyhow::Result<()> {
+    let temp_dir = tempfile::TempDir::new()?;
+    let python_wasmer_path = temp_dir.path().join("python.exe");
+
+    let output = Command::new(get_wasmer_path())
+        .arg("create-exe")
+        .arg(wasi_test_python_path())
+        .arg("--target")
+        .arg("x86_64-windows-gnu")
+        .arg("-o")
+        .arg(python_wasmer_path)
+        .output()?;
+
+    if !output.status.success() {
+        bail!(
+            "linking failed with: stdout: {}\n\nstderr: {}",
+            std::str::from_utf8(&output.stdout)
+                .expect("stdout is not utf8! need to handle arbitrary bytes"),
+            std::str::from_utf8(&output.stderr)
+                .expect("stderr is not utf8! need to handle arbitrary bytes")
+        );
+    }
+
+    Ok(())
+}
+
 #[test]
 fn run_wasi_works() -> anyhow::Result<()> {
     let output = Command::new(get_wasmer_path())