From 1346252a1842b925a1e9f24ec8e0212688f51851 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 30 Nov 2022 18:15:46 +0100 Subject: [PATCH 01/14] Fix error in searching for file path on Windows --- lib/cli/src/commands/create_exe.rs | 9 ++++----- tests/integration/cli/tests/run.rs | 1 - 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index c13a38d1185..34ccb81390e 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -403,7 +403,6 @@ impl CreateExe { v.canonicalize().unwrap_or(v) } else { { - let libwasmer_path = "lib/libwasmer.a"; let tarball_dir; let filename = if let Some(local_tarball) = cross_subc.tarball.as_ref() { let target_file_path = local_tarball @@ -419,8 +418,8 @@ impl CreateExe { let _ = std::fs::create_dir_all(&target_file_path); let files = untar(local_tarball.clone(), target_file_path.clone())?; tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path); - files.iter().find(|f| f.contains(libwasmer_path)).cloned().ok_or_else(|| { - anyhow!("Could not find libwasmer for {} target in the provided tarball path (files = {files:#?}, libwasmer_path = {libwasmer_path:?})", target)})? + files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { + anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)})? } else { #[cfg(feature = "http")] { @@ -440,8 +439,8 @@ impl CreateExe { .canonicalize() .unwrap_or_else(|_| target_file_path.clone()); let files = untar(tarball, target_file_path)?; - files.into_iter().find(|f| f.contains(libwasmer_path)).ok_or_else(|| { - anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag.", target)})? + files.into_iter().find(|f| f.ends_with("libwasmer.a")).ok_or_else(|| { + anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag. (files = {files:#?}", target)})? } #[cfg(not(feature = "http"))] return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag.")); diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 8146e55789b..b1c9bf2ffe8 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -21,7 +21,6 @@ fn test_no_start_wat_path() -> PathBuf { Path::new(ASSET_PATH).join("no_start.wat") } -#[cfg(any(target_os = "linux", target_os = "macos"))] #[test] fn test_cross_compile_python_windows() -> anyhow::Result<()> { let temp_dir = tempfile::TempDir::new()?; From 80b7cb99f628ed30d8692e9240b2593beefa0e05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 30 Nov 2022 18:24:23 +0100 Subject: [PATCH 02/14] Adjust unit test to test for all configurations for create-exe --- tests/integration/cli/tests/run.rs | 78 +++++++++++++++++------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index b1c9bf2ffe8..4ae906a1986 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -33,41 +33,51 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { "x86_64-windows-gnu", ]; - for t in targets { - let python_wasmer_path = temp_dir.path().join(format!("{t}-python")); - - let mut output = Command::new(get_wasmer_path()); - - output.arg("create-exe"); - output.arg(wasi_test_python_path()); - output.arg("--target"); - output.arg(t); - output.arg("-o"); - output.arg(python_wasmer_path.clone()); - let output = output.output()?; - - let stdout = std::str::from_utf8(&output.stdout) - .expect("stdout is not utf8! need to handle arbitrary bytes"); - - let stderr = std::str::from_utf8(&output.stderr) - .expect("stderr is not utf8! need to handle arbitrary bytes"); - - if !output.status.success() { - bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}"); - } + let compilers = &[ + "cranelift", + "singlepass", + "llvm", + ]; - println!("stdout: {stdout}"); - println!("stderr: {stderr}"); - - if !python_wasmer_path.exists() { - let p = std::fs::read_dir(temp_dir.path()) - .unwrap() - .filter_map(|e| Some(e.ok()?.path())) - .collect::>(); - panic!( - "target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}", - p - ); + for t in targets { + for c in compilers { + println!("{t} target {c}"); + let python_wasmer_path = temp_dir.path().join(format!("{t}-python")); + + let mut output = Command::new(get_wasmer_path()); + + output.arg("create-exe"); + output.arg(wasi_test_python_path()); + output.arg("--target"); + output.arg(t); + output.arg("-o"); + output.arg(python_wasmer_path.clone()); + output.arg(format!("--{c}")); + let output = output.output()?; + + let stdout = std::str::from_utf8(&output.stdout) + .expect("stdout is not utf8! need to handle arbitrary bytes"); + + let stderr = std::str::from_utf8(&output.stderr) + .expect("stderr is not utf8! need to handle arbitrary bytes"); + + if !output.status.success() { + bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}"); + } + + println!("stdout: {stdout}"); + println!("stderr: {stderr}"); + + if !python_wasmer_path.exists() { + let p = std::fs::read_dir(temp_dir.path()) + .unwrap() + .filter_map(|e| Some(e.ok()?.path())) + .collect::>(); + panic!( + "target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}", + p + ); + } } } From 2c254a55ccbf8c9fc06d2e08ccb7487e854cc0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 30 Nov 2022 18:37:28 +0100 Subject: [PATCH 03/14] Fix compilation issue in create-exe function --- lib/cli/src/commands/create_exe.rs | 2 +- tests/integration/cli/tests/run.rs | 18 +++++++----------- 2 files changed, 8 insertions(+), 12 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 34ccb81390e..87623e8ba97 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -439,7 +439,7 @@ impl CreateExe { .canonicalize() .unwrap_or_else(|_| target_file_path.clone()); let files = untar(tarball, target_file_path)?; - files.into_iter().find(|f| f.ends_with("libwasmer.a")).ok_or_else(|| { + files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag. (files = {files:#?}", target)})? } #[cfg(not(feature = "http"))] diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 4ae906a1986..98435c036d4 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -33,11 +33,7 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { "x86_64-windows-gnu", ]; - let compilers = &[ - "cranelift", - "singlepass", - "llvm", - ]; + let compilers = &["cranelift", "singlepass", "llvm"]; for t in targets { for c in compilers { @@ -45,7 +41,7 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { let python_wasmer_path = temp_dir.path().join(format!("{t}-python")); let mut output = Command::new(get_wasmer_path()); - + output.arg("create-exe"); output.arg(wasi_test_python_path()); output.arg("--target"); @@ -54,20 +50,20 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { output.arg(python_wasmer_path.clone()); output.arg(format!("--{c}")); let output = output.output()?; - + let stdout = std::str::from_utf8(&output.stdout) .expect("stdout is not utf8! need to handle arbitrary bytes"); - + let stderr = std::str::from_utf8(&output.stderr) .expect("stderr is not utf8! need to handle arbitrary bytes"); - + if !output.status.success() { bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}"); } - + println!("stdout: {stdout}"); println!("stderr: {stderr}"); - + if !python_wasmer_path.exists() { let p = std::fs::read_dir(temp_dir.path()) .unwrap() From 21c9098faa0ec1b5c9470190b7187c6d1d7851bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 30 Nov 2022 18:48:29 +0100 Subject: [PATCH 04/14] Make test pass again by excluding certain combinations --- tests/integration/cli/tests/run.rs | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 98435c036d4..23a02848cf5 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -35,8 +35,23 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { let compilers = &["cranelift", "singlepass", "llvm"]; + let excluded_combinations = &[ + ("x86_64-darwin", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") + ("x86_64-linux-gnu", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") + ("x86_64-windows-gnu", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") + + ("aarch64-darwin", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported + ("aarch64-linux-gnu", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported + + ("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' + ("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown + ]; + for t in targets { for c in compilers { + if excluded_combinations.contains(&(t, c)) { + continue; + } println!("{t} target {c}"); let python_wasmer_path = temp_dir.path().join(format!("{t}-python")); From 0822773c0993bc4f5efe595bead00068afb8b188 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 30 Nov 2022 19:10:12 +0100 Subject: [PATCH 05/14] Don't ping GitHub if local tarball can be found --- lib/cli/src/commands/create_exe.rs | 100 ++++++++++++++++++++++++----- tests/integration/cli/tests/run.rs | 11 +++- 2 files changed, 93 insertions(+), 18 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 87623e8ba97..562a6b558d9 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -421,29 +421,99 @@ impl CreateExe { files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)})? } else { - #[cfg(feature = "http")] - { - let release = http_fetch::get_latest_release()?; - let tarball = http_fetch::download_release(release, target.clone())?; - let target_file_path = tarball + // check if the tarball for the target already exists locally + let local_tarball = std::fs::read_dir(get_libwasmer_cache_path()?)? + .filter_map(|e| e.ok()) + .filter_map(|e| { + let path = format!("{}", e.path().display()); + if path.ends_with(".tar.gz") { + Some(e.path()) + } else { + None + } + }) + .filter_map(|p| { + if let Architecture::Aarch64(_) = target.architecture { + if !p.file_name()?.to_str()?.contains("aarch64") { + return None; + } + } + + if let Architecture::X86_64 = target.architecture { + if !p.file_name()?.to_str()?.contains("x86_64") { + return None; + } + } + + if let OperatingSystem::Windows = target.operating_system { + if !p.file_name()?.to_str()?.contains("windows") { + return None; + } + } + + if let OperatingSystem::Darwin = target.operating_system { + if !(p.file_name()?.to_str()?.contains("apple") + || p.file_name()?.to_str()?.contains("darwin")) + { + return None; + } + } + + if let OperatingSystem::Linux = target.operating_system { + if !p.file_name()?.to_str()?.contains("linux") { + return None; + } + } + + Some(p) + }) + .next(); + + if let Some(local_tarball) = local_tarball.as_ref() { + let target_file_path = local_tarball .parent() - .and_then(|parent| Some(parent.join(tarball.file_stem()?))) - .unwrap_or_else(|| tarball.clone()); + .and_then(|parent| Some(parent.join(local_tarball.file_stem()?))) + .unwrap_or_else(|| local_tarball.clone()); let target_file_path = target_file_path .parent() .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) .unwrap_or_else(|| target_file_path.clone()); - tarball_dir = target_file_path - .canonicalize() - .unwrap_or_else(|_| target_file_path.clone()); - let files = untar(tarball, target_file_path)?; + let _ = std::fs::create_dir_all(&target_file_path); + let files = untar(local_tarball.clone(), target_file_path.clone())?; + tarball_dir = + target_file_path.canonicalize().unwrap_or(target_file_path); files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { - anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag. (files = {files:#?}", target)})? + anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)})? + } else { + #[cfg(feature = "http")] + { + let release = http_fetch::get_latest_release()?; + let tarball = + http_fetch::download_release(release, target.clone())?; + let target_file_path = tarball + .parent() + .and_then(|parent| Some(parent.join(tarball.file_stem()?))) + .unwrap_or_else(|| tarball.clone()); + + let target_file_path = target_file_path + .parent() + .and_then(|parent| { + Some(parent.join(target_file_path.file_stem()?)) + }) + .unwrap_or_else(|| target_file_path.clone()); + + tarball_dir = target_file_path + .canonicalize() + .unwrap_or_else(|_| target_file_path.clone()); + let files = untar(tarball, target_file_path)?; + files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { + anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag. (files = {files:#?}", target)})? + } + #[cfg(not(feature = "http"))] + return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag.")); } - #[cfg(not(feature = "http"))] - return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag.")); }; tarball_dir.join(&filename) } @@ -1378,7 +1448,7 @@ mod http_fetch { } Err(anyhow!( - "Could not get expected Github API response.\n\nReason: response format is not recognized:\n{:#?}", "" + "Could not get expected Github API response.\n\nReason: response format is not recognized:\n{response:#?}", )) } diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 23a02848cf5..c0dc3992fdd 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -39,12 +39,11 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { ("x86_64-darwin", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") ("x86_64-linux-gnu", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") ("x86_64-windows-gnu", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") - ("aarch64-darwin", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported ("aarch64-linux-gnu", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported - ("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' - ("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown + // ("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' kind Unknown + // ("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown ]; for t in targets { @@ -64,6 +63,12 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { output.arg("-o"); output.arg(python_wasmer_path.clone()); output.arg(format!("--{c}")); + + if t.contains("x86_64") && *c == "singlepass" { + output.arg("-m"); + output.arg("avx"); + } + let output = output.output()?; let stdout = std::str::from_utf8(&output.stdout) From 991ad55bdb660a15a50f249298bc5706c19fab82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 1 Dec 2022 12:33:40 +0100 Subject: [PATCH 06/14] Debug undefined wasmer_vm_raise_trap error --- lib/cli/src/commands/create_exe.rs | 17 +++++++++++++---- lib/compiler-singlepass/src/machine_x64.rs | 2 +- tests/integration/cli/tests/run.rs | 9 ++++----- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 562a6b558d9..8d046bdc2f9 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -579,6 +579,9 @@ impl CreateExe { pirita_main_atom: Option<&str>, pirita_volume_path: Option, ) -> anyhow::Result<()> { + + println!("compile zig {:?}", setup); + let tempdir = tempdir::TempDir::new("wasmer-static-compile-zig")?; let tempdir_path = tempdir.path(); let c_src_path = tempdir_path.join("wasmer_main.c"); @@ -642,9 +645,6 @@ impl CreateExe { } cmd.arg("-lunwind"); cmd.arg("-OReleaseSafe"); - cmd.arg("-fstrip"); - cmd.arg("-dead_strip"); - cmd.arg("-dead_strip_dylibs"); cmd.arg("-fno-compiler-rt"); cmd.arg(&format!("-femit-bin={}", output_path.display())); @@ -652,7 +652,16 @@ impl CreateExe { cmd.arg(o); } cmd.arg(&c_src_path); - cmd.arg(libwasmer_path.join(&lib_filename)); + if zig_triple != "x86_64-macos-none" { + cmd.arg(libwasmer_path.join(&lib_filename)); + } else { + // cmd.arg("/Users/fs/Development/wasmer/target/x86_64-apple-darwin/release/libwasmer.a"); + for file in std::fs::read_dir("/Users/fs/Development/wasmer/target/x86_64-apple-darwin/release/objects").unwrap() { + let path = file.unwrap().path().canonicalize().unwrap(); + println!("linking {}", path.display()); + cmd.arg(path); + } + } if zig_triple.contains("windows") { let mut libwasmer_parent = libwasmer_path.clone(); libwasmer_parent.pop(); diff --git a/lib/compiler-singlepass/src/machine_x64.rs b/lib/compiler-singlepass/src/machine_x64.rs index dad625c69aa..22dba235796 100644 --- a/lib/compiler-singlepass/src/machine_x64.rs +++ b/lib/compiler-singlepass/src/machine_x64.rs @@ -45,7 +45,7 @@ impl AssemblerX64 { Some(CpuFeature::SSE42) } else { return Err(CompileError::UnsupportedTarget( - "x86_64 without AVX or SSE 4.2".to_string(), + "x86_64 without AVX or SSE 4.2, use -m avx to enable".to_string(), )); } }; diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index c0dc3992fdd..63441ce4e8f 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -35,15 +35,14 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { let compilers = &["cranelift", "singlepass", "llvm"]; + // llvm-objdump --disassemble-all --demangle ./objects/wasmer_vm-50cb118b098c15db.wasmer_vm.60425a0a-cgu.12.rcgu.o + // llvm-objdump --macho --exports-trie ~/.wasmer/cache/wasmer-darwin-arm64/lib/libwasmer.dylib let excluded_combinations = &[ - ("x86_64-darwin", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") - ("x86_64-linux-gnu", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") - ("x86_64-windows-gnu", "singlepass"), // UnsupportedTarget("x86_64 without AVX or SSE 4.2") ("aarch64-darwin", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported ("aarch64-linux-gnu", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported - // ("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' kind Unknown - // ("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown + // ("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' kind Unknown + // ("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown ]; for t in targets { From 5d4817c4a1b896f6f5001ac44179b9791d64c37a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 1 Dec 2022 19:46:24 +0100 Subject: [PATCH 07/14] Undo changes, recomment out failing tests + add link to zig bug report --- lib/cli/src/commands/create_exe.rs | 15 +-------------- tests/integration/cli/tests/run.rs | 6 +++--- 2 files changed, 4 insertions(+), 17 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 8d046bdc2f9..46b2a37700d 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -579,9 +579,6 @@ impl CreateExe { pirita_main_atom: Option<&str>, pirita_volume_path: Option, ) -> anyhow::Result<()> { - - println!("compile zig {:?}", setup); - let tempdir = tempdir::TempDir::new("wasmer-static-compile-zig")?; let tempdir_path = tempdir.path(); let c_src_path = tempdir_path.join("wasmer_main.c"); @@ -644,7 +641,6 @@ impl CreateExe { cmd.arg("-lc"); } cmd.arg("-lunwind"); - cmd.arg("-OReleaseSafe"); cmd.arg("-fno-compiler-rt"); cmd.arg(&format!("-femit-bin={}", output_path.display())); @@ -652,16 +648,7 @@ impl CreateExe { cmd.arg(o); } cmd.arg(&c_src_path); - if zig_triple != "x86_64-macos-none" { - cmd.arg(libwasmer_path.join(&lib_filename)); - } else { - // cmd.arg("/Users/fs/Development/wasmer/target/x86_64-apple-darwin/release/libwasmer.a"); - for file in std::fs::read_dir("/Users/fs/Development/wasmer/target/x86_64-apple-darwin/release/objects").unwrap() { - let path = file.unwrap().path().canonicalize().unwrap(); - println!("linking {}", path.display()); - cmd.arg(path); - } - } + cmd.arg(libwasmer_path.join(&lib_filename)); if zig_triple.contains("windows") { let mut libwasmer_parent = libwasmer_path.clone(); libwasmer_parent.pop(); diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 63441ce4e8f..0ef5011f43d 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -40,9 +40,9 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { let excluded_combinations = &[ ("aarch64-darwin", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported ("aarch64-linux-gnu", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported - - // ("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' kind Unknown - // ("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown + // https://github.com/ziglang/zig/issues/13729 + ("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' kind Unknown + ("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown ]; for t in targets { From 54a0b087ce0d5722421598958601c14f6607835c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= <12084016+fschutt@users.noreply.github.com> Date: Fri, 2 Dec 2022 11:27:25 +0100 Subject: [PATCH 08/14] Update lib/cli/src/commands/create_exe.rs Co-authored-by: Michael Bryan --- lib/cli/src/commands/create_exe.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 46b2a37700d..c2739785dfa 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -419,7 +419,7 @@ impl CreateExe { let files = untar(local_tarball.clone(), target_file_path.clone())?; tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path); files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { - anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)})? + anyhow::bail!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)}); } else { // check if the tarball for the target already exists locally let local_tarball = std::fs::read_dir(get_libwasmer_cache_path()?)? From 14dc05cc08aa38bb7ce4616d2f8b1569aace9a33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 11:47:05 +0100 Subject: [PATCH 09/14] Refactor methods to filter tarballs --- lib/cli/src/commands/create_exe.rs | 173 +++++++++++++---------------- 1 file changed, 75 insertions(+), 98 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index c2739785dfa..09b938bb2ce 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -402,24 +402,9 @@ impl CreateExe { let library = if let Some(v) = cross_subc.library_path.clone() { v.canonicalize().unwrap_or(v) } else { - { - let tarball_dir; - let filename = if let Some(local_tarball) = cross_subc.tarball.as_ref() { - let target_file_path = local_tarball - .parent() - .and_then(|parent| Some(parent.join(local_tarball.file_stem()?))) - .unwrap_or_else(|| local_tarball.clone()); - - let target_file_path = target_file_path - .parent() - .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) - .unwrap_or_else(|| target_file_path.clone()); - - let _ = std::fs::create_dir_all(&target_file_path); - let files = untar(local_tarball.clone(), target_file_path.clone())?; - tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path); - files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { - anyhow::bail!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)}); + let (filename, tarball_dir) = + if let Some(local_tarball) = cross_subc.tarball.as_ref() { + Self::find_filename(local_tarball, &target) } else { // check if the tarball for the target already exists locally let local_tarball = std::fs::read_dir(get_libwasmer_cache_path()?)? @@ -432,91 +417,19 @@ impl CreateExe { None } }) - .filter_map(|p| { - if let Architecture::Aarch64(_) = target.architecture { - if !p.file_name()?.to_str()?.contains("aarch64") { - return None; - } - } - - if let Architecture::X86_64 = target.architecture { - if !p.file_name()?.to_str()?.contains("x86_64") { - return None; - } - } - - if let OperatingSystem::Windows = target.operating_system { - if !p.file_name()?.to_str()?.contains("windows") { - return None; - } - } - - if let OperatingSystem::Darwin = target.operating_system { - if !(p.file_name()?.to_str()?.contains("apple") - || p.file_name()?.to_str()?.contains("darwin")) - { - return None; - } - } - - if let OperatingSystem::Linux = target.operating_system { - if !p.file_name()?.to_str()?.contains("linux") { - return None; - } - } - - Some(p) - }) + .filter_map(|p| Self::filter_tarballs(&p, &target)) .next(); if let Some(local_tarball) = local_tarball.as_ref() { - let target_file_path = local_tarball - .parent() - .and_then(|parent| Some(parent.join(local_tarball.file_stem()?))) - .unwrap_or_else(|| local_tarball.clone()); - - let target_file_path = target_file_path - .parent() - .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) - .unwrap_or_else(|| target_file_path.clone()); - - let _ = std::fs::create_dir_all(&target_file_path); - let files = untar(local_tarball.clone(), target_file_path.clone())?; - tarball_dir = - target_file_path.canonicalize().unwrap_or(target_file_path); - files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { - anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)})? + Self::find_filename(local_tarball, &target) } else { - #[cfg(feature = "http")] - { - let release = http_fetch::get_latest_release()?; - let tarball = - http_fetch::download_release(release, target.clone())?; - let target_file_path = tarball - .parent() - .and_then(|parent| Some(parent.join(tarball.file_stem()?))) - .unwrap_or_else(|| tarball.clone()); - - let target_file_path = target_file_path - .parent() - .and_then(|parent| { - Some(parent.join(target_file_path.file_stem()?)) - }) - .unwrap_or_else(|| target_file_path.clone()); - - tarball_dir = target_file_path - .canonicalize() - .unwrap_or_else(|_| target_file_path.clone()); - let files = untar(tarball, target_file_path)?; - files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| { - anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag. (files = {files:#?}", target)})? - } - #[cfg(not(feature = "http"))] - return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag.")); + let release = http_fetch::get_latest_release()?; + let tarball = http_fetch::download_release(release, target.clone())?; + Self::find_filename(&tarball, &target) } - }; - tarball_dir.join(&filename) - } + }?; + + tarball_dir.join(&filename) }; let ccs = CrossCompileSetup { target, @@ -529,6 +442,70 @@ impl CreateExe { } } + fn find_filename( + local_tarball: &PathBuf, + target: &Triple, + ) -> Result<(String, PathBuf), anyhow::Error> { + let target_file_path = local_tarball + .parent() + .and_then(|parent| Some(parent.join(local_tarball.file_stem()?))) + .unwrap_or_else(|| local_tarball.clone()); + + let target_file_path = target_file_path + .parent() + .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) + .unwrap_or_else(|| target_file_path.clone()); + + let _ = std::fs::create_dir_all(&target_file_path); + let files = untar(local_tarball.clone(), target_file_path.clone())?; + let tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path); + + let file = files + .iter() + .find(|f| f.ends_with("libwasmer.a")).cloned() + .ok_or_else(|| { + anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target) + })?; + + Ok((file, tarball_dir)) + } + + fn filter_tarballs(p: &PathBuf, target: &Triple) -> Option { + if let Architecture::Aarch64(_) = target.architecture { + if !p.file_name()?.to_str()?.contains("aarch64") { + return None; + } + } + + if let Architecture::X86_64 = target.architecture { + if !p.file_name()?.to_str()?.contains("x86_64") { + return None; + } + } + + if let OperatingSystem::Windows = target.operating_system { + if !p.file_name()?.to_str()?.contains("windows") { + return None; + } + } + + if let OperatingSystem::Darwin = target.operating_system { + if !(p.file_name()?.to_str()?.contains("apple") + || p.file_name()?.to_str()?.contains("darwin")) + { + return None; + } + } + + if let OperatingSystem::Linux = target.operating_system { + if !p.file_name()?.to_str()?.contains("linux") { + return None; + } + } + + Some(p.clone()) + } + fn compile_c( &self, wasm_object_path: PathBuf, From 81cde2b982518d9d6fe891612fa0323d0f4c3c58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 11:51:38 +0100 Subject: [PATCH 10/14] Fix make lint --- lib/cli/src/commands/create_exe.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 09b938bb2ce..963d9e9f3e4 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -443,13 +443,13 @@ impl CreateExe { } fn find_filename( - local_tarball: &PathBuf, + local_tarball: &Path, target: &Triple, ) -> Result<(String, PathBuf), anyhow::Error> { let target_file_path = local_tarball .parent() .and_then(|parent| Some(parent.join(local_tarball.file_stem()?))) - .unwrap_or_else(|| local_tarball.clone()); + .unwrap_or_else(|| local_tarball.to_path_buf()); let target_file_path = target_file_path .parent() @@ -457,7 +457,7 @@ impl CreateExe { .unwrap_or_else(|| target_file_path.clone()); let _ = std::fs::create_dir_all(&target_file_path); - let files = untar(local_tarball.clone(), target_file_path.clone())?; + let files = untar(local_tarball.to_path_buf(), target_file_path.clone())?; let tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path); let file = files @@ -470,7 +470,7 @@ impl CreateExe { Ok((file, tarball_dir)) } - fn filter_tarballs(p: &PathBuf, target: &Triple) -> Option { + fn filter_tarballs(p: &Path, target: &Triple) -> Option { if let Architecture::Aarch64(_) = target.architecture { if !p.file_name()?.to_str()?.contains("aarch64") { return None; @@ -503,7 +503,7 @@ impl CreateExe { } } - Some(p.clone()) + Some(p.to_path_buf()) } fn compile_c( From cdbacbb67be98cf57a7a3e3b138a7b955e12bc8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 11:56:28 +0100 Subject: [PATCH 11/14] Remove --feature http because wasmer_registry requires HTTP deps --- lib/cli/Cargo.toml | 19 +++++-------------- lib/cli/src/commands/create_exe.rs | 1 - 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/lib/cli/Cargo.toml b/lib/cli/Cargo.toml index 95d962bcf3d..f25e1485d30 100644 --- a/lib/cli/Cargo.toml +++ b/lib/cli/Cargo.toml @@ -55,11 +55,11 @@ fern = { version = "0.6", features = ["colored"], optional = true } log = { version = "0.4", optional = true } tempfile = "3" tempdir = "0.3.7" -http_req = { version="^0.8", default-features = false, features = ["rust-tls"], optional = true } -reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json", "multipart"], optional = true } -serde = { version = "1.0.147", features = ["derive"], optional = true } -dirs = { version = "4.0", optional = true } -serde_json = { version = "1.0", optional = true } +http_req = { version="^0.8", default-features = false, features = ["rust-tls"] } +reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json", "multipart"] } +serde = { version = "1.0.147", features = ["derive"] } +dirs = { version = "4.0" } +serde_json = { version = "1.0" } target-lexicon = { version = "0.12", features = ["std"] } prettytable-rs = "0.9.0" wapm-toml = "0.2.0" @@ -86,7 +86,6 @@ unix_mode = "0.1.3" default = [ "wat", "wast", - "http", "cache", "wasi", "emscripten", @@ -158,14 +157,6 @@ enable-serde = [ "wasmer-wasi/enable-serde", ] -http = [ - "http_req", - "reqwest", - "dirs", - "serde_json", - "serde", -] - [target.'cfg(target_os = "windows")'.dependencies] colored = "2.0.0" diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 963d9e9f3e4..4bf3061228c 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -1371,7 +1371,6 @@ impl LinkCode { } } -#[cfg(feature = "http")] mod http_fetch { use anyhow::{anyhow, Context, Result}; use http_req::{request::Request, response::StatusCode, uri::Uri}; From 2e419eba94d187985dc6f7a1840cd83ae7f16b01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 12:38:57 +0100 Subject: [PATCH 12/14] Disable LLVM testing on linux-musl due to missing C-API support --- tests/integration/cli/tests/run.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 0ef5011f43d..619a6d0a72f 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -33,6 +33,10 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { "x86_64-windows-gnu", ]; + // MUSL has no support for LLVM in C-API + #[cfg(target_env = "musl")] + let compilers = &["cranelift", "singlepass"]; + #[cfg(not(target_env = "musl"))] let compilers = &["cranelift", "singlepass", "llvm"]; // llvm-objdump --disassemble-all --demangle ./objects/wasmer_vm-50cb118b098c15db.wasmer_vm.60425a0a-cgu.12.rcgu.o From 2c88eb83a820b772b6a9a7bce7898d50f8c88aef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 14:51:22 +0100 Subject: [PATCH 13/14] Do not ignore error on fs::create_dir_all --- lib/cli/src/commands/create_exe.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 4bf3061228c..83aa52b300c 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -456,7 +456,9 @@ impl CreateExe { .and_then(|parent| Some(parent.join(target_file_path.file_stem()?))) .unwrap_or_else(|| target_file_path.clone()); - let _ = std::fs::create_dir_all(&target_file_path); + std::fs::create_dir_all(&target_file_path) + .map_err(|e| anyhow::anyhow!("{e}")) + .context(anyhow::anyhow!("{}", target_file_path.display()))?; let files = untar(local_tarball.to_path_buf(), target_file_path.clone())?; let tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path); From cdd71d3da3352543f5c0defedbd31696eb56a042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Fri, 2 Dec 2022 14:52:03 +0100 Subject: [PATCH 14/14] Add -OReleaseSafe back --- lib/cli/src/commands/create_exe.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/cli/src/commands/create_exe.rs b/lib/cli/src/commands/create_exe.rs index 83aa52b300c..c8f326ba6cf 100644 --- a/lib/cli/src/commands/create_exe.rs +++ b/lib/cli/src/commands/create_exe.rs @@ -620,6 +620,7 @@ impl CreateExe { cmd.arg("-lc"); } cmd.arg("-lunwind"); + cmd.arg("-OReleaseSafe"); cmd.arg("-fno-compiler-rt"); cmd.arg(&format!("-femit-bin={}", output_path.display()));