Skip to content

Commit

Permalink
Merge #3395
Browse files Browse the repository at this point in the history 
3395: Fix create-exe to be able to cross-compile on Windows r=fschutt a=fschutt

Fixes #3374.

Co-authored-by: Felix Schütt <[email protected]>
Co-authored-by: Felix Schütt <[email protected]>
  • Loading branch information
@bors @fschutt
3 people authored Dec 2, 2022
2 parents ecde2aa + cdd71d3 commit b3ea89b
Show file tree
Hide file tree
Showing 4 changed files with 160 additions and 97 deletions.
19 changes: 5 additions & 14 deletions lib/cli/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,11 +55,11 @@ fern = { version = "0.6", features = ["colored"], optional = true }
log = { version = "0.4", optional = true }
tempfile = "3"
tempdir = "0.3.7"
http_req = { version="^0.8", default-features = false, features = ["rust-tls"], optional = true }
reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json", "multipart"], optional = true }
serde = { version = "1.0.147", features = ["derive"], optional = true }
dirs = { version = "4.0", optional = true }
serde_json = { version = "1.0", optional = true }
http_req = { version="^0.8", default-features = false, features = ["rust-tls"] }
reqwest = { version = "^0.11", default-features = false, features = ["rustls-tls", "json", "multipart"] }
serde = { version = "1.0.147", features = ["derive"] }
dirs = { version = "4.0" }
serde_json = { version = "1.0" }
target-lexicon = { version = "0.12", features = ["std"] }
prettytable-rs = "0.9.0"
wapm-toml = "0.2.0"
Expand All @@ -86,7 +86,6 @@ unix_mode = "0.1.3"
default = [
"wat",
"wast",
"http",
"cache",
"wasi",
"emscripten",
Expand Down Expand Up @@ -158,14 +157,6 @@ enable-serde = [
"wasmer-wasi/enable-serde",
]

http = [
"http_req",
"reqwest",
"dirs",
"serde_json",
"serde",
]

[target.'cfg(target_os = "windows")'.dependencies]
colored = "2.0.0"

Expand Down
138 changes: 91 additions & 47 deletions lib/cli/src/commands/create_exe.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,52 +402,34 @@ impl CreateExe {
let library = if let Some(v) = cross_subc.library_path.clone() {
v.canonicalize().unwrap_or(v)
} else {
{
let libwasmer_path = "lib/libwasmer.a";
let tarball_dir;
let filename = if let Some(local_tarball) = cross_subc.tarball.as_ref() {
let target_file_path = local_tarball
.parent()
.and_then(|parent| Some(parent.join(local_tarball.file_stem()?)))
.unwrap_or_else(|| local_tarball.clone());

let target_file_path = target_file_path
.parent()
.and_then(|parent| Some(parent.join(target_file_path.file_stem()?)))
.unwrap_or_else(|| target_file_path.clone());

let _ = std::fs::create_dir_all(&target_file_path);
let files = untar(local_tarball.clone(), target_file_path.clone())?;
tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path);
files.iter().find(|f| f.contains(libwasmer_path)).cloned().ok_or_else(|| {
anyhow!("Could not find libwasmer for {} target in the provided tarball path (files = {files:#?}, libwasmer_path = {libwasmer_path:?})", target)})?
let (filename, tarball_dir) =
if let Some(local_tarball) = cross_subc.tarball.as_ref() {
Self::find_filename(local_tarball, &target)
} else {
#[cfg(feature = "http")]
{
// check if the tarball for the target already exists locally
let local_tarball = std::fs::read_dir(get_libwasmer_cache_path()?)?
.filter_map(|e| e.ok())
.filter_map(|e| {
let path = format!("{}", e.path().display());
if path.ends_with(".tar.gz") {
Some(e.path())
} else {
None
}
})
.filter_map(|p| Self::filter_tarballs(&p, &target))
.next();

if let Some(local_tarball) = local_tarball.as_ref() {
Self::find_filename(local_tarball, &target)
} else {
let release = http_fetch::get_latest_release()?;
let tarball = http_fetch::download_release(release, target.clone())?;
let target_file_path = tarball
.parent()
.and_then(|parent| Some(parent.join(tarball.file_stem()?)))
.unwrap_or_else(|| tarball.clone());

let target_file_path = target_file_path
.parent()
.and_then(|parent| Some(parent.join(target_file_path.file_stem()?)))
.unwrap_or_else(|| target_file_path.clone());

tarball_dir = target_file_path
.canonicalize()
.unwrap_or_else(|_| target_file_path.clone());
let files = untar(tarball, target_file_path)?;
files.into_iter().find(|f| f.contains(libwasmer_path)).ok_or_else(|| {
anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag.", target)})?
Self::find_filename(&tarball, &target)
}
#[cfg(not(feature = "http"))]
return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag."));
};
tarball_dir.join(&filename)
}
}?;

tarball_dir.join(&filename)
};
let ccs = CrossCompileSetup {
target,
Expand All @@ -460,6 +442,72 @@ impl CreateExe {
}
}

fn find_filename(
local_tarball: &Path,
target: &Triple,
) -> Result<(String, PathBuf), anyhow::Error> {
let target_file_path = local_tarball
.parent()
.and_then(|parent| Some(parent.join(local_tarball.file_stem()?)))
.unwrap_or_else(|| local_tarball.to_path_buf());

let target_file_path = target_file_path
.parent()
.and_then(|parent| Some(parent.join(target_file_path.file_stem()?)))
.unwrap_or_else(|| target_file_path.clone());

std::fs::create_dir_all(&target_file_path)
.map_err(|e| anyhow::anyhow!("{e}"))
.context(anyhow::anyhow!("{}", target_file_path.display()))?;
let files = untar(local_tarball.to_path_buf(), target_file_path.clone())?;
let tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path);

let file = files
.iter()
.find(|f| f.ends_with("libwasmer.a")).cloned()
.ok_or_else(|| {
anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)
})?;

Ok((file, tarball_dir))
}

fn filter_tarballs(p: &Path, target: &Triple) -> Option<PathBuf> {
if let Architecture::Aarch64(_) = target.architecture {
if !p.file_name()?.to_str()?.contains("aarch64") {
return None;
}
}

if let Architecture::X86_64 = target.architecture {
if !p.file_name()?.to_str()?.contains("x86_64") {
return None;
}
}

if let OperatingSystem::Windows = target.operating_system {
if !p.file_name()?.to_str()?.contains("windows") {
return None;
}
}

if let OperatingSystem::Darwin = target.operating_system {
if !(p.file_name()?.to_str()?.contains("apple")
|| p.file_name()?.to_str()?.contains("darwin"))
{
return None;
}
}

if let OperatingSystem::Linux = target.operating_system {
if !p.file_name()?.to_str()?.contains("linux") {
return None;
}
}

Some(p.to_path_buf())
}

fn compile_c(
&self,
wasm_object_path: PathBuf,
Expand Down Expand Up @@ -573,9 +621,6 @@ impl CreateExe {
}
cmd.arg("-lunwind");
cmd.arg("-OReleaseSafe");
cmd.arg("-fstrip");
cmd.arg("-dead_strip");
cmd.arg("-dead_strip_dylibs");
cmd.arg("-fno-compiler-rt");
cmd.arg(&format!("-femit-bin={}", output_path.display()));

Expand Down Expand Up @@ -1329,7 +1374,6 @@ impl LinkCode {
}
}

#[cfg(feature = "http")]
mod http_fetch {
use anyhow::{anyhow, Context, Result};
use http_req::{request::Request, response::StatusCode, uri::Uri};
Expand Down Expand Up @@ -1379,7 +1423,7 @@ mod http_fetch {
}

Err(anyhow!(
"Could not get expected Github API response.\n\nReason: response format is not recognized:\n{:#?}", ""
"Could not get expected Github API response.\n\nReason: response format is not recognized:\n{response:#?}",
))
}

Expand Down
2 changes: 1 addition & 1 deletion lib/compiler-singlepass/src/machine_x64.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ impl AssemblerX64 {
Some(CpuFeature::SSE42)
} else {
return Err(CompileError::UnsupportedTarget(
"x86_64 without AVX or SSE 4.2".to_string(),
"x86_64 without AVX or SSE 4.2, use -m avx to enable".to_string(),
));
}
};
Expand Down
98 changes: 63 additions & 35 deletions tests/integration/cli/tests/run.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ fn test_no_start_wat_path() -> PathBuf {
Path::new(ASSET_PATH).join("no_start.wat")
}

#[cfg(any(target_os = "linux", target_os = "macos"))]
#[test]
fn test_cross_compile_python_windows() -> anyhow::Result<()> {
let temp_dir = tempfile::TempDir::new()?;
Expand All @@ -34,41 +33,70 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> {
"x86_64-windows-gnu",
];

for t in targets {
let python_wasmer_path = temp_dir.path().join(format!("{t}-python"));

let mut output = Command::new(get_wasmer_path());

output.arg("create-exe");
output.arg(wasi_test_python_path());
output.arg("--target");
output.arg(t);
output.arg("-o");
output.arg(python_wasmer_path.clone());
let output = output.output()?;

let stdout = std::str::from_utf8(&output.stdout)
.expect("stdout is not utf8! need to handle arbitrary bytes");

let stderr = std::str::from_utf8(&output.stderr)
.expect("stderr is not utf8! need to handle arbitrary bytes");

if !output.status.success() {
bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}");
}
// MUSL has no support for LLVM in C-API
#[cfg(target_env = "musl")]
let compilers = &["cranelift", "singlepass"];
#[cfg(not(target_env = "musl"))]
let compilers = &["cranelift", "singlepass", "llvm"];

// llvm-objdump --disassemble-all --demangle ./objects/wasmer_vm-50cb118b098c15db.wasmer_vm.60425a0a-cgu.12.rcgu.o
// llvm-objdump --macho --exports-trie ~/.wasmer/cache/wasmer-darwin-arm64/lib/libwasmer.dylib
let excluded_combinations = &[
("aarch64-darwin", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported
("aarch64-linux-gnu", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported
// https://github.com/ziglang/zig/issues/13729
("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' kind Unknown
("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown
];

println!("stdout: {stdout}");
println!("stderr: {stderr}");

if !python_wasmer_path.exists() {
let p = std::fs::read_dir(temp_dir.path())
.unwrap()
.filter_map(|e| Some(e.ok()?.path()))
.collect::<Vec<_>>();
panic!(
"target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}",
p
);
for t in targets {
for c in compilers {
if excluded_combinations.contains(&(t, c)) {
continue;
}
println!("{t} target {c}");
let python_wasmer_path = temp_dir.path().join(format!("{t}-python"));

let mut output = Command::new(get_wasmer_path());

output.arg("create-exe");
output.arg(wasi_test_python_path());
output.arg("--target");
output.arg(t);
output.arg("-o");
output.arg(python_wasmer_path.clone());
output.arg(format!("--{c}"));

if t.contains("x86_64") && *c == "singlepass" {
output.arg("-m");
output.arg("avx");
}

let output = output.output()?;

let stdout = std::str::from_utf8(&output.stdout)
.expect("stdout is not utf8! need to handle arbitrary bytes");

let stderr = std::str::from_utf8(&output.stderr)
.expect("stderr is not utf8! need to handle arbitrary bytes");

if !output.status.success() {
bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}");
}

println!("stdout: {stdout}");
println!("stderr: {stderr}");

if !python_wasmer_path.exists() {
let p = std::fs::read_dir(temp_dir.path())
.unwrap()
.filter_map(|e| Some(e.ok()?.path()))
.collect::<Vec<_>>();
panic!(
"target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}",
p
);
}
}
}

Expand Down

0 comments on commit b3ea89b

Please sign in to comment.