Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix create-exe to be able to cross-compile on Windows #3395

Merged
merged 15 commits into from
Dec 2, 2022
Merged

Fix create-exe to be able to cross-compile on Windows #3395

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
1346252
Fix error in searching for file path on Windows
fschutt Nov 30, 2022
80b7cb9
Adjust unit test to test for all configurations for create-exe
fschutt Nov 30, 2022
2c254a5
Fix compilation issue in create-exe function
fschutt Nov 30, 2022
21c9098
Make test pass again by excluding certain combinations
fschutt Nov 30, 2022
0822773
Don't ping GitHub if local tarball can be found
fschutt Nov 30, 2022
991ad55
Debug undefined wasmer_vm_raise_trap error
fschutt Dec 1, 2022
5d4817c
Undo changes, recomment out failing tests + add link to zig bug report
fschutt Dec 1, 2022
9ec93ac
Merge branch 'master' into fix-create-exe
fschutt Dec 1, 2022
54a0b08
Update lib/cli/src/commands/create_exe.rs
fschutt Dec 2, 2022
14dc05c
Refactor methods to filter tarballs
fschutt Dec 2, 2022
81cde2b
Fix make lint
fschutt Dec 2, 2022
cdbacbb
Remove --feature http because wasmer_registry requires HTTP deps
fschutt Dec 2, 2022
2e419eb
Disable LLVM testing on linux-musl due to missing C-API support
fschutt Dec 2, 2022
2c88eb8
Do not ignore error on fs::create_dir_all
fschutt Dec 2, 2022
cdd71d3
Add -OReleaseSafe back
fschutt Dec 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
111 changes: 88 additions & 23 deletions lib/cli/src/commands/create_exe.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,7 +403,6 @@ impl CreateExe {
v.canonicalize().unwrap_or(v)
} else {
{
let libwasmer_path = "lib/libwasmer.a";
let tarball_dir;
let filename = if let Some(local_tarball) = cross_subc.tarball.as_ref() {
let target_file_path = local_tarball
Expand All @@ -419,32 +418,102 @@ impl CreateExe {
let _ = std::fs::create_dir_all(&target_file_path);
let files = untar(local_tarball.clone(), target_file_path.clone())?;
tarball_dir = target_file_path.canonicalize().unwrap_or(target_file_path);
files.iter().find(|f| f.contains(libwasmer_path)).cloned().ok_or_else(|| {
anyhow!("Could not find libwasmer for {} target in the provided tarball path (files = {files:#?}, libwasmer_path = {libwasmer_path:?})", target)})?
files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| {
fschutt marked this conversation as resolved.
Show resolved Hide resolved
anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)})?
fschutt marked this conversation as resolved.
Show resolved Hide resolved
} else {
#[cfg(feature = "http")]
{
let release = http_fetch::get_latest_release()?;
let tarball = http_fetch::download_release(release, target.clone())?;
let target_file_path = tarball
// check if the tarball for the target already exists locally
let local_tarball = std::fs::read_dir(get_libwasmer_cache_path()?)?
.filter_map(|e| e.ok())
.filter_map(|e| {
let path = format!("{}", e.path().display());
if path.ends_with(".tar.gz") {
Some(e.path())
} else {
None
}
})
.filter_map(|p| {
if let Architecture::Aarch64(_) = target.architecture {
if !p.file_name()?.to_str()?.contains("aarch64") {
return None;
}
}

if let Architecture::X86_64 = target.architecture {
if !p.file_name()?.to_str()?.contains("x86_64") {
return None;
}
}

if let OperatingSystem::Windows = target.operating_system {
if !p.file_name()?.to_str()?.contains("windows") {
return None;
}
}

if let OperatingSystem::Darwin = target.operating_system {
if !(p.file_name()?.to_str()?.contains("apple")
|| p.file_name()?.to_str()?.contains("darwin"))
{
return None;
}
}

if let OperatingSystem::Linux = target.operating_system {
if !p.file_name()?.to_str()?.contains("linux") {
return None;
}
}

Some(p)
fschutt marked this conversation as resolved.
Show resolved Hide resolved
})
.next();

if let Some(local_tarball) = local_tarball.as_ref() {
let target_file_path = local_tarball
.parent()
.and_then(|parent| Some(parent.join(tarball.file_stem()?)))
.unwrap_or_else(|| tarball.clone());
.and_then(|parent| Some(parent.join(local_tarball.file_stem()?)))
.unwrap_or_else(|| local_tarball.clone());

let target_file_path = target_file_path
.parent()
.and_then(|parent| Some(parent.join(target_file_path.file_stem()?)))
.unwrap_or_else(|| target_file_path.clone());

tarball_dir = target_file_path
.canonicalize()
.unwrap_or_else(|_| target_file_path.clone());
let files = untar(tarball, target_file_path)?;
files.into_iter().find(|f| f.contains(libwasmer_path)).ok_or_else(|| {
anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag.", target)})?
let _ = std::fs::create_dir_all(&target_file_path);
let files = untar(local_tarball.clone(), target_file_path.clone())?;
tarball_dir =
target_file_path.canonicalize().unwrap_or(target_file_path);
files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| {
anyhow!("Could not find libwasmer.a for {} target in the provided tarball path (files = {files:#?})", target)})?
fschutt marked this conversation as resolved.
Show resolved Hide resolved
} else {
#[cfg(feature = "http")]
{
let release = http_fetch::get_latest_release()?;
let tarball =
http_fetch::download_release(release, target.clone())?;
let target_file_path = tarball
.parent()
.and_then(|parent| Some(parent.join(tarball.file_stem()?)))
.unwrap_or_else(|| tarball.clone());

let target_file_path = target_file_path
.parent()
.and_then(|parent| {
Some(parent.join(target_file_path.file_stem()?))
})
.unwrap_or_else(|| target_file_path.clone());

tarball_dir = target_file_path
.canonicalize()
.unwrap_or_else(|_| target_file_path.clone());
let files = untar(tarball, target_file_path)?;
files.iter().find(|f| f.ends_with("libwasmer.a")).cloned().ok_or_else(|| {
anyhow!("Could not find libwasmer for {} target in the fetched release from Github: you can download it manually and specify its path with the --cross-compilation-library-path LIBRARY_PATH flag. (files = {files:#?}", target)})?
}
#[cfg(not(feature = "http"))]
return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag."));
fschutt marked this conversation as resolved.
Show resolved Hide resolved
}
#[cfg(not(feature = "http"))]
return Err(anyhow!("This wasmer binary isn't compiled with an HTTP request library (feature flag `http`). To cross-compile, specify the path of the non-native libwasmer or release tarball with the --library-path LIBRARY_PATH or --tarball TARBALL_PATH flag."));
};
tarball_dir.join(&filename)
}
Expand Down Expand Up @@ -572,10 +641,6 @@ impl CreateExe {
cmd.arg("-lc");
}
cmd.arg("-lunwind");
cmd.arg("-OReleaseSafe");
fschutt marked this conversation as resolved.
Show resolved Hide resolved
cmd.arg("-fstrip");
fschutt marked this conversation as resolved.
Show resolved Hide resolved
cmd.arg("-dead_strip");
cmd.arg("-dead_strip_dylibs");
cmd.arg("-fno-compiler-rt");
cmd.arg(&format!("-femit-bin={}", output_path.display()));

Expand Down Expand Up @@ -1379,7 +1444,7 @@ mod http_fetch {
}

Err(anyhow!(
"Could not get expected Github API response.\n\nReason: response format is not recognized:\n{:#?}", ""
"Could not get expected Github API response.\n\nReason: response format is not recognized:\n{response:#?}",
))
}

Expand Down
2 changes: 1 addition & 1 deletion lib/compiler-singlepass/src/machine_x64.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ impl AssemblerX64 {
Some(CpuFeature::SSE42)
} else {
return Err(CompileError::UnsupportedTarget(
"x86_64 without AVX or SSE 4.2".to_string(),
"x86_64 without AVX or SSE 4.2, use -m avx to enable".to_string(),
));
}
};
Expand Down
92 changes: 58 additions & 34 deletions tests/integration/cli/tests/run.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ fn test_no_start_wat_path() -> PathBuf {
Path::new(ASSET_PATH).join("no_start.wat")
}

#[cfg(any(target_os = "linux", target_os = "macos"))]
#[test]
fn test_cross_compile_python_windows() -> anyhow::Result<()> {
let temp_dir = tempfile::TempDir::new()?;
Expand All @@ -34,41 +33,66 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> {
"x86_64-windows-gnu",
];

for t in targets {
let python_wasmer_path = temp_dir.path().join(format!("{t}-python"));

let mut output = Command::new(get_wasmer_path());

output.arg("create-exe");
output.arg(wasi_test_python_path());
output.arg("--target");
output.arg(t);
output.arg("-o");
output.arg(python_wasmer_path.clone());
let output = output.output()?;

let stdout = std::str::from_utf8(&output.stdout)
.expect("stdout is not utf8! need to handle arbitrary bytes");

let stderr = std::str::from_utf8(&output.stderr)
.expect("stderr is not utf8! need to handle arbitrary bytes");
let compilers = &["cranelift", "singlepass", "llvm"];

if !output.status.success() {
bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}");
}
// llvm-objdump --disassemble-all --demangle ./objects/wasmer_vm-50cb118b098c15db.wasmer_vm.60425a0a-cgu.12.rcgu.o
// llvm-objdump --macho --exports-trie ~/.wasmer/cache/wasmer-darwin-arm64/lib/libwasmer.dylib
let excluded_combinations = &[
("aarch64-darwin", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported
("aarch64-linux-gnu", "llvm"), // LLVM: aarch64 not supported relocation Arm64MovwG0 not supported
// https://github.com/ziglang/zig/issues/13729
("x86_64-darwin", "llvm"), // undefined reference to symbol 'wasmer_vm_raise_trap' kind Unknown
("x86_64-windows-gnu", "llvm"), // unimplemented symbol `wasmer_vm_raise_trap` kind Unknown
];

println!("stdout: {stdout}");
println!("stderr: {stderr}");

if !python_wasmer_path.exists() {
let p = std::fs::read_dir(temp_dir.path())
.unwrap()
.filter_map(|e| Some(e.ok()?.path()))
.collect::<Vec<_>>();
panic!(
"target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}",
p
);
for t in targets {
for c in compilers {
if excluded_combinations.contains(&(t, c)) {
continue;
}
println!("{t} target {c}");
let python_wasmer_path = temp_dir.path().join(format!("{t}-python"));

let mut output = Command::new(get_wasmer_path());

output.arg("create-exe");
output.arg(wasi_test_python_path());
output.arg("--target");
output.arg(t);
output.arg("-o");
output.arg(python_wasmer_path.clone());
output.arg(format!("--{c}"));

if t.contains("x86_64") && *c == "singlepass" {
output.arg("-m");
output.arg("avx");
}

let output = output.output()?;

let stdout = std::str::from_utf8(&output.stdout)
.expect("stdout is not utf8! need to handle arbitrary bytes");

let stderr = std::str::from_utf8(&output.stderr)
.expect("stderr is not utf8! need to handle arbitrary bytes");

if !output.status.success() {
bail!("linking failed with: stdout: {stdout}\n\nstderr: {stderr}");
}

println!("stdout: {stdout}");
println!("stderr: {stderr}");

if !python_wasmer_path.exists() {
let p = std::fs::read_dir(temp_dir.path())
.unwrap()
.filter_map(|e| Some(e.ok()?.path()))
.collect::<Vec<_>>();
panic!(
"target {t} was not compiled correctly {stdout} {stderr}, tempdir: {:#?}",
p
);
}
}
}

Expand Down