Miscompilation when running create-exe with self-built .tar.gz tarball

[fschutt]

#[cfg(feature = "webc_runner")]
fn package_directory(in_dir: &[(&str, PathBuf)], out: &PathBuf) {
    use flate2::write::GzEncoder;
    use flate2::Compression;
    use std::fs::File;
    let tar = File::create(out).unwrap();
    let enc = GzEncoder::new(tar, Compression::default());
    let mut a = tar::Builder::new(enc);
    for (k, i) in in_dir {
        a.append_dir_all(k, i).unwrap();
    }
    a.finish().unwrap();
}

#[allow(dead_code)]
#[cfg(test)]
fn make_package(root_path: &PathBuf) -> anyhow::Result<()> {
    let current_dir = std::env::current_dir().unwrap();
    println!("running make && make build-capi && make package-capi && make package...");
    println!("current dir = {}", current_dir.display());
    println!("setting current dir = {}", root_path.display());
    // make && make build-capi && make package-capi && make package
    let mut c1 = std::process::Command::new("make");
    c1.current_dir(&root_path);
    let r = c1.output().unwrap();
    if !r.status.success() {
        let stdout = String::from_utf8_lossy(&r.stdout);
        let stderr = String::from_utf8_lossy(&r.stdout);
        println!("make failed: (stdout = {stdout}, stderr = {stderr})");
    }
    println!("make ok!");
    let mut c1 = std::process::Command::new("make");
    c1.arg("build-wasmer");
    c1.current_dir(&root_path);
    let r = c1.output().unwrap();
    if !r.status.success() {
        let stdout = String::from_utf8_lossy(&r.stdout);
        let stderr = String::from_utf8_lossy(&r.stdout);
        println!("make failed: (stdout = {stdout}, stderr = {stderr})");
    }
    println!("make build-wasmer ok!");
    let mut c1 = std::process::Command::new("make");
    c1.arg("build-capi");
    c1.current_dir(&root_path);
    let r = c1.output().unwrap();
    if !r.status.success() {
        let stdout = String::from_utf8_lossy(&r.stdout);
        let stderr = String::from_utf8_lossy(&r.stdout);
        println!("make build-capi failed: (stdout = {stdout}, stderr = {stderr})");
    }
    println!("make build-capi ok!");

    let mut c1 = std::process::Command::new("make");
    c1.arg("build-wasmer");
    c1.current_dir(&root_path);
    let r = c1.output().unwrap();
    if !r.status.success() {
        let stdout = String::from_utf8_lossy(&r.stdout);
        let stderr = String::from_utf8_lossy(&r.stdout);
        println!("make build-wasmer failed: (stdout = {stdout}, stderr = {stderr})");
    }
    println!("make build-wasmer ok!");

    let mut c1 = std::process::Command::new("make");
    c1.arg("package-capi");
    c1.current_dir(&root_path);
    let r = c1.output().unwrap();
    if !r.status.success() {
        let stdout = String::from_utf8_lossy(&r.stdout);
        let stderr = String::from_utf8_lossy(&r.stdout);
        println!("make package-capi: (stdout = {stdout}, stderr = {stderr})");
    }
    println!("make package-capi ok!");

    let mut c1 = std::process::Command::new("make");
    c1.arg("package");
    c1.current_dir(&root_path);
    let r = c1.output().unwrap();
    if !r.status.success() {
        let stdout = String::from_utf8_lossy(&r.stdout);
        let stderr = String::from_utf8_lossy(&r.stdout);
        println!("make package failed: (stdout = {stdout}, stderr = {stderr})");
    }
    println!("make package ok!");
    Ok(())
}

/// TODO: on linux-musl, the packaging of libwasmer.a doesn't work properly
/// Tracked in https://github.com/wasmerio/wasmer/issues/3271
#[cfg(not(target_env = "musl"))]
#[cfg(feature = "webc_runner")]
#[test]
fn test_wasmer_create_exe_pirita_works() -> anyhow::Result<()> {
    let temp_dir = tempfile::TempDir::new()?;
    let python_wasmer_path = temp_dir.path().join("python.wasmer");
    std::fs::copy(wasi_test_python_path(), &python_wasmer_path)?;
    let python_exe_output_path = temp_dir.path().join("python");

    let native_target = target_lexicon::HOST;
    let root_path = get_repo_root_path().unwrap();
    let package_path = root_path.join("package");
    make_package(&root_path)?;
    if !package_path.exists() {
        panic!("package path {} does not exist", package_path.display());
    }
    let tmp_targz_path = tempfile::TempDir::new()?;
    let tmp_targz_path = tmp_targz_path.path().join("link.tar.gz");
    println!("compiling to target {native_target}");
    println!(
        "packaging /package to .tar.gz: {}",
        tmp_targz_path.display()
    );
    package_directory(
        &[
            ("bin", package_path.join("bin")),
            ("include", package_path.join("include")),
            ("lib", package_path.join("lib")),
        ],
        &std::path::Path::new("./out.tar.gz").to_path_buf(),
    );
    std::fs::copy("./out.tar.gz", &tmp_targz_path).unwrap();
    println!("packaging done");
    println!(
        "tmp tar gz path: {} - exists: {:?}",
        tmp_targz_path.display(),
        tmp_targz_path.exists()
    );

    let mut cmd = Command::new(get_wasmer_path());
    cmd.arg("create-exe");
    cmd.arg(&python_wasmer_path);
    cmd.arg("--tarball");
    cmd.arg(&tmp_targz_path);
    cmd.arg("--target");
    cmd.arg(format!("{native_target}"));
    cmd.arg("-o");
    cmd.arg(&python_exe_output_path);

    println!("running: {cmd:?}");

    let output = cmd.output()?;

    if !output.status.success() {
        let stdout = std::str::from_utf8(&output.stdout)
            .expect("stdout is not utf8! need to handle arbitrary bytes");

        bail!(
            "running wasmer create-exe {} failed with: stdout: {}\n\nstderr: {}",
            python_wasmer_path.display(),
            stdout,
            std::str::from_utf8(&output.stderr)
                .expect("stderr is not utf8! need to handle arbitrary bytes")
        );
    }

    println!("compilation ok!");

    if !python_exe_output_path.exists() {
        return Err(anyhow::anyhow!(
            "python_exe_output_path {} does not exist",
            python_exe_output_path.display()
        ));
    }

    println!("invoking command...");

    let mut command = Command::new(&python_exe_output_path);
    command.arg("-c");
    command.arg("print(\"hello\")");

    let output = command
        .output()
        .map_err(|e| anyhow::anyhow!("{e}: {command:?}"))?;

    let stdout = std::str::from_utf8(&output.stdout)
        .expect("stdout is not utf8! need to handle arbitrary bytes");

    if stdout != "hello\n" {
        bail!(
            "1 running python.wasmer failed with: stdout: {}\n\nstderr: {}",
            stdout,
            std::str::from_utf8(&output.stderr)
                .expect("stderr is not utf8! need to handle arbitrary bytes")
        );
    }

    Ok(())
}

... results in a 30 MB binary that can be executed, but only crashes with Trap is not NULL: TODO:. However, create-exe does work with the tarball from the GitHub release, so this test is disabled for now.

[stale]

This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.

[stale]

Feel free to reopen the issue if it has been closed by mistake.