diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4c11cf472ab..f5e625f57ed 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -129,7 +129,7 @@ jobs: build-what: [ { key: capi, - build-cmd: 'make build-capi', + build-cmd: 'make build-capi && make package-capi', name: 'Build C-API' }, { @@ -159,6 +159,26 @@ jobs: PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig PKG_CONFIG_ALLOW_CROSS: true ENABLE_LLVM: 0 + - name: Dist + if: ${{ matrix.build-what.key == 'capi' }} + run: | + make distribution + env: + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross + CROSS_DOCKER_IN_DOCKER: true + CARGO_TARGET: aarch64-unknown-linux-gnu + PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig + PKG_CONFIG_ALLOW_CROSS: true + TARGET: aarch64-unknown-linux-gnu + TARGET_DIR: target/aarch64-unknown-linux-gnu/release + - name: Upload Artifacts + if: ${{ matrix.build-what.key == 'capi' }} + uses: actions/upload-artifact@v2 + with: + name: capi-linux-aarch64 + path: dist + if-no-files-found: error + retention-days: 2 build: name: ${{ matrix.build-what.name }} on ${{ matrix.metadata.build }} @@ -575,6 +595,73 @@ jobs: aws-endpoint: https://storage.googleapis.com aws-s3-bucket-endpoint: false aws-s3-force-path-style: true + - name: Prepare package directory + shell: bash + run: | + mkdir -p package + mkdir -p package/cache + - uses: actions/download-artifact@v3 + with: + name: capi-linux-aarch64 + path: package/cache/wasmercache1 + - uses: actions/download-artifact@v3 + with: + name: capi-windows-gnu + path: package/cache/wasmercache2 + - uses: actions/download-artifact@v3 + with: + name: capi-macos-arm + path: package/cache/wasmercache3 + - uses: actions/download-artifact@v3 + with: + name: capi-macos-x64 + path: package/cache/wasmercache4 + - uses: actions/download-artifact@v3 + with: + name: capi-linux-x64 + path: package/cache/wasmercache5 + - name: Copy .tar.gz files to proper location + shell: bash + run: | + ls package/cache/wasmercache1 + ls package/cache/wasmercache2 + ls package/cache/wasmercache3 + ls package/cache/wasmercache4 + ls package/cache/wasmercache5 + cp package/cache/wasmercache1/wasmer.tar.gz package/cache/wasmer-linux-aarch64.tar.gz + cp package/cache/wasmercache2/build-capi.tar.gz package/cache/wasmer-windows-gnu64.tar.gz + cp package/cache/wasmercache3/build-capi.tar.gz package/cache/wasmer-darwin-arm64.tar.gz + cp package/cache/wasmercache4/build-capi.tar.gz package/cache/wasmer-darwin-amd64.tar.gz + cp package/cache/wasmercache5/build-capi.tar.gz package/cache/wasmer-linux-amd64.tar.gz + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'windows-x64' }} + with: + name: capi-windows-gnu + path: download_link + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'linux-musl' }} + with: + name: capi-linux-musl + path: download_link + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'macos-arm' }} + with: + name: capi-macos-arm + path: download_link + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'macos-x64' }} + with: + name: capi-macos-x64 + path: download_link + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'linux-x64' }} + with: + name: capi-linux-x64 + path: download_link + - name: Copy build-capi.tar.gz to link.tar.gz + shell: bash + run: | + cp download_link/build-capi.tar.gz link.tar.gz - name: Unzip Artifacts shell: bash run: | diff --git a/.gitignore b/.gitignore index 1a08ad112dd..54290d7d6e7 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,6 @@ out.txt wapm.toml build-capi.tar.gz build-wasmer.tar.gz -lcov.info \ No newline at end of file +lcov.info +link/ +link.tar.gz \ No newline at end of file diff --git a/Makefile b/Makefile index e2944c9f7a2..a0454820fb6 100644 --- a/Makefile +++ b/Makefile @@ -561,10 +561,12 @@ test-wasi-unit: test-wasi: $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --release --tests $(compiler_features) -- wasi::wasitests -test-integration-cli: +test-integration-cli: build-wasmer build-capi package dist + cp ./dist/wasmer.tar.gz ./link.tar.gz rustup target add wasm32-wasi $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture --test-threads=1 +# Before running this in the CI, we need to set up link.tar.gz and /cache/wasmer-[target].tar.gz test-integration-cli-ci: rustup target add wasm32-wasi $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner -p wasmer-integration-tests-cli -- --nocapture --test-threads=1 || $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture --test-threads=1 @@ -618,6 +620,7 @@ endif package-capi: mkdir -p "package/include" mkdir -p "package/lib" + mkdir -p "package/winsdk" cp lib/c-api/wasmer.h* package/include cp lib/c-api/wasmer_wasm.h* package/include cp lib/c-api/tests/wasm-c-api/include/wasm.h* package/include @@ -728,7 +731,7 @@ package: package-wasmer package-minimal-headless-wasmer package-capi tar-capi: ls -R package - tar -C package -zcvf build-capi.tar.gz lib include + tar -C package -zcvf build-capi.tar.gz lib include winsdk untar-capi: mkdir -p package diff --git a/tests/integration/cli/Cargo.toml b/tests/integration/cli/Cargo.toml index 6fd3da57da5..0b3b9a17cbd 100644 --- a/tests/integration/cli/Cargo.toml +++ b/tests/integration/cli/Cargo.toml @@ -9,8 +9,6 @@ publish = false [dev-dependencies] rand = "0.8.5" -tar = "0.4.38" -flate2 = "1.0.24" target-lexicon = "0.12.4" pretty_assertions = "1.3.0" object = "0.30.0" @@ -19,6 +17,8 @@ object = "0.30.0" anyhow = "1" tempfile = "3" target-lexicon = "0.12.5" +tar = "0.4.38" +flate2 = "1.0.24" [features] default = ["webc_runner"] diff --git a/tests/integration/cli/tests/create_exe.rs b/tests/integration/cli/tests/create_exe.rs index 0da0d1b5471..1061f1c8999 100644 --- a/tests/integration/cli/tests/create_exe.rs +++ b/tests/integration/cli/tests/create_exe.rs @@ -66,6 +66,12 @@ impl WasmerCreateExe { output.args(self.extra_cli_flags.iter()); output.arg("-o"); output.arg(&self.native_executable_path); + if !self.extra_cli_flags.contains(&"--target".to_string()) { + let tarball_path = get_repo_root_path().unwrap().join("link.tar.gz"); + assert!(tarball_path.exists(), "link.tar.gz does not exist"); + output.arg("--tarball"); + output.arg(&tarball_path); + } let cmd = format!("{:?}", output); println!("(integration-test) running create-exe: {cmd}"); @@ -257,14 +263,8 @@ fn test_create_exe_with_precompiled_works_1() { // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_works() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -301,14 +301,8 @@ fn create_exe_works() -> anyhow::Result<()> { /// Tests that "-c" and "-- -c" are treated differently // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_works_multi_command_args_handling() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -373,14 +367,8 @@ fn create_exe_works_multi_command_args_handling() -> anyhow::Result<()> { // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_works_multi_command() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -436,14 +424,8 @@ fn create_exe_works_multi_command() -> anyhow::Result<()> { // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_works_with_file() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -509,12 +491,6 @@ fn create_exe_works_with_file() -> anyhow::Result<()> { // see https://github.com/wasmerio/wasmer/issues/3459 #[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_serialized_works() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -697,42 +673,24 @@ fn create_exe_with_object_input(args: Vec) -> anyhow::Result<()> { // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_with_object_input_default() -> anyhow::Result<()> { create_exe_with_object_input(vec![]) } // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_with_object_input_symbols() -> anyhow::Result<()> { create_exe_with_object_input(vec!["--object-format".to_string(), "symbols".to_string()]) } // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_with_object_input_serialized() -> anyhow::Result<()> { create_exe_with_object_input(vec![ "--object-format".to_string(), diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index fba20733e7b..56fd8178d88 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -3,7 +3,7 @@ use anyhow::{bail, Context}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use wasmer_integration_tests_cli::{get_repo_root_path, get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; +use wasmer_integration_tests_cli::{get_libwasmer_path, get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; fn wasi_test_python_path() -> PathBuf { Path::new(C_ASSET_PATH).join("python-0.1.0.wasmer") @@ -21,6 +21,27 @@ fn test_no_start_wat_path() -> PathBuf { Path::new(ASSET_PATH).join("no_start.wat") } +fn assert_tarball_is_present_local(target: &str) -> Result { + let wasmer_dir = std::env::var("WASMER_DIR").expect("no WASMER_DIR set"); + let directory = match target { + "aarch64-darwin" => "wasmer-darwin-arm64.tar.gz", + "x86_64-darwin" => "wasmer-darwin-amd64.tar.gz", + "x86_64-linux-gnu" => "wasmer-linux-amd64.tar.gz", + "aarch64-linux-gnu" => "wasmer-linux-aarch64.tar.gz", + "x86_64-windows-gnu" => "wasmer-windows-gnu64.tar.gz", + _ => return Err(anyhow::anyhow!("unknown target {target}")), + }; + let libwasmer_cache_path = Path::new(&wasmer_dir).join("cache").join(directory); + if !libwasmer_cache_path.exists() { + return Err(anyhow::anyhow!( + "targz {} does not exist", + libwasmer_cache_path.display() + )); + } + println!("using targz {}", libwasmer_cache_path.display()); + Ok(libwasmer_cache_path) +} + #[test] fn test_cross_compile_python_windows() -> anyhow::Result<()> { let temp_dir = tempfile::TempDir::new()?; @@ -66,6 +87,10 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { println!("{t} target {c}"); let python_wasmer_path = temp_dir.path().join(format!("{t}-python")); + let tarball = match std::env::var("GITHUB_TOKEN") { + Ok(_) => Some(assert_tarball_is_present_local(t)?), + Err(_) => None, + }; let mut output = Command::new(get_wasmer_path()); output.arg("create-exe"); @@ -75,12 +100,21 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { output.arg("-o"); output.arg(python_wasmer_path.clone()); output.arg(format!("--{c}")); + output.arg("--debug-dir"); + output.arg(format!("{t}-{c}")); if t.contains("x86_64") && *c == "singlepass" { output.arg("-m"); output.arg("avx"); } + if let Some(t) = tarball { + output.arg("--tarball"); + output.arg(t); + } + + println!("command {:?}", output); + let output = output.output()?; let stdout = std::str::from_utf8(&output.stdout) @@ -195,20 +229,6 @@ fn run_wasi_works() -> anyhow::Result<()> { Ok(()) } -#[cfg(feature = "webc_runner")] -fn package_directory(in_dir: &PathBuf, out: &PathBuf) { - use flate2::write::GzEncoder; - use flate2::Compression; - use std::fs::File; - let tar = File::create(out).unwrap(); - let enc = GzEncoder::new(tar, Compression::none()); - let mut a = tar::Builder::new(enc); - a.append_dir_all("bin", in_dir.join("bin")).unwrap(); - a.append_dir_all("lib", in_dir.join("lib")).unwrap(); - a.append_dir_all("include", in_dir.join("include")).unwrap(); - a.finish().unwrap(); -} - /// TODO: on linux-musl, the packaging of libwasmer.a doesn't work properly /// Tracked in https://github.com/wasmerio/wasmer/issues/3271 #[cfg(not(any(target_env = "musl", target_os = "windows")))] @@ -217,6 +237,8 @@ fn package_directory(in_dir: &PathBuf, out: &PathBuf) { fn test_wasmer_create_exe_pirita_works() -> anyhow::Result<()> { // let temp_dir = Path::new("debug"); // std::fs::create_dir_all(&temp_dir); + + use wasmer_integration_tests_cli::get_repo_root_path; let temp_dir = tempfile::TempDir::new()?; let temp_dir = temp_dir.path().to_path_buf(); let python_wasmer_path = temp_dir.join("python.wasmer"); @@ -224,24 +246,9 @@ fn test_wasmer_create_exe_pirita_works() -> anyhow::Result<()> { let python_exe_output_path = temp_dir.join("python"); let native_target = target_lexicon::HOST; - let root_path = get_repo_root_path().unwrap(); - let package_path = root_path.join("package"); - if !package_path.exists() { - panic!("package path {} does not exist", package_path.display()); - } - let tmp_targz_path = temp_dir.join("link.tar.gz"); + let tmp_targz_path = get_repo_root_path().unwrap().join("link.tar.gz"); + println!("compiling to target {native_target}"); - println!( - "packaging /package to .tar.gz: {}", - tmp_targz_path.display() - ); - package_directory(&package_path, &tmp_targz_path); - println!("packaging done"); - println!( - "tmp tar gz path: {} - exists: {:?}", - tmp_targz_path.display(), - tmp_targz_path.exists() - ); let mut cmd = Command::new(get_wasmer_path()); cmd.arg("create-exe");