From f9e91a2c197655488ce2aa8f8efa24480a561508 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 15:03:15 +0100 Subject: [PATCH 01/18] Reenable create-exe tests --- tests/integration/cli/tests/create_exe.rs | 62 +++-------------------- 1 file changed, 7 insertions(+), 55 deletions(-) diff --git a/tests/integration/cli/tests/create_exe.rs b/tests/integration/cli/tests/create_exe.rs index 0da0d1b5471..25edb4617a9 100644 --- a/tests/integration/cli/tests/create_exe.rs +++ b/tests/integration/cli/tests/create_exe.rs @@ -257,14 +257,8 @@ fn test_create_exe_with_precompiled_works_1() { // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_works() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -301,14 +295,8 @@ fn create_exe_works() -> anyhow::Result<()> { /// Tests that "-c" and "-- -c" are treated differently // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_works_multi_command_args_handling() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -373,14 +361,8 @@ fn create_exe_works_multi_command_args_handling() -> anyhow::Result<()> { // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_works_multi_command() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -436,14 +418,8 @@ fn create_exe_works_multi_command() -> anyhow::Result<()> { // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_works_with_file() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -509,12 +485,6 @@ fn create_exe_works_with_file() -> anyhow::Result<()> { // see https://github.com/wasmerio/wasmer/issues/3459 #[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_serialized_works() -> anyhow::Result<()> { let temp_dir = tempfile::tempdir()?; let operating_dir: PathBuf = temp_dir.path().to_owned(); @@ -697,42 +667,24 @@ fn create_exe_with_object_input(args: Vec) -> anyhow::Result<()> { // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_with_object_input_default() -> anyhow::Result<()> { create_exe_with_object_input(vec![]) } // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_with_object_input_symbols() -> anyhow::Result<()> { create_exe_with_object_input(vec!["--object-format".to_string(), "symbols".to_string()]) } // Ignored because of -lunwind linker issue on Windows // see https://github.com/wasmerio/wasmer/issues/3459 -// #[cfg_attr(target_os = "windows", ignore)] +#[cfg_attr(target_os = "windows", ignore)] #[test] -// Test temporarily ignored during the release of 3.2.0-alpha -// because create-exe links to the old libwasmer.a which expects -// MetadataHeader::VERSION == 1, but we want to upgrade to version 2. -// -// https://github.com/wasmerio/wasmer/issues/3513 -#[ignore] fn create_exe_with_object_input_serialized() -> anyhow::Result<()> { create_exe_with_object_input(vec![ "--object-format".to_string(), From 94be948b426a81d4c9bd6bd643672c7300990e8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 15:31:04 +0100 Subject: [PATCH 02/18] Integration tests: use link.tar.gz instead of downloading from CI --- .gitignore | 4 ++- tests/integration/cli/Cargo.toml | 4 +-- tests/integration/cli/src/util.rs | 38 +++++++++++++++++++++++ tests/integration/cli/tests/create_exe.rs | 6 ++++ tests/integration/cli/tests/run.rs | 35 +++------------------ 5 files changed, 53 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 1a08ad112dd..54290d7d6e7 100644 --- a/.gitignore +++ b/.gitignore @@ -21,4 +21,6 @@ out.txt wapm.toml build-capi.tar.gz build-wasmer.tar.gz -lcov.info \ No newline at end of file +lcov.info +link/ +link.tar.gz \ No newline at end of file diff --git a/tests/integration/cli/Cargo.toml b/tests/integration/cli/Cargo.toml index 6fd3da57da5..0b3b9a17cbd 100644 --- a/tests/integration/cli/Cargo.toml +++ b/tests/integration/cli/Cargo.toml @@ -9,8 +9,6 @@ publish = false [dev-dependencies] rand = "0.8.5" -tar = "0.4.38" -flate2 = "1.0.24" target-lexicon = "0.12.4" pretty_assertions = "1.3.0" object = "0.30.0" @@ -19,6 +17,8 @@ object = "0.30.0" anyhow = "1" tempfile = "3" target-lexicon = "0.12.5" +tar = "0.4.38" +flate2 = "1.0.24" [features] default = ["webc_runner"] diff --git a/tests/integration/cli/src/util.rs b/tests/integration/cli/src/util.rs index 5fd43866085..e17f071f016 100644 --- a/tests/integration/cli/src/util.rs +++ b/tests/integration/cli/src/util.rs @@ -1,5 +1,7 @@ +use crate::get_repo_root_path; use anyhow::bail; use std::path::Path; +use std::path::PathBuf; use std::process::Command; #[derive(Debug, Copy, Clone)] @@ -48,3 +50,39 @@ pub fn run_code( Ok(output.to_owned()) } + +// Take the wasmer/package directory and package it to a .tar.gz tarball +pub fn package_wasmer_to_tarball(tmp_targz_path: &PathBuf) { + if tmp_targz_path.exists() { + return; + } + let root_path = get_repo_root_path().unwrap(); + let package_path = root_path.join("package"); + if !package_path.exists() { + panic!("package path {} does not exist", package_path.display()); + } + println!( + "packaging /package to .tar.gz: {}", + tmp_targz_path.display() + ); + package_directory(&package_path, &tmp_targz_path); + println!("packaging done"); + println!( + "tmp tar gz path: {} - exists: {:?}", + tmp_targz_path.display(), + tmp_targz_path.exists() + ); +} + +fn package_directory(in_dir: &PathBuf, out: &PathBuf) { + use flate2::write::GzEncoder; + use flate2::Compression; + use std::fs::File; + let tar = File::create(out).unwrap(); + let enc = GzEncoder::new(tar, Compression::none()); + let mut a = tar::Builder::new(enc); + a.append_dir_all("bin", in_dir.join("bin")).unwrap(); + a.append_dir_all("lib", in_dir.join("lib")).unwrap(); + a.append_dir_all("include", in_dir.join("include")).unwrap(); + a.finish().unwrap(); +} diff --git a/tests/integration/cli/tests/create_exe.rs b/tests/integration/cli/tests/create_exe.rs index 25edb4617a9..56d8595f57c 100644 --- a/tests/integration/cli/tests/create_exe.rs +++ b/tests/integration/cli/tests/create_exe.rs @@ -66,6 +66,12 @@ impl WasmerCreateExe { output.args(self.extra_cli_flags.iter()); output.arg("-o"); output.arg(&self.native_executable_path); + if !self.extra_cli_flags.contains(&"--target".to_string()) { + let tarball_path = get_repo_root_path().unwrap().join("link.tar.gz"); + package_wasmer_to_tarball(&tarball_path); + output.arg("--tarball"); + output.arg(&tarball_path); + } let cmd = format!("{:?}", output); println!("(integration-test) running create-exe: {cmd}"); diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index fba20733e7b..d1dbb9e44be 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -3,7 +3,9 @@ use anyhow::{bail, Context}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use wasmer_integration_tests_cli::{get_repo_root_path, get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; +use wasmer_integration_tests_cli::{ + get_wasmer_path, package_wasmer_to_tarball, ASSET_PATH, C_ASSET_PATH, +}; fn wasi_test_python_path() -> PathBuf { Path::new(C_ASSET_PATH).join("python-0.1.0.wasmer") @@ -195,20 +197,6 @@ fn run_wasi_works() -> anyhow::Result<()> { Ok(()) } -#[cfg(feature = "webc_runner")] -fn package_directory(in_dir: &PathBuf, out: &PathBuf) { - use flate2::write::GzEncoder; - use flate2::Compression; - use std::fs::File; - let tar = File::create(out).unwrap(); - let enc = GzEncoder::new(tar, Compression::none()); - let mut a = tar::Builder::new(enc); - a.append_dir_all("bin", in_dir.join("bin")).unwrap(); - a.append_dir_all("lib", in_dir.join("lib")).unwrap(); - a.append_dir_all("include", in_dir.join("include")).unwrap(); - a.finish().unwrap(); -} - /// TODO: on linux-musl, the packaging of libwasmer.a doesn't work properly /// Tracked in https://github.com/wasmerio/wasmer/issues/3271 #[cfg(not(any(target_env = "musl", target_os = "windows")))] @@ -224,24 +212,9 @@ fn test_wasmer_create_exe_pirita_works() -> anyhow::Result<()> { let python_exe_output_path = temp_dir.join("python"); let native_target = target_lexicon::HOST; - let root_path = get_repo_root_path().unwrap(); - let package_path = root_path.join("package"); - if !package_path.exists() { - panic!("package path {} does not exist", package_path.display()); - } let tmp_targz_path = temp_dir.join("link.tar.gz"); + package_wasmer_to_tarball(&tmp_targz_path); println!("compiling to target {native_target}"); - println!( - "packaging /package to .tar.gz: {}", - tmp_targz_path.display() - ); - package_directory(&package_path, &tmp_targz_path); - println!("packaging done"); - println!( - "tmp tar gz path: {} - exists: {:?}", - tmp_targz_path.display(), - tmp_targz_path.exists() - ); let mut cmd = Command::new(get_wasmer_path()); cmd.arg("create-exe"); From df0615d0e6aa1703d7b701a6c76f45ea014fdde2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 16:20:59 +0100 Subject: [PATCH 03/18] create-exe: always use locally downloaded tarballs --- tests/integration/cli/tests/run.rs | 32 +++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index d1dbb9e44be..6f1d46a1e0a 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -4,7 +4,7 @@ use anyhow::{bail, Context}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use wasmer_integration_tests_cli::{ - get_wasmer_path, package_wasmer_to_tarball, ASSET_PATH, C_ASSET_PATH, + get_libwasmer_path, get_wasmer_path, package_wasmer_to_tarball, ASSET_PATH, C_ASSET_PATH, }; fn wasi_test_python_path() -> PathBuf { @@ -23,6 +23,27 @@ fn test_no_start_wat_path() -> PathBuf { Path::new(ASSET_PATH).join("no_start.wat") } +fn assert_tarball_is_present_local(target: &str) -> Result { + let wasmer_dir = std::env::var("WASMER_DIR").expect("no WASMER_DIR set"); + let directory = match target { + "aarch64-darwin" => "wasmer-darwin-arm64.tar.gz", + "x86_64-darwin" => "wasmer-darwin-amd64.tar.gz", + "x86_64-linux-gnu" => "wasmer-linux-amd64.tar.gz", + "aarch64-linux-gnu" => "wasmer-linux-aarch64.tar.gz", + "x86_64-windows-gnu" => "wasmer-windows-gnu64.tar.gz", + _ => return Err(anyhow::anyhow!("unknown target {target}")), + }; + let libwasmer_cache_path = Path::new(&wasmer_dir).join("cache").join(directory); + if !libwasmer_cache_path.exists() { + return Err(anyhow::anyhow!( + "targz {} does not exist", + libwasmer_cache_path.display() + )); + } + println!("using targz {}", libwasmer_cache_path.display()); + Ok(libwasmer_cache_path) +} + #[test] fn test_cross_compile_python_windows() -> anyhow::Result<()> { let temp_dir = tempfile::TempDir::new()?; @@ -68,6 +89,10 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { println!("{t} target {c}"); let python_wasmer_path = temp_dir.path().join(format!("{t}-python")); + let tarball = match std::env::var("GITHUB_TOKEN") { + Ok(_) => Some(assert_tarball_is_present_local(target)?), + Err(_) => None, + }; let mut output = Command::new(get_wasmer_path()); output.arg("create-exe"); @@ -83,6 +108,11 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { output.arg("avx"); } + if let Some(t) = tarball { + output.arg("--tarball"); + output.arg(t); + } + let output = output.output()?; let stdout = std::str::from_utf8(&output.stdout) From dfd5d29b9308642fbde92d09f851cedca319f15d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 16:45:48 +0100 Subject: [PATCH 04/18] Package link.tar.gz via Makefile instead of Rust code --- .github/workflows/test.yaml | 25 ++++++++++++++++ Makefile | 4 ++- tests/integration/cli/src/util.rs | 36 ----------------------- tests/integration/cli/tests/create_exe.rs | 2 +- tests/integration/cli/tests/run.rs | 12 ++++---- 5 files changed, 35 insertions(+), 44 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4c11cf472ab..c7b113fb717 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -575,6 +575,31 @@ jobs: aws-endpoint: https://storage.googleapis.com aws-s3-bucket-endpoint: false aws-s3-force-path-style: true + - name: Prepare package directory + shell: bash + run: | + mkdir -p package + mkdir -p package/cache + - uses: actions/download-artifact@v3 + with: + name: capi-linux-aarch64 + download-path: package/cache/wasmer-linux-aarch64.tar.gz + - uses: actions/download-artifact@v3 + with: + name: capi-windows-gnu64 + download-path: package/cache/wasmer-windows-gnu64.tar.gz + - uses: actions/download-artifact@v3 + with: + name: capi-darwin-arm64 + download-path: package/cache/wasmer-darwin-arm64.tar.gz + - uses: actions/download-artifact@v3 + with: + name: capi-darwin-amd64 + download-path: package/cache/wasmer-darwin-amd64.tar.gz + - uses: actions/download-artifact@v3 + with: + name: capi-linux-amd64 + download-path: package/cache/wasmer-linux-amd64.tar.gz - name: Unzip Artifacts shell: bash run: | diff --git a/Makefile b/Makefile index e2944c9f7a2..a7c1d1b3d72 100644 --- a/Makefile +++ b/Makefile @@ -561,10 +561,12 @@ test-wasi-unit: test-wasi: $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --release --tests $(compiler_features) -- wasi::wasitests -test-integration-cli: +test-integration-cli: build-wasmer build-capi package dist + cp ./dist/wasmer.tar.gz ./link.tar.gz rustup target add wasm32-wasi $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture --test-threads=1 +# Before running this in the CI, we need to set up link.tar.gz and /cache/wasmer-[target].tar.gz test-integration-cli-ci: rustup target add wasm32-wasi $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner -p wasmer-integration-tests-cli -- --nocapture --test-threads=1 || $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner --no-fail-fast -p wasmer-integration-tests-cli -- --nocapture --test-threads=1 diff --git a/tests/integration/cli/src/util.rs b/tests/integration/cli/src/util.rs index e17f071f016..06f692b1270 100644 --- a/tests/integration/cli/src/util.rs +++ b/tests/integration/cli/src/util.rs @@ -50,39 +50,3 @@ pub fn run_code( Ok(output.to_owned()) } - -// Take the wasmer/package directory and package it to a .tar.gz tarball -pub fn package_wasmer_to_tarball(tmp_targz_path: &PathBuf) { - if tmp_targz_path.exists() { - return; - } - let root_path = get_repo_root_path().unwrap(); - let package_path = root_path.join("package"); - if !package_path.exists() { - panic!("package path {} does not exist", package_path.display()); - } - println!( - "packaging /package to .tar.gz: {}", - tmp_targz_path.display() - ); - package_directory(&package_path, &tmp_targz_path); - println!("packaging done"); - println!( - "tmp tar gz path: {} - exists: {:?}", - tmp_targz_path.display(), - tmp_targz_path.exists() - ); -} - -fn package_directory(in_dir: &PathBuf, out: &PathBuf) { - use flate2::write::GzEncoder; - use flate2::Compression; - use std::fs::File; - let tar = File::create(out).unwrap(); - let enc = GzEncoder::new(tar, Compression::none()); - let mut a = tar::Builder::new(enc); - a.append_dir_all("bin", in_dir.join("bin")).unwrap(); - a.append_dir_all("lib", in_dir.join("lib")).unwrap(); - a.append_dir_all("include", in_dir.join("include")).unwrap(); - a.finish().unwrap(); -} diff --git a/tests/integration/cli/tests/create_exe.rs b/tests/integration/cli/tests/create_exe.rs index 56d8595f57c..1061f1c8999 100644 --- a/tests/integration/cli/tests/create_exe.rs +++ b/tests/integration/cli/tests/create_exe.rs @@ -68,7 +68,7 @@ impl WasmerCreateExe { output.arg(&self.native_executable_path); if !self.extra_cli_flags.contains(&"--target".to_string()) { let tarball_path = get_repo_root_path().unwrap().join("link.tar.gz"); - package_wasmer_to_tarball(&tarball_path); + assert!(tarball_path.exists(), "link.tar.gz does not exist"); output.arg("--tarball"); output.arg(&tarball_path); } diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index 6f1d46a1e0a..ac0815632e7 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -3,9 +3,7 @@ use anyhow::{bail, Context}; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; -use wasmer_integration_tests_cli::{ - get_libwasmer_path, get_wasmer_path, package_wasmer_to_tarball, ASSET_PATH, C_ASSET_PATH, -}; +use wasmer_integration_tests_cli::{get_libwasmer_path, get_wasmer_path, ASSET_PATH, C_ASSET_PATH}; fn wasi_test_python_path() -> PathBuf { Path::new(C_ASSET_PATH).join("python-0.1.0.wasmer") @@ -90,7 +88,7 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { let python_wasmer_path = temp_dir.path().join(format!("{t}-python")); let tarball = match std::env::var("GITHUB_TOKEN") { - Ok(_) => Some(assert_tarball_is_present_local(target)?), + Ok(_) => Some(assert_tarball_is_present_local(t)?), Err(_) => None, }; let mut output = Command::new(get_wasmer_path()); @@ -235,6 +233,8 @@ fn run_wasi_works() -> anyhow::Result<()> { fn test_wasmer_create_exe_pirita_works() -> anyhow::Result<()> { // let temp_dir = Path::new("debug"); // std::fs::create_dir_all(&temp_dir); + + use wasmer_integration_tests_cli::get_repo_root_path; let temp_dir = tempfile::TempDir::new()?; let temp_dir = temp_dir.path().to_path_buf(); let python_wasmer_path = temp_dir.join("python.wasmer"); @@ -242,8 +242,8 @@ fn test_wasmer_create_exe_pirita_works() -> anyhow::Result<()> { let python_exe_output_path = temp_dir.join("python"); let native_target = target_lexicon::HOST; - let tmp_targz_path = temp_dir.join("link.tar.gz"); - package_wasmer_to_tarball(&tmp_targz_path); + let tmp_targz_path = get_repo_root_path().unwrap().join("link.tar.gz"); + println!("compiling to target {native_target}"); let mut cmd = Command::new(get_wasmer_path()); From a223de20c50f4e2ed90e1ff723a79fe1ce3a0e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 17:13:43 +0100 Subject: [PATCH 05/18] Fix make lint --- tests/integration/cli/src/util.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/cli/src/util.rs b/tests/integration/cli/src/util.rs index 06f692b1270..5fd43866085 100644 --- a/tests/integration/cli/src/util.rs +++ b/tests/integration/cli/src/util.rs @@ -1,7 +1,5 @@ -use crate::get_repo_root_path; use anyhow::bail; use std::path::Path; -use std::path::PathBuf; use std::process::Command; #[derive(Debug, Copy, Clone)] From b949e2f858208a69a26e98ecbd9870da9dd6a69d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 17:32:35 +0100 Subject: [PATCH 06/18] Add linux-aarch64 .tar.gz to build pipeline --- .github/workflows/test.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c7b113fb717..92d2830a2e7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -129,7 +129,7 @@ jobs: build-what: [ { key: capi, - build-cmd: 'make build-capi', + build-cmd: 'make build-capi && make package-capi && make dist', name: 'Build C-API' }, { @@ -159,6 +159,13 @@ jobs: PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig PKG_CONFIG_ALLOW_CROSS: true ENABLE_LLVM: 0 + - name: Archive production artifacts + if: ${{ matrix.build-what.key == 'capi' }} + uses: actions/upload-artifact@v3 + with: + name: capi-linux-aarch64 + path: dist/wasmer.tar.gz + retention-days: 2 build: name: ${{ matrix.build-what.name }} on ${{ matrix.metadata.build }} From 947bfcb5fe36a0bcc900ebfd45011bf83bd80a31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 18:01:18 +0100 Subject: [PATCH 07/18] make dist -> make distribution --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 92d2830a2e7..86306a7e5bb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -129,7 +129,7 @@ jobs: build-what: [ { key: capi, - build-cmd: 'make build-capi && make package-capi && make dist', + build-cmd: 'make build-capi && make package-capi && make distribution', name: 'Build C-API' }, { From 1866a98da6bc62b9b138129fb29d6e3188a6531c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 18:20:37 +0100 Subject: [PATCH 08/18] Fix artifact naming --- .github/workflows/test.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 86306a7e5bb..5b384545fe0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -590,23 +590,23 @@ jobs: - uses: actions/download-artifact@v3 with: name: capi-linux-aarch64 - download-path: package/cache/wasmer-linux-aarch64.tar.gz + path: package/cache/wasmer-linux-aarch64.tar.gz - uses: actions/download-artifact@v3 with: - name: capi-windows-gnu64 - download-path: package/cache/wasmer-windows-gnu64.tar.gz + name: capi-windows-gnu + path: package/cache/wasmer-windows-gnu64.tar.gz - uses: actions/download-artifact@v3 with: - name: capi-darwin-arm64 - download-path: package/cache/wasmer-darwin-arm64.tar.gz + name: capi-macos-arm + path: package/cache/wasmer-darwin-arm64.tar.gz - uses: actions/download-artifact@v3 with: - name: capi-darwin-amd64 - download-path: package/cache/wasmer-darwin-amd64.tar.gz + name: capi-macos-x64 + path: package/cache/wasmer-darwin-amd64.tar.gz - uses: actions/download-artifact@v3 with: - name: capi-linux-amd64 - download-path: package/cache/wasmer-linux-amd64.tar.gz + name: capi-linux-x64 + path: package/cache/wasmer-linux-amd64.tar.gz - name: Unzip Artifacts shell: bash run: | From 3b75396bee8beeda6380b3e9dfe64f5dd75fbc5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 18:51:31 +0100 Subject: [PATCH 09/18] Download link.tar.gz --- .github/workflows/test.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5b384545fe0..27a407b1496 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -607,6 +607,26 @@ jobs: with: name: capi-linux-x64 path: package/cache/wasmer-linux-amd64.tar.gz + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'windows-gnu' }} + with: + name: capi-windows-gnu + path: link.tar.gz + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'macos-arm' }} + with: + name: capi-macos-arm + path: link.tar.gz + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'macos-x64' }} + with: + name: capi-macos-x64 + path: link.tar.gz + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'linux-x64' }} + with: + name: capi-linux-x64 + path: link.tar.gz - name: Unzip Artifacts shell: bash run: | From bc4b24c4ac88025f2bedb42e8ee3ed5e9185a81c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 19:18:35 +0100 Subject: [PATCH 10/18] Try to download CAPI artifacts correctly --- .github/workflows/test.yaml | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 27a407b1496..6a55c7c43c1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -611,22 +611,31 @@ jobs: if: ${{ matrix.build == 'windows-gnu' }} with: name: capi-windows-gnu - path: link.tar.gz + path: download_link + - uses: actions/download-artifact@v3 + if: ${{ matrix.build == 'linux-musl' }} + with: + name: capi-linux-musl + path: download_link - uses: actions/download-artifact@v3 if: ${{ matrix.build == 'macos-arm' }} with: name: capi-macos-arm - path: link.tar.gz + path: download_link - uses: actions/download-artifact@v3 if: ${{ matrix.build == 'macos-x64' }} with: name: capi-macos-x64 - path: link.tar.gz + path: download_link - uses: actions/download-artifact@v3 if: ${{ matrix.build == 'linux-x64' }} with: name: capi-linux-x64 - path: link.tar.gz + path: download_link + - name: List download_link + shell: bash + run: | + ls download_link - name: Unzip Artifacts shell: bash run: | From 04e4042a8f638e6bb47489a3b48fa150254a858a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 20:10:17 +0100 Subject: [PATCH 11/18] Fix missing link.tar.gz --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 6a55c7c43c1..3c696f9d4aa 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -632,10 +632,10 @@ jobs: with: name: capi-linux-x64 path: download_link - - name: List download_link + - name: Copy build-capi.tar.gz to link.tar.gz shell: bash run: | - ls download_link + cp download_link/build-capi.tar.gz link.tar.gz - name: Unzip Artifacts shell: bash run: | From a5acc35260fdfb1809e56c63ce3701cd7b3c269a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 20:25:36 +0100 Subject: [PATCH 12/18] Fix copying .tar.gz files to the correct location --- .github/workflows/test.yaml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 3c696f9d4aa..7adc7b80c25 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -590,23 +590,31 @@ jobs: - uses: actions/download-artifact@v3 with: name: capi-linux-aarch64 - path: package/cache/wasmer-linux-aarch64.tar.gz + path: package/cache/wasmercache1 - uses: actions/download-artifact@v3 with: name: capi-windows-gnu - path: package/cache/wasmer-windows-gnu64.tar.gz + path: package/cache/wasmercache2 - uses: actions/download-artifact@v3 with: name: capi-macos-arm - path: package/cache/wasmer-darwin-arm64.tar.gz + path: package/cache/wasmercache3 - uses: actions/download-artifact@v3 with: name: capi-macos-x64 - path: package/cache/wasmer-darwin-amd64.tar.gz + path: package/cache/wasmercache4 - uses: actions/download-artifact@v3 with: name: capi-linux-x64 - path: package/cache/wasmer-linux-amd64.tar.gz + path: package/cache/wasmercache5 + - name: Copy .tar.gz files to proper location + shell: bash + run: | + cp package/cache/wasmercache1/build-wasmer.tar.gz package/cache/wasmer-linux-aarch64.tar.gz + cp package/cache/wasmercache2/build-wasmer.tar.gz package/cache/wasmer-windows-gnu64.tar.gz + cp package/cache/wasmercache3/build-wasmer.tar.gz package/cache/wasmer-darwin-arm64.tar.gz + cp package/cache/wasmercache4/build-wasmer.tar.gz package/cache/wasmer-darwin-amd64.tar.gz + cp package/cache/wasmercache5/build-wasmer.tar.gz package/cache/wasmer-linux-amd64.tar.gz - uses: actions/download-artifact@v3 if: ${{ matrix.build == 'windows-gnu' }} with: From b0e133405dc24866db204bd7562bbc73b9ec1db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 20:50:15 +0100 Subject: [PATCH 13/18] Add debugging to test.yaml --- .github/workflows/test.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7adc7b80c25..47711b4d3fb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -610,6 +610,11 @@ jobs: - name: Copy .tar.gz files to proper location shell: bash run: | + ls package/cache/wasmercache1 + ls package/cache/wasmercache2 + ls package/cache/wasmercache3 + ls package/cache/wasmercache4 + ls package/cache/wasmercache5 cp package/cache/wasmercache1/build-wasmer.tar.gz package/cache/wasmer-linux-aarch64.tar.gz cp package/cache/wasmercache2/build-wasmer.tar.gz package/cache/wasmer-windows-gnu64.tar.gz cp package/cache/wasmercache3/build-wasmer.tar.gz package/cache/wasmer-darwin-arm64.tar.gz From 215ea41434484db71c7a17332957853ae63d2295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 20:54:02 +0100 Subject: [PATCH 14/18] Fix cp commands in copy step --- .github/workflows/test.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 47711b4d3fb..14ff0847235 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -615,11 +615,11 @@ jobs: ls package/cache/wasmercache3 ls package/cache/wasmercache4 ls package/cache/wasmercache5 - cp package/cache/wasmercache1/build-wasmer.tar.gz package/cache/wasmer-linux-aarch64.tar.gz - cp package/cache/wasmercache2/build-wasmer.tar.gz package/cache/wasmer-windows-gnu64.tar.gz - cp package/cache/wasmercache3/build-wasmer.tar.gz package/cache/wasmer-darwin-arm64.tar.gz - cp package/cache/wasmercache4/build-wasmer.tar.gz package/cache/wasmer-darwin-amd64.tar.gz - cp package/cache/wasmercache5/build-wasmer.tar.gz package/cache/wasmer-linux-amd64.tar.gz + cp package/cache/wasmercache1/build-capi.tar.gz package/cache/wasmer-linux-aarch64.tar.gz + cp package/cache/wasmercache2/build-capi.tar.gz package/cache/wasmer-windows-gnu64.tar.gz + cp package/cache/wasmercache3/build-capi.tar.gz package/cache/wasmer-darwin-arm64.tar.gz + cp package/cache/wasmercache4/build-capi.tar.gz package/cache/wasmer-darwin-amd64.tar.gz + cp package/cache/wasmercache5/build-capi.tar.gz package/cache/wasmer-linux-amd64.tar.gz - uses: actions/download-artifact@v3 if: ${{ matrix.build == 'windows-gnu' }} with: From 45a28d91549a8ad1d1b674509222ab98d92a4a34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 21:09:46 +0100 Subject: [PATCH 15/18] Fix cp command --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 14ff0847235..fd8d48836ce 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -615,7 +615,7 @@ jobs: ls package/cache/wasmercache3 ls package/cache/wasmercache4 ls package/cache/wasmercache5 - cp package/cache/wasmercache1/build-capi.tar.gz package/cache/wasmer-linux-aarch64.tar.gz + cp package/cache/wasmercache1/wasmer.tar.gz package/cache/wasmer-linux-aarch64.tar.gz cp package/cache/wasmercache2/build-capi.tar.gz package/cache/wasmer-windows-gnu64.tar.gz cp package/cache/wasmercache3/build-capi.tar.gz package/cache/wasmer-darwin-arm64.tar.gz cp package/cache/wasmercache4/build-capi.tar.gz package/cache/wasmer-darwin-amd64.tar.gz From 4ef22bd2568c8cb041238b8e02c924a3e0d0600f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Wed, 25 Jan 2023 21:56:02 +0100 Subject: [PATCH 16/18] Fix make distribution on linux-aarch64 --- .github/workflows/test.yaml | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fd8d48836ce..f5e625f57ed 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -129,7 +129,7 @@ jobs: build-what: [ { key: capi, - build-cmd: 'make build-capi && make package-capi && make distribution', + build-cmd: 'make build-capi && make package-capi', name: 'Build C-API' }, { @@ -159,12 +159,25 @@ jobs: PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig PKG_CONFIG_ALLOW_CROSS: true ENABLE_LLVM: 0 - - name: Archive production artifacts + - name: Dist if: ${{ matrix.build-what.key == 'capi' }} - uses: actions/upload-artifact@v3 + run: | + make distribution + env: + CARGO_BINARY: docker run -v /var/run/docker.sock:/var/run/docker.sock -v ${GITHUB_WORKSPACE}:/project -w /project wasmer/aarch64 cross + CROSS_DOCKER_IN_DOCKER: true + CARGO_TARGET: aarch64-unknown-linux-gnu + PKG_CONFIG_PATH: /usr/lib/aarch64-linux-gnu/pkgconfig + PKG_CONFIG_ALLOW_CROSS: true + TARGET: aarch64-unknown-linux-gnu + TARGET_DIR: target/aarch64-unknown-linux-gnu/release + - name: Upload Artifacts + if: ${{ matrix.build-what.key == 'capi' }} + uses: actions/upload-artifact@v2 with: name: capi-linux-aarch64 - path: dist/wasmer.tar.gz + path: dist + if-no-files-found: error retention-days: 2 build: @@ -621,7 +634,7 @@ jobs: cp package/cache/wasmercache4/build-capi.tar.gz package/cache/wasmer-darwin-amd64.tar.gz cp package/cache/wasmercache5/build-capi.tar.gz package/cache/wasmer-linux-amd64.tar.gz - uses: actions/download-artifact@v3 - if: ${{ matrix.build == 'windows-gnu' }} + if: ${{ matrix.build == 'windows-x64' }} with: name: capi-windows-gnu path: download_link From b3b7613ad26150832687c6008c21dd2d89f9b7c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 26 Jan 2023 09:59:00 +0100 Subject: [PATCH 17/18] Debug why linking on Windows doesn't work --- tests/integration/cli/tests/run.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/cli/tests/run.rs b/tests/integration/cli/tests/run.rs index ac0815632e7..56fd8178d88 100644 --- a/tests/integration/cli/tests/run.rs +++ b/tests/integration/cli/tests/run.rs @@ -100,6 +100,8 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { output.arg("-o"); output.arg(python_wasmer_path.clone()); output.arg(format!("--{c}")); + output.arg("--debug-dir"); + output.arg(format!("{t}-{c}")); if t.contains("x86_64") && *c == "singlepass" { output.arg("-m"); @@ -111,6 +113,8 @@ fn test_cross_compile_python_windows() -> anyhow::Result<()> { output.arg(t); } + println!("command {:?}", output); + let output = output.output()?; let stdout = std::str::from_utf8(&output.stdout) From 4fe57188aabcad1935142485cc46b46c8597ed5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20Sch=C3=BCtt?= Date: Thu, 26 Jan 2023 10:39:02 +0100 Subject: [PATCH 18/18] Add winsdk folder to package-capi --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a7c1d1b3d72..a0454820fb6 100644 --- a/Makefile +++ b/Makefile @@ -620,6 +620,7 @@ endif package-capi: mkdir -p "package/include" mkdir -p "package/lib" + mkdir -p "package/winsdk" cp lib/c-api/wasmer.h* package/include cp lib/c-api/wasmer_wasm.h* package/include cp lib/c-api/tests/wasm-c-api/include/wasm.h* package/include @@ -730,7 +731,7 @@ package: package-wasmer package-minimal-headless-wasmer package-capi tar-capi: ls -R package - tar -C package -zcvf build-capi.tar.gz lib include + tar -C package -zcvf build-capi.tar.gz lib include winsdk untar-capi: mkdir -p package