From de28dd4a3500ad51798313856f5398d91ccfb827 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 27 Sep 2023 12:31:10 +0800 Subject: [PATCH 1/7] Preparing the repo for use with "cargo nextest" --- .config/nextest.toml | 19 +++++++++++++++++++ .github/workflows/test.yaml | 7 +++++++ lib/virtual-net/src/tests.rs | 24 +++++++++++------------- 3 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 .config/nextest.toml diff --git a/.config/nextest.toml b/.config/nextest.toml new file mode 100644 index 00000000000..9f25c6ddb8a --- /dev/null +++ b/.config/nextest.toml @@ -0,0 +1,19 @@ +[profile.ci] +# Print out output for failing tests as soon as they fail, and also at the end +# of the run (for easy scrollability). +failure-output = "immediate-final" +# Do not cancel the test run on the first failure. +fail-fast = false +# Automatically mark a test as "slow" after 60 seconds, then kill it after 180s +slow-timeout = { period = "60s", terminate-after = 3 } + +[[profile.default.overrides]] +# Create-exe tests tend to be pretty flaky +filter = "test(create_exe)" +retries = 3 + +[[profile.default.overrides]] +# Snapshot tests tend to be flaky because they implicitly rely on how the OS will +# schedule threads. Our pipe flushing is also pretty unreliable. +filter = "test(snapshot)" +retries = 3 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c24e126239f..833bc65fdfe 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -25,6 +25,7 @@ env: # can override that behaviour CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git MSRV: "1.70" + NEXTEST_PROFILE: "ci" jobs: @@ -371,6 +372,8 @@ jobs: with: toolchain: "nightly-2023-05-25" target: ${{ matrix.metadata.target }} + - name: Install Nextest + uses: taiki-e/install-action@nextest - name: Install Windows-GNU linker if: ${{ matrix.metadata.build == 'windows-gnu' }} shell: bash @@ -595,6 +598,8 @@ jobs: with: toolchain: ${{ env.MSRV }} target: ${{ matrix.metadata.target }} + - name: Install Nextest + uses: taiki-e/install-action@nextest - name: Install LLVM (macOS Apple Silicon) if: matrix.metadata.os == 'macos-11.0' && !matrix.metadata.llvm_url run: | @@ -696,6 +701,8 @@ jobs: with: toolchain: ${{ env.MSRV }} target: ${{ matrix.metadata.target }} + - name: Install Nextest + uses: taiki-e/install-action@nextest - name: Cache uses: whywaita/actions-cache-s3@v2 with: diff --git a/lib/virtual-net/src/tests.rs b/lib/virtual-net/src/tests.rs index 960a4d2cf8b..6fa4ebd7b0c 100644 --- a/lib/virtual-net/src/tests.rs +++ b/lib/virtual-net/src/tests.rs @@ -71,19 +71,20 @@ async fn setup_pipe( #[cfg(feature = "remote")] async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServer) { - static PORT: AtomicU16 = AtomicU16::new(8000); - let addr = SocketAddr::V4(SocketAddrV4::new( - Ipv4Addr::LOCALHOST, - PORT.fetch_add(1, Ordering::SeqCst), - )); - tracing::info!("listening on {addr}"); let mut listener = client - .listen_tcp(addr.clone(), false, false, false) + .listen_tcp( + SocketAddr::from((Ipv4Addr::LOCALHOST, 0)), + false, + false, + false, + ) .await .unwrap(); + let addr = listener.addr_local().unwrap(); + tracing::info!("listening on {addr}"); - const TEST1: &'static str = "the cat ran up the wall!"; - const TEST2: &'static str = "...and fell off the roof! raise the roof! oop oop"; + const TEST1: &str = "the cat ran up the wall!"; + const TEST2: &str = "...and fell off the roof! raise the roof! oop oop"; tracing::info!("spawning acceptor worker thread"); tokio::task::spawn(async move { @@ -104,10 +105,7 @@ async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServe tracing::info!("connecting to listening socket"); let mut socket = client - .connect_tcp( - SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0)), - addr, - ) + .connect_tcp(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)), addr) .await .unwrap(); From 2db4f79be7ce8acbdeaa43fbeca03c7b8f2fcf3c Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 3 Oct 2023 14:26:39 +0800 Subject: [PATCH 2/7] Use nextest for the CLI integration tests --- Makefile | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1abafb8bfce..b49df857751 100644 --- a/Makefile +++ b/Makefile @@ -637,9 +637,9 @@ test-integration-cli: build-wasmer build-capi package-capi-headless package dist WASMER_DIR=`pwd`/package $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner --no-fail-fast -p wasmer-integration-tests-cli --locked # Before running this in the CI, we need to set up link.tar.gz and /cache/wasmer-[target].tar.gz -test-integration-cli-ci: +test-integration-cli-ci: require-nextest rustup target add wasm32-wasi - $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner -p wasmer-integration-tests-cli --locked + $(CARGO_BINARY) nextest run $(CARGO_TARGET_FLAG) --features webc_runner -p wasmer-integration-tests-cli --locked test-integration-ios: $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner -p wasmer-integration-tests-ios --locked @@ -940,3 +940,6 @@ test-minimal-versions: update-graphql-schema: curl -sSfL https://registry.wapm.io/graphql/schema.graphql > lib/registry/graphql/schema.graphql + +require-nextest: + cargo nextest --version > /dev/null || cargo binstall cargo-nextest --secure || cargo install cargo-nextest From ab065ff1a1781f6221260ae2f3fe4c3b29fe2d74 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 3 Oct 2023 15:49:40 +0800 Subject: [PATCH 3/7] Remove the filtering and just retry everything in CI --- .config/nextest.toml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.config/nextest.toml b/.config/nextest.toml index 9f25c6ddb8a..3d309082956 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -6,14 +6,5 @@ failure-output = "immediate-final" fail-fast = false # Automatically mark a test as "slow" after 60 seconds, then kill it after 180s slow-timeout = { period = "60s", terminate-after = 3 } - -[[profile.default.overrides]] -# Create-exe tests tend to be pretty flaky -filter = "test(create_exe)" -retries = 3 - -[[profile.default.overrides]] -# Snapshot tests tend to be flaky because they implicitly rely on how the OS will -# schedule threads. Our pipe flushing is also pretty unreliable. -filter = "test(snapshot)" +# Retry a couple times in case there are flaky tests retries = 3 From 68db558e3c2f67faf64810756b2f4fbf25c37d15 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Tue, 3 Oct 2023 15:50:26 +0800 Subject: [PATCH 4/7] Move the WAST tests over to nextest --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b49df857751..349e771a9d0 100644 --- a/Makefile +++ b/Makefile @@ -538,7 +538,7 @@ build-capi-headless-ios: # test compilers test-stage-0-wast: - $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --release --tests $(compiler_features) --locked + $(CARGO_BINARY) nextest run $(CARGO_TARGET_FLAG) --release $(compiler_features) --locked # test packages test-stage-1-test-all: From b0f98f608b3ec2af37b11fd3ad15ed4e96478385 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Wed, 4 Oct 2023 15:04:04 +0800 Subject: [PATCH 5/7] Use nextest when running unit tests for the main set of crates --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 349e771a9d0..7ed75500017 100644 --- a/Makefile +++ b/Makefile @@ -542,7 +542,8 @@ test-stage-0-wast: # test packages test-stage-1-test-all: - $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --all --release $(exclude_tests) --exclude wasmer-c-api-test-runner --exclude wasmer-capi-examples-runner $(compiler_features) --locked + $(CARGO_BINARY) nextest run $(CARGO_TARGET_FLAG) --workspace --release $(exclude_tests) --exclude wasmer-c-api-test-runner --exclude wasmer-capi-examples-runner $(compiler_features) --locked + $(CARGO_BINARY) test --doc $(CARGO_TARGET_FLAG) --workspace --release $(exclude_tests) --exclude wasmer-c-api-test-runner --exclude wasmer-capi-examples-runner $(compiler_features) --locked test-stage-2-test-compiler-cranelift-nostd: $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --manifest-path lib/compiler-cranelift/Cargo.toml --release --no-default-features --features=std --locked test-stage-3-test-compiler-singlepass-nostd: From bbe7246890624a2656b19bacafe2e469c91c52e0 Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 8 Dec 2023 20:44:58 +0800 Subject: [PATCH 6/7] Switch tests over to the multi-threaded runtime to avoid deadlocks --- lib/virtual-net/Cargo.toml | 2 +- lib/virtual-net/src/tests.rs | 29 +++++++++-------------------- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/lib/virtual-net/Cargo.toml b/lib/virtual-net/Cargo.toml index e0d51b1fe22..49a1a6deb19 100644 --- a/lib/virtual-net/Cargo.toml +++ b/lib/virtual-net/Cargo.toml @@ -33,7 +33,7 @@ hyper = { version = "0.14", optional = true } tokio-tungstenite = { version = "0.20", optional = true } [dev-dependencies] -tokio = { version = "1", default_features = false, features = [ "macros" ] } +tokio = { version = "1", default_features = false, features = [ "macros", "rt-multi-thread" ] } tracing-test = { version = "0.2" } [features] diff --git a/lib/virtual-net/src/tests.rs b/lib/virtual-net/src/tests.rs index 6fa4ebd7b0c..944d57c5495 100644 --- a/lib/virtual-net/src/tests.rs +++ b/lib/virtual-net/src/tests.rs @@ -123,27 +123,24 @@ async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServe } #[cfg(feature = "remote")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_mpsc() { let (client, server) = setup_mpsc().await; test_tcp(client, server).await } #[cfg(feature = "remote")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_small_pipe_using_bincode() { let (client, server) = setup_pipe(10, FrameSerializationFormat::Bincode).await; test_tcp(client, server).await } #[cfg(feature = "remote")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_large_pipe_using_bincode() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Bincode).await; test_tcp(client, server).await @@ -151,9 +148,8 @@ async fn test_tcp_with_large_pipe_using_bincode() { #[cfg(feature = "remote")] #[cfg(feature = "json")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_small_pipe_using_json() { let (client, server) = setup_pipe(10, FrameSerializationFormat::Json).await; test_tcp(client, server).await @@ -161,9 +157,8 @@ async fn test_tcp_with_small_pipe_using_json() { #[cfg(feature = "remote")] #[cfg(feature = "json")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_large_pipe_json_using_json() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Json).await; test_tcp(client, server).await @@ -171,9 +166,8 @@ async fn test_tcp_with_large_pipe_json_using_json() { #[cfg(feature = "remote")] #[cfg(feature = "messagepack")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_small_pipe_using_messagepack() { let (client, server) = setup_pipe(10, FrameSerializationFormat::MessagePack).await; test_tcp(client, server).await @@ -181,9 +175,8 @@ async fn test_tcp_with_small_pipe_using_messagepack() { #[cfg(feature = "remote")] #[cfg(feature = "messagepack")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_large_pipe_json_using_messagepack() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::MessagePack).await; test_tcp(client, server).await @@ -191,9 +184,8 @@ async fn test_tcp_with_large_pipe_json_using_messagepack() { #[cfg(feature = "remote")] #[cfg(feature = "cbor")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_small_pipe_using_cbor() { let (client, server) = setup_pipe(10, FrameSerializationFormat::Cbor).await; test_tcp(client, server).await @@ -201,15 +193,13 @@ async fn test_tcp_with_small_pipe_using_cbor() { #[cfg(feature = "remote")] #[cfg(feature = "cbor")] -#[cfg(target_os = "linux")] #[traced_test] -#[tokio::test] +#[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_large_pipe_json_using_cbor() { let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Cbor).await; test_tcp(client, server).await } -#[cfg(target_os = "linux")] #[traced_test] #[tokio::test] async fn test_google_poll() { @@ -300,7 +290,6 @@ async fn test_google_poll() { tracing::info!("done"); } -#[cfg(target_os = "linux")] #[traced_test] #[tokio::test] async fn test_google_epoll() { From f163071fa26647b5d4fa3e02ad0be1b7757ad84d Mon Sep 17 00:00:00 2001 From: Michael-F-Bryan Date: Fri, 8 Dec 2023 21:38:52 +0800 Subject: [PATCH 7/7] Looks like the virtual-net tests don't work on Windows --- lib/virtual-net/src/tests.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/lib/virtual-net/src/tests.rs b/lib/virtual-net/src/tests.rs index 944d57c5495..576c9cba029 100644 --- a/lib/virtual-net/src/tests.rs +++ b/lib/virtual-net/src/tests.rs @@ -123,6 +123,7 @@ async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServe } #[cfg(feature = "remote")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_mpsc() { @@ -131,6 +132,7 @@ async fn test_tcp_with_mpsc() { } #[cfg(feature = "remote")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_small_pipe_using_bincode() { @@ -139,6 +141,7 @@ async fn test_tcp_with_small_pipe_using_bincode() { } #[cfg(feature = "remote")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_large_pipe_using_bincode() { @@ -148,6 +151,7 @@ async fn test_tcp_with_large_pipe_using_bincode() { #[cfg(feature = "remote")] #[cfg(feature = "json")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_small_pipe_using_json() { @@ -157,6 +161,7 @@ async fn test_tcp_with_small_pipe_using_json() { #[cfg(feature = "remote")] #[cfg(feature = "json")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_large_pipe_json_using_json() { @@ -166,6 +171,7 @@ async fn test_tcp_with_large_pipe_json_using_json() { #[cfg(feature = "remote")] #[cfg(feature = "messagepack")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_small_pipe_using_messagepack() { @@ -175,6 +181,7 @@ async fn test_tcp_with_small_pipe_using_messagepack() { #[cfg(feature = "remote")] #[cfg(feature = "messagepack")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_large_pipe_json_using_messagepack() { @@ -184,6 +191,7 @@ async fn test_tcp_with_large_pipe_json_using_messagepack() { #[cfg(feature = "remote")] #[cfg(feature = "cbor")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_small_pipe_using_cbor() { @@ -193,6 +201,7 @@ async fn test_tcp_with_small_pipe_using_cbor() { #[cfg(feature = "remote")] #[cfg(feature = "cbor")] +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test(flavor = "multi_thread")] async fn test_tcp_with_large_pipe_json_using_cbor() { @@ -200,6 +209,7 @@ async fn test_tcp_with_large_pipe_json_using_cbor() { test_tcp(client, server).await } +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test] async fn test_google_poll() { @@ -290,6 +300,7 @@ async fn test_google_poll() { tracing::info!("done"); } +#[cfg_attr(windows, ignore)] #[traced_test] #[tokio::test] async fn test_google_epoll() {