Merge pull request #4229 from wasmerio/nextest

Use `cargo nextest` when running tests in CI

.config/nextest.toml

@@ -0,0 +1,10 @@
+[profile.ci]
+# Print out output for failing tests as soon as they fail, and also at the end
+# of the run (for easy scrollability).
+failure-output = "immediate-final"
+# Do not cancel the test run on the first failure.
+fail-fast = false
+# Automatically mark a test as "slow" after 60 seconds, then kill it after 180s
+slow-timeout = { period = "60s", terminate-after = 3 }
+# Retry a couple times in case there are flaky tests
+retries = 3

.github/workflows/test.yaml

@@ -25,6 +25,7 @@ env:
   # can override that behaviour
   CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git
   MSRV: "1.70"
+  NEXTEST_PROFILE: "ci"
 
 jobs:
 
@@ -371,6 +372,8 @@ jobs:
         with:
           toolchain: "nightly-2023-05-25"
           target: ${{ matrix.metadata.target }}
+      - name: Install Nextest
+        uses: taiki-e/install-action@nextest
       - name: Install Windows-GNU linker
         if: ${{ matrix.metadata.build == 'windows-gnu' }}
         shell: bash
@@ -595,6 +598,8 @@ jobs:
         with:
           toolchain: ${{ env.MSRV }}
           target: ${{ matrix.metadata.target }}
+      - name: Install Nextest
+        uses: taiki-e/install-action@nextest
       - name: Install LLVM (macOS Apple Silicon)
         if: matrix.metadata.os == 'macos-11.0' && !matrix.metadata.llvm_url
         run: |
@@ -696,6 +701,8 @@ jobs:
         with:
           toolchain: ${{ env.MSRV }}
           target: ${{ matrix.metadata.target }}
+      - name: Install Nextest
+        uses: taiki-e/install-action@nextest
       - name: Cache
         uses: whywaita/actions-cache-s3@v2
         with:

Makefile

@@ -538,11 +538,12 @@ build-capi-headless-ios:
 
 # test compilers
 test-stage-0-wast:
-	$(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --release --tests $(compiler_features) --locked
+	$(CARGO_BINARY) nextest run $(CARGO_TARGET_FLAG) --release $(compiler_features) --locked
 
 # test packages
 test-stage-1-test-all:
-	$(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --all --release $(exclude_tests) --exclude wasmer-c-api-test-runner --exclude wasmer-capi-examples-runner $(compiler_features) --locked
+	$(CARGO_BINARY) nextest run $(CARGO_TARGET_FLAG) --workspace --release $(exclude_tests) --exclude wasmer-c-api-test-runner --exclude wasmer-capi-examples-runner $(compiler_features) --locked
+	$(CARGO_BINARY) test --doc $(CARGO_TARGET_FLAG) --workspace --release $(exclude_tests) --exclude wasmer-c-api-test-runner --exclude wasmer-capi-examples-runner $(compiler_features) --locked
 test-stage-2-test-compiler-cranelift-nostd:
 	$(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --manifest-path lib/compiler-cranelift/Cargo.toml --release --no-default-features --features=std --locked
 test-stage-3-test-compiler-singlepass-nostd:
@@ -637,9 +638,9 @@ test-integration-cli: build-wasmer build-capi package-capi-headless package dist
 	WASMER_DIR=`pwd`/package $(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner --no-fail-fast -p wasmer-integration-tests-cli --locked
 
 # Before running this in the CI, we need to set up link.tar.gz and /cache/wasmer-[target].tar.gz
-test-integration-cli-ci:
+test-integration-cli-ci: require-nextest
 	rustup target add wasm32-wasi
-	$(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner -p wasmer-integration-tests-cli --locked
+	$(CARGO_BINARY) nextest run $(CARGO_TARGET_FLAG) --features webc_runner -p wasmer-integration-tests-cli --locked
 
 test-integration-ios:
 	$(CARGO_BINARY) test $(CARGO_TARGET_FLAG) --features webc_runner -p wasmer-integration-tests-ios --locked
@@ -940,3 +941,6 @@ test-minimal-versions:
 
 update-graphql-schema:
 	curl -sSfL https://registry.wapm.io/graphql/schema.graphql > lib/registry/graphql/schema.graphql
+
+require-nextest:
+	cargo nextest --version > /dev/null || cargo binstall cargo-nextest --secure || cargo install cargo-nextest

lib/virtual-net/Cargo.toml

@@ -33,7 +33,7 @@ hyper = { version = "0.14", optional = true }
 tokio-tungstenite = { version = "0.20", optional = true }
 
 [dev-dependencies]
-tokio = { version = "1", default_features = false, features = [ "macros" ] }
+tokio = { version = "1", default_features = false, features = [ "macros", "rt-multi-thread" ] }
 tracing-test = { version = "0.2" }
 
 [features]

lib/virtual-net/src/tests.rs

@@ -71,19 +71,20 @@ async fn setup_pipe(
 
 #[cfg(feature = "remote")]
 async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServer) {
-    static PORT: AtomicU16 = AtomicU16::new(8000);
-    let addr = SocketAddr::V4(SocketAddrV4::new(
-        Ipv4Addr::LOCALHOST,
-        PORT.fetch_add(1, Ordering::SeqCst),
-    ));
-    tracing::info!("listening on {addr}");
     let mut listener = client
-        .listen_tcp(addr.clone(), false, false, false)
+        .listen_tcp(
+            SocketAddr::from((Ipv4Addr::LOCALHOST, 0)),
+            false,
+            false,
+            false,
+        )
         .await
         .unwrap();
+    let addr = listener.addr_local().unwrap();
+    tracing::info!("listening on {addr}");
 
-    const TEST1: &'static str = "the cat ran up the wall!";
-    const TEST2: &'static str = "...and fell off the roof! raise the roof! oop oop";
+    const TEST1: &str = "the cat ran up the wall!";
+    const TEST2: &str = "...and fell off the roof! raise the roof! oop oop";
 
     tracing::info!("spawning acceptor worker thread");
     tokio::task::spawn(async move {
@@ -104,10 +105,7 @@ async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServe
 
     tracing::info!("connecting to listening socket");
     let mut socket = client
-        .connect_tcp(
-            SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0)),
-            addr,
-        )
+        .connect_tcp(SocketAddr::from((Ipv4Addr::LOCALHOST, 0)), addr)
         .await
         .unwrap();
 
@@ -125,93 +123,93 @@ async fn test_tcp(client: RemoteNetworkingClient, _server: RemoteNetworkingServe
 }
 
 #[cfg(feature = "remote")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_mpsc() {
     let (client, server) = setup_mpsc().await;
     test_tcp(client, server).await
 }
 
 #[cfg(feature = "remote")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_small_pipe_using_bincode() {
     let (client, server) = setup_pipe(10, FrameSerializationFormat::Bincode).await;
     test_tcp(client, server).await
 }
 
 #[cfg(feature = "remote")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_large_pipe_using_bincode() {
     let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Bincode).await;
     test_tcp(client, server).await
 }
 
 #[cfg(feature = "remote")]
 #[cfg(feature = "json")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_small_pipe_using_json() {
     let (client, server) = setup_pipe(10, FrameSerializationFormat::Json).await;
     test_tcp(client, server).await
 }
 
 #[cfg(feature = "remote")]
 #[cfg(feature = "json")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_large_pipe_json_using_json() {
     let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Json).await;
     test_tcp(client, server).await
 }
 
 #[cfg(feature = "remote")]
 #[cfg(feature = "messagepack")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_small_pipe_using_messagepack() {
     let (client, server) = setup_pipe(10, FrameSerializationFormat::MessagePack).await;
     test_tcp(client, server).await
 }
 
 #[cfg(feature = "remote")]
 #[cfg(feature = "messagepack")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_large_pipe_json_using_messagepack() {
     let (client, server) = setup_pipe(1024000, FrameSerializationFormat::MessagePack).await;
     test_tcp(client, server).await
 }
 
 #[cfg(feature = "remote")]
 #[cfg(feature = "cbor")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_small_pipe_using_cbor() {
     let (client, server) = setup_pipe(10, FrameSerializationFormat::Cbor).await;
     test_tcp(client, server).await
 }
 
 #[cfg(feature = "remote")]
 #[cfg(feature = "cbor")]
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
-#[tokio::test]
+#[tokio::test(flavor = "multi_thread")]
 async fn test_tcp_with_large_pipe_json_using_cbor() {
     let (client, server) = setup_pipe(1024000, FrameSerializationFormat::Cbor).await;
     test_tcp(client, server).await
 }
 
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
 #[tokio::test]
 async fn test_google_poll() {
@@ -302,7 +300,7 @@ async fn test_google_poll() {
     tracing::info!("done");
 }
 
-#[cfg(target_os = "linux")]
+#[cfg_attr(windows, ignore)]
 #[traced_test]
 #[tokio::test]
 async fn test_google_epoll() {