Zombienet smoke is flaky or what?

.github/workflows/test.yml

@@ -192,9 +192,85 @@ jobs:
           RUSTFLAGS: "-C debug-assertions -D warnings"
           SKIP_WASM_BUILD: 1
 
+  # Preparation for zombienet smoking
+  zombienet-smoke-setup:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Install updates and dependencies
+        run: .github/install-deps.sh
+
+      - name: Set rust version via common env file
+        run: cat .github/env >> $GITHUB_ENV
+
+      - name: Install stable toolchain
+        uses: dtolnay/rust-toolchain@master
+        with:
+          targets: "wasm32-unknown-unknown"
+          components: "rust-src"
+          toolchain: "${{env.RUST_STABLE_VERSION}}"
+
+      - name: Fetch cache
+        uses: Swatinem/rust-cache@f0deed1e0edfc6a9be95417288c0e1099b1eeec3 # v2.7.7
+        with:
+          shared-key: "fellowship-cache-zombienet-tests"
+          save-if: ${{ github.ref == 'refs/heads/main' }}
+
+      - name: Build chain-spec-generator
+        run: |
+          cargo build -p chain-spec-generator --no-default-features --features fast-runtime,polkadot,coretime-polkadot --locked
+
+      - name: Upload chain-spec-generator
+        uses: actions/upload-artifact@v4
+        with:
+          name: chain-spec-generator-for-smoke
+          path: target/debug/chain-spec-generator
+
+      # Define supported Polkadot versions
+      - name: Download node binaries
+        shell: bash
+        run: |
+          POLKADOT_VERSIONS=("polkadot-stable2412-4" "polkadot-stable2503")
+          NODE_BINARIES=("polkadot" "polkadot-execute-worker" "polkadot-prepare-worker")
+          PARACHAIN_BINARIES=("polkadot-parachain")
+          ALL_BINARIES=("${NODE_BINARIES[@]}" "${PARACHAIN_BINARIES[@]}")
+          mkdir -p ./node-binaries
+          for polkadot_version in "${POLKADOT_VERSIONS[@]}"; do
+            for binary in "${ALL_BINARIES[@]}"; do
+              echo "Downloading $binary for $polkadot_version"
+              curl -sL https://github.com/paritytech/polkadot-sdk/releases/download/$polkadot_version/${binary} -o ./node-binaries/${binary}-$polkadot_version
+              chmod +x ./node-binaries/${binary}-$polkadot_version
+            done
+          done
+          ls -lrt ./node-binaries
+
+      - name: Upload node binaries to cache
+        uses: actions/upload-artifact@v4
+        with:
+          name: node-binaries-for-smoke
+          path: node-binaries
+
   # Job required by "confirmTestPassed"
   zombienet-smoke:
+    needs: [zombienet-smoke-setup]
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        docker:
+          - polkadot: polkadot-stable2412-4
+            cumulus: polkadot-stable2412-4
+            can_fail: false
+          - polkadot: polkadot-stable2412-4
+            cumulus: polkadot-stable2503
+            can_fail: false
+          - polkadot: polkadot-stable2503
+            cumulus: polkadot-stable2503
+            can_fail: false
+          - polkadot: polkadot-stable2503
+            cumulus: polkadot-stable2412-4
+            can_fail: false
     steps:
       - name: Cancel previous runs
         uses: styfle/cancel-workflow-action@b173b6ec0100793626c2d9e6b90435061f4fc3e5 # v0.11.0
@@ -223,26 +299,51 @@ jobs:
           shared-key: "fellowship-cache-zombienet-tests"
           save-if: ${{ github.ref == 'refs/heads/main' }}
 
-      - name: Build
-        run: |
-          cargo build -p chain-spec-generator --no-default-features --features fast-runtime,polkadot,coretime-polkadot --release --locked
+      - name: Download binary
+        uses: actions/download-artifact@v4
+        with:
+          name: chain-spec-generator-for-smoke
+          path: ./for-zombienet-smoke
 
-      - name: Wait for Docker to Start
-        run: |
-          timeout 30 sh -c 'until docker info; do sleep 1; done'
-          docker --version
+      - name: Download node binaries
+        uses: actions/download-artifact@v4
+        with:
+          name: node-binaries-for-smoke
+          path: ./node-binaries
 
-      - name: Zombienet smoke test
+      - name: Zombienet smoke test for polkadot-${{ matrix.docker.polkadot }} with polkadot-parachain-${{ matrix.docker.cumulus }})
         timeout-minutes: 20
+        continue-on-error: ${{ matrix.docker.can_fail }}
         run: |
-          export PATH=$(pwd)/target/release:$PATH
+          # Add binaries to PATH
+          export PATH=$(pwd)/for-zombienet-smoke:$PATH
+          export PATH=$(pwd)/node-binaries:$PATH
+
+          chmod +x ./for-zombienet-smoke/chain-spec-generator
+          # Ensure a matrix version for PATH
+          # WIP: externalize
+          NODE_BINARIES=("polkadot" "polkadot-execute-worker" "polkadot-prepare-worker")
+          PARACHAIN_BINARIES=("polkadot-parachain")
+          for binary in "${NODE_BINARIES[@]}"; do
+              cp ./node-binaries/${binary}-${{ matrix.docker.polkadot }} ./node-binaries/${binary}
+              chmod +x ./node-binaries/${binary}
+              file ./node-binaries/${binary}
+          done
+          for binary in "${PARACHAIN_BINARIES[@]}"; do
+              cp ./node-binaries/${binary}-${{ matrix.docker.cumulus }} ./node-binaries/${binary}
+              chmod +x ./node-binaries/${binary}
+              file ./node-binaries/${binary}
+          done
+          ls -lrt ./node-binaries
+
+          export ZOMBIE_PROVIDER="Native"
           cargo test --manifest-path integration-tests/zombienet/Cargo.toml
 
   build-runtimes:
     needs: [ runtime-matrix ]
     runs-on: ubuntu-latest
     strategy:
-      # Ensure the other jobs are continue
+      # Ensure the other jobs are continued
       fail-fast: false
       matrix:
         runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }}
@@ -331,7 +432,7 @@ jobs:
           command: cd ecosystem-tests && yarn test
 
   # This will only run if all the tests in its "needs" array passed.
-  # Add this as your required job, becuase if the matrix changes size (new things get added)
+  # Add this as your required job, because if the matrix changes size (new things get added),
   # it will still require all the steps to succeed.
   # If you add more jobs, remember to add them to the "needs" array.
   confirmTestPassed:

Cargo.toml

@@ -61,6 +61,7 @@ encointer-balances-tx-payment-rpc-runtime-api = { version = "~15.1.0", default-f
 encointer-kusama-runtime = { path = "system-parachains/encointer" }
 encointer-primitives = { version = "~15.4.0", default-features = false }
 enumflags2 = { version = "0.7.7" }
+env_logger = { version = "0.10.2" }
 frame-benchmarking = { version = "39.0.0", default-features = false }
 frame-election-provider-support = { version = "39.0.0", default-features = false }
 frame-executive = { version = "39.1.0", default-features = false }

integration-tests/zombienet/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 [dependencies]
 subxt = { features = ["native"], workspace = true }
 tokio = { workspace = true }
-tracing-subscriber = { workspace = true }
+env_logger = { workspace = true }
 zombienet-sdk = { workspace = true }
 log = { workspace = true, default-features = true }
 anyhow = { workspace = true }

integration-tests/zombienet/src/lib.rs

@@ -1,10 +1,6 @@
 use anyhow::anyhow;
 use zombienet_sdk::{NetworkConfig, NetworkConfigBuilder};
 
-pub mod environment;
-
-pub type Error = Box<dyn std::error::Error>;
-
 // Chain generator command template
 const CMD_TPL: &str = "chain-spec-generator {{chainName}}";
 
@@ -14,8 +10,8 @@ const BOB: &str = "bob";
 // Collator
 const COLLATOR: &str = "collator";
 
-pub fn small_network() -> Result<NetworkConfig, Error> {
-	let images = environment::get_images_from_env();
+pub fn small_network() -> Result<NetworkConfig, anyhow::Error> {
+	let images = zombienet_sdk::environment::get_images_from_env();
 	let config = NetworkConfigBuilder::new()
 		.with_relaychain(|r| {
 			r.with_chain("polkadot-local")

integration-tests/zombienet/tests/smoke.rs

@@ -1,16 +1,22 @@
 use std::time::Instant;
 use subxt::{ext::futures::StreamExt, OnlineClient, PolkadotConfig};
-use zombienet_sdk_tests::{
-	environment::{get_images_from_env, get_provider_from_env, get_spawn_fn, Provider},
-	small_network,
-};
+use zombienet_sdk_tests::small_network;
 
 fn dump_provider_and_versions() {
-	let provider = get_provider_from_env();
-	log::info!("Using zombienet provider: {:?}", provider);
+	use zombienet_sdk::environment::Provider;
+	let provider = zombienet_sdk::environment::get_provider_from_env();
+	log::info!(
+		"Using zombienet provider: {:?}",
+		match provider {
+			Provider::Native => "Native",
+			Provider::K8s => "K8s",
+			Provider::Docker => "Docker",
+		}
+	);
 
 	if let Provider::Docker = provider {
-		let images = get_images_from_env();
+		let images = zombienet_sdk::environment::get_images_from_env();
+		log::info!("Using Docker images: {:?}", images);
 
 		for image in [images.polkadot, images.cumulus] {
 			let output = std::process::Command::new("docker")
@@ -33,16 +39,18 @@ fn dump_provider_and_versions() {
 
 #[tokio::test(flavor = "multi_thread")]
 async fn smoke() -> Result<(), anyhow::Error> {
-	tracing_subscriber::fmt::init();
+	let _ = env_logger::try_init_from_env(
+		env_logger::Env::default().filter_or(env_logger::DEFAULT_FILTER_ENV, "info"),
+	);
 
 	// config and env
 	dump_provider_and_versions();
-	let spawn_fn = get_spawn_fn();
-	let config = small_network().unwrap();
+	let spawn_fn = zombienet_sdk::environment::get_spawn_fn();
+	let config = small_network()?;
 
 	// spawn
 	let now = Instant::now();
-	let network = spawn_fn(config).await.unwrap();
+	let network = spawn_fn(config).await?;
 	let elapsed = now.elapsed();
 	log::info!("🚀🚀🚀🚀 network deployed in {:.2?}", elapsed);