Authorities are staking finalization stall

.github/actions/run-e2e-test/action.yml

@@ -91,7 +91,7 @@ runs:
 
         RESERVED_SEATS="${{ inputs.reserved-seats }}"
         NON_RESERVED_SEATS="${{ inputs.non-reserved-seats }}"
-
+        
         if [[ -n "${RANDOMIZED}" ]]; then
           ARGS+=(-r "${RANDOMIZED}")
         fi

.github/scripts/run_consensus.sh

@@ -3,8 +3,10 @@
 set -euo pipefail
 
 # default node count
-# change when increasing the number of node containers
 NODE_COUNT=5
+# max node count that will not crash current GH machines
+MAX_NODE_COUNT=6
+# default minimum validator count
 MIN_VALIDATOR_COUNT=4
 DOCKER_COMPOSE=${DOCKER_COMPOSE:-"docker/docker-compose.yml"}
 OVERRIDE_DOCKER_COMPOSE=${OVERRIDE_DOCKER_COMPOSE:-""}
@@ -51,6 +53,11 @@ done
 
 export NODE_COUNT
 
+if [[ ${NODE_COUNT} -gt ${MAX_NODE_COUNT} ]]; then
+  echo "Tried to run ${NODE_COUNT} nodes. Max node count allowed: ${MAX_NODE_COUNT}."
+  exit 1
+fi
+
 function generate_authorities {
   local authorities_count="$1"
 

.github/workflows/nightly-pipeline.yaml

@@ -170,7 +170,8 @@ jobs:
         uses: ./.github/actions/run-e2e-test
         with:
           test-case: authorities_are_staking
-          randomized: true
+          node-count: 6
+          follow-up-finalization-check: true
         timeout-minutes: 60
 
   run-e2e-high-out-latency:

docker/docker-compose.base.yml

@@ -77,3 +77,19 @@ services:
       # key derived from "//4"
       - BASE_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW
       - NODE_KEY_PATH=/data/5DiDShBWa1fQx6gLzpf3SFBhMinCoyvHM1BWjPNsmXS8hkrW/p2p_secret
+
+  Node5:
+    extends:
+      file: common.yml
+      service: AlephNonBootNode
+    container_name: Node5
+    environment:
+      - RPC_PORT=9938
+      - WS_PORT=9948
+      - PORT=30338
+      - VALIDATOR_PORT=30348
+      - PUBLIC_VALIDATOR_ADDRESS=127.0.0.1:30348
+      - NAME=Node5
+      # key derived from "//5"
+      - BASE_PATH=/data/5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR
+      - NODE_KEY_PATH=/data/5EFb84yH9tpcFuiKUcsmdoF7xeeY3ajG1ZLQimxQoFt9HMKR/p2p_secret

docker/docker-compose.bridged.yml

@@ -53,6 +53,17 @@ services:
       - PUBLIC_VALIDATOR_ADDRESS=Node4:30347
       - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
 
+  Node5:
+    extends:
+      file: docker-compose.base.yml
+      service: Node5
+    networks:
+      - main
+      - Node5
+    environment:
+      - PUBLIC_VALIDATOR_ADDRESS=Node4:30348
+      - BOOT_NODES=/dns4/Node0/tcp/30333/p2p/$BOOTNODE_PEER_ID
+
 networks:
   main:
     name: main-network
@@ -66,3 +77,5 @@ networks:
     name: Node3-network
   Node4:
     name: Node4-network
+  Node5:
+    name: Node5-network

docker/docker-compose.yml

@@ -30,3 +30,9 @@ services:
       file: docker-compose.base.yml
       service: Node4
     network_mode: host
+
+  Node5:
+    extends:
+      file: docker-compose.base.yml
+      service: Node5
+    network_mode: host

e2e-tests/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "aleph-e2e-client"
-version = "0.10.0"
+version = "0.10.1"
 edition = "2021"
 license = "Apache 2.0"
 

e2e-tests/src/config.rs

@@ -86,7 +86,7 @@ impl Config {
         RootConnection::new(&self.node, sudo_keypair).await.unwrap()
     }
 
-    pub fn validator_names<'a>(&'a self) -> Vec<String> {
+    pub fn validator_names(&self) -> Vec<String> {
         (0..self.validator_count)
             .map(|id| format!("Node{}", id))
             .collect()

e2e-tests/src/test/electing_validators.rs

@@ -15,7 +15,7 @@ use primitives::EraIndex;
 
 use crate::{
     config::setup_test,
-    validators::{prepare_validators, setup_accounts},
+    validators::{get_controller_connections_to_nodes, prepare_validators, setup_accounts},
 };
 
 /// Verify that `pallet_staking::ErasStakers` contains all target validators.
@@ -179,7 +179,10 @@ pub async fn authorities_are_staking() -> anyhow::Result<()> {
 
     let desired_validator_count = reserved_seats + non_reserved_seats;
     let accounts = setup_accounts(desired_validator_count);
-    prepare_validators(&root_connection, node, &accounts).await?;
+    let controller_connections =
+        get_controller_connections_to_nodes(node, accounts.get_controller_raw_keys().clone())
+            .await?;
+    prepare_validators(&root_connection, node, &accounts, controller_connections).await?;
     info!("New validators are set up");
 
     let reserved_validators = accounts.get_stash_accounts()[..reserved_seats as usize].to_vec();

e2e-tests/src/validators.rs

@@ -108,6 +108,7 @@ pub async fn prepare_validators<S: SignedConnectionApi + AuthorRpc>(
     connection: &S,
     node: &str,
     accounts: &Accounts,
+    controller_connections: Vec<SignedConnection>,
 ) -> anyhow::Result<()> {
     connection
         .batch_transfer(
@@ -138,9 +139,8 @@ pub async fn prepare_validators<S: SignedConnectionApi + AuthorRpc>(
         }));
     }
 
-    for controller in accounts.controller_raw_keys.iter() {
+    for connection in controller_connections {
         let keys = connection.author_rotate_keys().await?;
-        let connection = SignedConnection::new(node, KeyPair::new(controller.clone())).await;
         handles.push(tokio::spawn(async move {
             connection
                 .set_keys(keys, TxStatus::Finalized)
@@ -153,3 +153,28 @@ pub async fn prepare_validators<S: SignedConnectionApi + AuthorRpc>(
     join_all(handles).await;
     Ok(())
 }
+
+// Assumes the same ip address and consecutive ports for nodes, e.g. ws://127.0.0.1:9943,
+// ws://127.0.0.1:9944, etc.
+pub async fn get_controller_connections_to_nodes(
+    first_node_address: &str,
+    controller_raw_keys: Vec<RawKeyPair>,
+) -> anyhow::Result<Vec<SignedConnection>> {
+    let address_tokens = first_node_address.split(':').collect::<Vec<_>>();
+    let prefix = format!("{}:{}", address_tokens[0], address_tokens[1]);
+    let address_prefix = prefix.as_str();
+    let first_port = address_tokens[2].parse::<u16>()?;
+    let controller_connections =
+        controller_raw_keys
+            .into_iter()
+            .enumerate()
+            .map(|(port_idx, controller)| async move {
+                SignedConnection::new(
+                    format!("{}:{}", address_prefix, first_port + port_idx as u16).as_str(),
+                    KeyPair::new(controller),
+                )
+                .await
+            });
+    let connections = join_all(controller_connections.collect::<Vec<_>>()).await;
+    Ok(connections)
+}