diff --git a/docker/Dockerfile.network_tests b/docker/Dockerfile.network_tests new file mode 100644 index 0000000000..18741d6f19 --- /dev/null +++ b/docker/Dockerfile.network_tests @@ -0,0 +1,5 @@ +FROM aleph-node:latest + +RUN apt update && \ + apt install curl iproute2 iputils-ping net-tools netwox tcpdump gdb gdbserver -y && \ + apt clean diff --git a/docker/docker-compose.network_tests.yml b/docker/docker-compose.network_tests.yml new file mode 100644 index 0000000000..c437d05bc1 --- /dev/null +++ b/docker/docker-compose.network_tests.yml @@ -0,0 +1,50 @@ +services: + Node0: + image: aleph-node:network_tests + networks: + - main + environment: + - PURGE_BEFORE_START=false + cap_add: + - NET_ADMIN + - SYS_PTRACE + + Node1: + image: aleph-node:network_tests + networks: + - main + environment: + - PURGE_BEFORE_START=false + cap_add: + - NET_ADMIN + - SYS_PTRACE + + Node2: + image: aleph-node:network_tests + networks: + - main + environment: + - PURGE_BEFORE_START=false + cap_add: + - NET_ADMIN + - SYS_PTRACE + + Node3: + image: aleph-node:network_tests + networks: + - main + environment: + - PURGE_BEFORE_START=false + cap_add: + - NET_ADMIN + - SYS_PTRACE + + Node4: + image: aleph-node:network_tests + networks: + - main + environment: + - PURGE_BEFORE_START=false + cap_add: + - NET_ADMIN + - SYS_PTRACE diff --git a/scripts/catchup_version_upgrade_test.sh b/scripts/catchup_version_upgrade_test.sh index fdfd440ee0..9f42d5fb55 100755 --- a/scripts/catchup_version_upgrade_test.sh +++ b/scripts/catchup_version_upgrade_test.sh @@ -14,42 +14,12 @@ ALL_NODES_PORTS=${ALL_NODES_PORTS:-"9933:9934:9935:9936:9937"} WAIT_BLOCKS=${WAIT_BLOCKS:-30} EXT_STATUS=${EXT_STATUS:-"in-block"} -function log() { - echo $1 1>&2 -} - -function into_array() { - result=() - local tmp=$IFS - IFS=: - for e in $1; do - result+=($e) - done - IFS=$tmp -} +source ./scripts/common.sh function initialize { wait_for_finalized_block $1 $2 $3 } -function wait_for_finalized_block() { - local block_to_be_finalized=$1 - local node=$2 - local port=$3 - - while [[ $(get_best_finalized $node $port) -le $block_to_be_finalized ]]; do - sleep 3 - done -} - -function get_best_finalized { - local validator=$1 - local rpc_port=$2 - - local best_finalized=$(VALIDATOR=$validator RPC_HOST="127.0.0.1" RPC_PORT=$rpc_port ./.github/scripts/check_finalization.sh | sed 's/Last finalized block number: "\(.*\)"/\1/') - printf "%d" $best_finalized -} - function set_upgrade_session { local session=$1 local version=$2 @@ -110,45 +80,6 @@ function disconnect_nodes { done } -function wait_for_block { - local block=$1 - local validator=$2 - local rpc_port=$3 - - local last_block="" - while [[ -z "$last_block" ]]; do - last_block=$(docker run --rm --network container:$validator appropriate/curl:latest \ - -H "Content-Type: application/json" \ - -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params": '$block'}' http://127.0.0.1:$rpc_port | jq '.result') - done -} - -function get_last_block { - local validator=$1 - local rpc_port=$2 - - local last_block_number=$(docker run --rm --network container:$validator appropriate/curl:latest \ - -H "Content-Type: application/json" \ - -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock"}' http://127.0.0.1:$rpc_port | jq '.result.block.header.number') - printf "%d" $last_block_number -} - -function check_finalization { - local block_to_check=$1 - local -n nodes=$2 - local -n ports=$3 - - log "checking finalization for block $block_to_check" - - for i in "${!nodes[@]}"; do - local node=${nodes[$i]} - local rpc_port=${ports[$i]} - - log "checking finalization at node $node" - wait_for_finalized_block $block_to_check $node $rpc_port - done -} - into_array $NODES NODES=(${result[@]}) diff --git a/scripts/common.sh b/scripts/common.sh new file mode 100644 index 0000000000..7e575d265b --- /dev/null +++ b/scripts/common.sh @@ -0,0 +1,112 @@ +#!/bin/env bash + +function log() { + echo $1 1>&2 +} + +function into_array() { + result=() + local tmp=$IFS + IFS=: + for e in $1; do + result+=($e) + done + IFS=$tmp +} + +function check_finalization() { + local block_to_check=$1 + local -n nodes=$2 + local -n ports=$3 + + log "checking finalization for block $block_to_check" + + for i in "${!nodes[@]}"; do + local node=${nodes[$i]} + local rpc_port=${ports[$i]} + + log "checking finalization at node $node" + wait_for_finalized_block $block_to_check $node $rpc_port + done +} + +function check_relative_finalization_at_node() { + local node=$1 + local rpc_port=$2 + local awaited_blocks=$3 + + local last_block=$(get_last_block $node $rpc_port) + local awaited_finalized=$(($last_block+$awaited_blocks)) + + log "Last block seen at node $node was $last_block, awaiting block $awaited_finalized to be finalized" + + wait_for_finalized_block $awaited_finalized $node $rpc_port +} + +function check_relative_finalization() { + local awaited_blocks=$1 + local -n nodes=$2 + local -n ports=$3 + + log "checking finalization for $awaited_blocks block(s) in the future" + + for i in "${!nodes[@]}"; do + local node=${nodes[$i]} + local rpc_port=${ports[$i]} + + log "checking finalization at node $node (${node}:$rpc_port)" + check_relative_finalization_at_node $node $rpc_port $awaited_blocks + done +} + +function get_best_finalized() { + local validator=$1 + local rpc_port=$2 + + local best_finalized=$(VALIDATOR=$validator RPC_HOST="127.0.0.1" RPC_PORT=$rpc_port ./.github/scripts/check_finalization.sh | sed 's/Last finalized block number: "\(.*\)"/\1/') + printf "%d" $best_finalized +} + +function wait_for_finalized_block() { + local block_to_be_finalized=$1 + local node=$2 + local port=$3 + + while [[ $(get_best_finalized $node $port) -le $block_to_be_finalized ]]; do + sleep 3 + done +} + +function wait_for_block() { + local block=$1 + local validator=$2 + local rpc_port=$3 + + local last_block="" + while [[ -z "$last_block" ]]; do + last_block=$(docker run --rm --network container:$validator appropriate/curl:latest \ + -H "Content-Type: application/json" \ + -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlockHash", "params": '$block'}' http://127.0.0.1:$rpc_port | jq '.result') + done +} + +function retrieve_last_block() { + local validator=$1 + local rpc_port=$2 + + docker run --rm --network container:$validator appropriate/curl:latest \ + -H "Content-Type: application/json" \ + -d '{"id":1, "jsonrpc":"2.0", "method": "chain_getBlock"}' http://127.0.0.1:$rpc_port | jq '.result.block.header.number' +} + +function get_last_block() { + local validator=$1 + local rpc_port=$2 + + local last_block=0 + while [[ -z "$last_block" ]]; do + last_block=$(retrieve_last_block $validator $rpc_port) + sleep 1 + done + printf "%d" $last_block +} diff --git a/scripts/run_consensus_network_delay.sh b/scripts/run_consensus_network_delay.sh new file mode 100755 index 0000000000..133a57f643 --- /dev/null +++ b/scripts/run_consensus_network_delay.sh @@ -0,0 +1,110 @@ +#!/bin/env bash + +set -euo pipefail + +source ./scripts/common.sh + +function usage(){ + cat << EOF +Usage: + $0 + --network-delays "500:300" + list of delays for each node in ms; default="500:500:500:500:500" + --no-build-image + skip docker image build + --nodes "Node0:9933:Node1:9934" + list of pairs node:rpc_port; default="Node0:9933:Node1:9934:Node2:9935:Node3:9936:Node4:9937" + --check-block number + check finalization for a given block number, 0 means no-check; default=42 +EOF + exit 0 +} + +function build_test_image() { + docker build -t aleph-node:network_tests -f docker/Dockerfile.network_tests . +} + +function set_network_delay() { + local node=$1 + local delay=$2 + + log "setting network delay for node $node" + docker exec $node tc qdisc add dev eth1 root netem delay ${delay}ms +} + +while [[ $# -gt 0 ]]; do + case $1 in + --network-delays) + NETWORK_DELAYS="$2" + shift;shift + ;; + --no-build-image) + BUILD_IMAGE=false + shift + ;; + --nodes) + NODES="$2" + shift;shift + ;; + --check-block) + CHECK_BLOCK_FINALIZATION="$2" + shift;shift + ;; + --help) + usage + shift + ;; + *) + error "Unrecognized argument $1!" + ;; + esac +done + +NETWORK_DELAYS=${NETWORK_DELAYS:-"500:500:500:500:500"} +BUILD_IMAGE=${BUILD_IMAGE:-true} +NODE_PAIRS=${NODES:-"Node0:9933:Node1:9934:Node2:9935:Node3:9936:Node4:9937"} +NODES_PORTS=${NODES_PORTS:-"9933:9934:9935:9936:9937"} +CHECK_BLOCK_FINALIZATION=${CHECK_BLOCK_FINALIZATION:-44} + +into_array $NETWORK_DELAYS +NETWORK_DELAYS=(${result[@]}) + +into_array $NODE_PAIRS +NODE_PAIRS=(${result[@]}) +NODES=() +NODES_PORTS=() +for ((i=0; i<${#NODE_PAIRS[@]}; i+=2)); do + node=${NODE_PAIRS[$i]} + port=${NODE_PAIRS[(($i + 1))]} + + NODES+=($node) + NODES_PORTS+=($port) +done + + +if [[ "$BUILD_IMAGE" = true ]]; then + log "building custom docker image for network tests" + build_test_image +fi + +log "starting network" +OVERRIDE_DOCKER_COMPOSE=./docker/docker-compose.network_tests.yml DOCKER_COMPOSE=./docker/docker-compose.bridged.yml ./.github/scripts/run_consensus.sh 1>&2 +log "network started" + +for i in "${!NODES[@]}"; do + node=${NODES[$i]} + delay=${NETWORK_DELAYS[$i]} + log "setting network delay for node $node to ${delay}ms" + + set_network_delay $node $delay +done + +if [[ $CHECK_BLOCK_FINALIZATION -gt 0 ]]; then + log "checking finalization" + check_relative_finalization $CHECK_BLOCK_FINALIZATION NODES NODES_PORTS + log "finalization checked" +fi + +log "done" + +exit 0