From 7a901467d8e10137d56bad12b7f791377249234c Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 7 Mar 2018 09:51:10 +0300 Subject: [PATCH 01/73] Initial test integration Signed-off-by: ArtObr --- plenum/test/bls/conftest.py | 1 + plenum/test/bls/helper.py | 128 ++++++++++++++---- plenum/test/bls/test_add_bls_key.py | 47 +++---- plenum/test/bls/test_add_incorrect_bls_key.py | 46 ++++--- plenum/test/conftest.py | 13 ++ plenum/test/pool_transactions/helper.py | 13 +- 6 files changed, 168 insertions(+), 80 deletions(-) diff --git a/plenum/test/bls/conftest.py b/plenum/test/bls/conftest.py index c95eb40a93..3c03dd4061 100644 --- a/plenum/test/bls/conftest.py +++ b/plenum/test/bls/conftest.py @@ -4,6 +4,7 @@ from plenum.common.constants import DOMAIN_LEDGER_ID from plenum.common.util import get_utc_epoch from plenum.test.bls.helper import generate_state_root +from plenum.test.pool_transactions.conftest import looper participants = ["Node1", "Node2", "Node3"] diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 96298fa419..e30b249227 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -1,14 +1,17 @@ import base58 import os +from plenum.test.node_request.helper import sdk_ensure_pool_functional + from common.serializers.serialization import state_roots_serializer from plenum.common.constants import DOMAIN_LEDGER_ID, ALIAS, BLS_KEY from plenum.common.keygen_utils import init_bls_keys from plenum.common.messages.node_messages import Commit, Prepare, PrePrepare -from plenum.common.util import get_utc_epoch, randomString, random_from_alphabet -from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests +from plenum.common.util import get_utc_epoch, randomString, random_from_alphabet, hexToFriendly +from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, ensureClientConnectedToNodesAndPoolLedgerSame -from plenum.test.pool_transactions.helper import updateNodeData, new_client +from plenum.test.pool_transactions.helper import updateNodeData, new_client, sdk_send_update_node, sdk_add_new_nym, \ + sdk_pool_refresh def generate_state_root(): @@ -45,7 +48,46 @@ def check_bls_multi_sig_after_send(looper, txnPoolNodeSet, # 3. check how many multi-sigs are saved for multi_sigs in multi_sigs_for_batch: - assert len(multi_sigs) == saved_multi_sigs_count,\ + assert len(multi_sigs) == saved_multi_sigs_count, \ + "{} != {}".format(len(multi_sigs), saved_multi_sigs_count) + + # 3. check that bls multi-sig is the same for all nodes we get PrePrepare for (that is for all expect the last one) + for multi_sigs in multi_sigs_for_batch[:-1]: + if multi_sigs: + assert multi_sigs.count(multi_sigs[0]) == len(multi_sigs) + + +def sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_handle, + saved_multi_sigs_count): + # at least two because first request could have no + # signature since state can be clear + number_of_requests = 3 + + # 1. send requests + # Using loop to avoid 3pc batching + state_roots = [] + for i in range(number_of_requests): + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_handle, 1) + waitNodeDataEquality(looper, txnPoolNodeSet[0], *txnPoolNodeSet[:-1]) + state_roots.append( + state_roots_serializer.serialize( + bytes(txnPoolNodeSet[0].getState(DOMAIN_LEDGER_ID).committedHeadHash))) + + # 2. get all saved multi-sigs + multi_sigs_for_batch = [] + for state_root in state_roots: + multi_sigs = [] + for node in txnPoolNodeSet: + multi_sig = node.bls_bft.bls_store.get(state_root) + if multi_sig: + multi_sigs.append(multi_sig) + multi_sigs_for_batch.append(multi_sigs) + + # 3. check how many multi-sigs are saved + for multi_sigs in multi_sigs_for_batch: + assert len(multi_sigs) == saved_multi_sigs_count, \ "{} != {}".format(len(multi_sigs), saved_multi_sigs_count) # 3. check that bls multi-sig is the same for all nodes we get PrePrepare for (that is for all expect the last one) @@ -89,20 +131,20 @@ def calculate_multi_sig(creator, bls_bft_with_commits, quorums, pre_prepare): def create_pre_prepare_params(state_root, - ledger_id = DOMAIN_LEDGER_ID, + ledger_id=DOMAIN_LEDGER_ID, txn_root=None, timestamp=None, bls_multi_sig=None): - params= [0, - 0, - 0, - timestamp or get_utc_epoch(), - [('1' * 16, 1)], - 0, - "random digest", - ledger_id, - state_root, - txn_root or '1' * 32] + params = [0, + 0, + 0, + timestamp or get_utc_epoch(), + [('1' * 16, 1)], + 0, + "random digest", + ledger_id, + state_root, + txn_root or '1' * 32] if bls_multi_sig: params.append(bls_multi_sig.as_list()) return params @@ -122,12 +164,14 @@ def create_commit_no_bls_sig(req_key): params = create_commit_params(view_no, pp_seq_no) return Commit(*params) + def create_commit_with_bls_sig(req_key, bls_sig): view_no, pp_seq_no = req_key params = create_commit_params(view_no, pp_seq_no) params.append(bls_sig) return Commit(*params) + def create_commit_bls_sig(bls_bft, req_key, pre_prepare): view_no, pp_seq_no = req_key params = create_commit_params(view_no, pp_seq_no) @@ -159,8 +203,8 @@ def change_bls_key(looper, txnPoolNodeSet, key_in_txn = \ new_blspk \ - if not add_wrong \ - else ''.join(random_from_alphabet(32, base58.alphabet)) + if not add_wrong \ + else ''.join(random_from_alphabet(32, base58.alphabet)) node_data = { ALIAS: node.name, @@ -174,6 +218,30 @@ def change_bls_key(looper, txnPoolNodeSet, return new_blspk +def sdk_change_bls_key(looper, txnPoolNodeSet, + node, + sdk_pool_handle, + sdk_wallet_steward, + add_wrong=False): + new_blspk = init_bls_keys(node.keys_dir, node.name) + + key_in_txn = new_blspk \ + if not add_wrong \ + else ''.join(random_from_alphabet(32, base58.alphabet)) + + node_dest = hexToFriendly(node.nodestack.verhex) + sdk_send_update_node(looper, sdk_wallet_steward, + sdk_pool_handle, + node_dest, node.name, + None, None, + None, None, + bls_key=key_in_txn) + sdk_pool_refresh(looper, sdk_pool_handle) + waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) + return new_blspk + + def check_bls_key(blskey, node, nodes, add_wrong=False): ''' Check that each node has the same and correct blskey for this node @@ -195,24 +263,24 @@ def check_bls_key(blskey, node, nodes, add_wrong=False): def check_update_bls_key(node_num, saved_multi_sigs_count, looper, txnPoolNodeSet, - client_tdir, - poolTxnClientData, - stewards_and_wallets, + sdk_wallet_stewards, + sdk_pool_handle, add_wrong=False): # 1. Change BLS key for a specified NODE node = txnPoolNodeSet[node_num] - steward_client, steward_wallet = stewards_and_wallets[node_num] - new_blspk = change_bls_key(looper, txnPoolNodeSet, node, - steward_client, steward_wallet, - add_wrong) + sdk_wallet_steward = sdk_wallet_stewards[node_num] + new_blspk = sdk_change_bls_key(looper, txnPoolNodeSet, + node, + sdk_pool_handle, + sdk_wallet_steward, + add_wrong) # 2. Check that all Nodes see the new BLS key value check_bls_key(new_blspk, node, txnPoolNodeSet, add_wrong) # 3. Check that we can send new requests and have correct multisigs - client, wallet = new_client(looper, - poolTxnClientData, - txnPoolNodeSet, client_tdir) - check_bls_multi_sig_after_send(looper, txnPoolNodeSet, - client, wallet, - saved_multi_sigs_count=saved_multi_sigs_count) + sdk_wallet_new_client = sdk_add_new_nym(looper, sdk_pool_handle, + sdk_wallet_steward) + sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_new_client, + saved_multi_sigs_count) diff --git a/plenum/test/bls/test_add_bls_key.py b/plenum/test/bls/test_add_bls_key.py index 8d2ec7bca4..a503d55b9a 100644 --- a/plenum/test/bls/test_add_bls_key.py +++ b/plenum/test/bls/test_add_bls_key.py @@ -1,7 +1,5 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 4 nodes_wth_bls = 0 @@ -10,8 +8,10 @@ # As we use tests with Module scope, results from previous tests are accumulated, so # rotating BLS keys one by one, eventually we will have all keys changed -def test_add_bls_one_node(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_add_bls_one_node(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_stewards): ''' Added BLS key for 1st Node; do not expect that BLS multi-sigs are applied since no consensus (n-f) @@ -19,14 +19,14 @@ def test_add_bls_one_node(looper, txnPoolNodeSet, client_tdir, check_update_bls_key(node_num=0, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle) - -def test_add_bls_two_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_add_bls_two_nodes(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_stewards): ''' Added BLS key for 1st and 2d Nodes; do not expect that BLS multi-sigs are applied since no consensus (n-f) @@ -34,13 +34,14 @@ def test_add_bls_two_nodes(looper, txnPoolNodeSet, client_tdir, check_update_bls_key(node_num=1, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle) -def test_add_bls_three_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_add_bls_three_nodes(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_stewards): ''' Added BLS key for 1st, 2d and 3d Nodes; expect that BLS multi-sigs are applied since we have consensus now (3=n-f) @@ -52,13 +53,14 @@ def test_add_bls_three_nodes(looper, txnPoolNodeSet, client_tdir, check_update_bls_key(node_num=2, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle) -def test_add_bls_all_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_add_bls_all_nodes(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_stewards): ''' Eventually added BLS key for all Nodes; expect that BLS multi-sigs are applied since we have consensus now (4 > n-f) @@ -66,6 +68,5 @@ def test_add_bls_all_nodes(looper, txnPoolNodeSet, client_tdir, check_update_bls_key(node_num=3, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle) diff --git a/plenum/test/bls/test_add_incorrect_bls_key.py b/plenum/test/bls/test_add_incorrect_bls_key.py index 1e520746e2..a9bd70a08c 100644 --- a/plenum/test/bls/test_add_incorrect_bls_key.py +++ b/plenum/test/bls/test_add_incorrect_bls_key.py @@ -1,7 +1,5 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 4 nodes_wth_bls = 0 @@ -10,8 +8,10 @@ # As we use tests with Module scope, results from previous tests are accumulated, so # rotating BLS keys one by one, eventually we will have all keys changed -def test_add_incorrect_bls_one_node(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_add_incorrect_bls_one_node(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_stewards): ''' Added wrong BLS key for 1st Node; do not expect that BLS multi-sigs are applied @@ -19,14 +19,15 @@ def test_add_incorrect_bls_one_node(looper, txnPoolNodeSet, client_tdir, check_update_bls_key(node_num=0, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets, + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle, add_wrong=True) -def test_add_incorrect_bls_two_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_add_incorrect_bls_two_nodes(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_stewards): ''' Added wrong BLS key for 1st and 2d Nodes; do not expect that BLS multi-sigs are applied @@ -34,14 +35,15 @@ def test_add_incorrect_bls_two_nodes(looper, txnPoolNodeSet, client_tdir, check_update_bls_key(node_num=1, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets, + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle, add_wrong=True) -def test_add_incorrect_bls_three_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_add_incorrect_bls_three_nodes(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_stewards): ''' Added wrong BLS key for 1-3 Nodes; do not expect that BLS multi-sigs are applied @@ -53,14 +55,15 @@ def test_add_incorrect_bls_three_nodes(looper, txnPoolNodeSet, client_tdir, check_update_bls_key(node_num=2, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets, + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle, add_wrong=True) -def test_add_incorrect_bls_all_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_add_incorrect_bls_all_nodes(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_stewards): ''' Added wrong BLS key for all Nodes; Still do not expect that BLS multi-sigs are applied @@ -68,7 +71,6 @@ def test_add_incorrect_bls_all_nodes(looper, txnPoolNodeSet, client_tdir, check_update_bls_key(node_num=3, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets, + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle, add_wrong=True) diff --git a/plenum/test/conftest.py b/plenum/test/conftest.py index 0f6b871e16..72e3d3c26e 100644 --- a/plenum/test/conftest.py +++ b/plenum/test/conftest.py @@ -1080,6 +1080,19 @@ def sdk_wallet_steward(looper, sdk_wallet_handle, sdk_steward_seed): return sdk_wallet_handle, steward_did +@pytest.fixture(scope='module') +def sdk_wallet_stewards(looper, sdk_wallet_handle, poolTxnStewardNames, poolTxnData): + stewards = [] + for name in poolTxnStewardNames: + seed = poolTxnData["seeds"][name] + (steward_did, steward_verkey) = looper.loop.run_until_complete( + create_and_store_my_did(sdk_wallet_handle, + json.dumps({'seed': seed}))) + stewards.append((sdk_wallet_handle, steward_did)) + + yield stewards + + @pytest.fixture(scope='module') def sdk_wallet_client(looper, sdk_wallet_handle, sdk_client_seed): (client_did, _) = looper.loop.run_until_complete( diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index 5a6bd8a159..dba3ba87c5 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -370,7 +370,7 @@ async def prepare_nym_request(wallet, named_seed, alias, role): async def prepare_node_request(steward_did, new_node_name=None, clientIp=None, - clientPort=None, nodeIp=None, nodePort=None, bls_key=None, + clientPort=None, nodeIp=None, nodePort=None, blsKey=None, sigseed=None, destination=None, services=[VALIDATOR]): use_sigseed = sigseed is not None use_dest = destination is not None @@ -391,8 +391,8 @@ async def prepare_node_request(steward_did, new_node_name=None, clientIp=None, data['node_ip'] = nodeIp if nodePort is not None: data['node_port'] = nodePort - if bls_key is not None: - data['blskey'] = bls_key + if blsKey is not None: + data['blskey'] = blsKey if services is not None: data['services'] = services @@ -421,11 +421,13 @@ def sendUpdateNode(stewardClient, stewardWallet, node, node_data): return req -def sdk_send_update_node(looper, sdk_submitter_wallet, sdk_pool_handle, +def sdk_send_update_node(looper, sdk_submitter_wallet, + sdk_pool_handle, destination, alias, node_ip, node_port, client_ip, client_port, - services=[VALIDATOR]): + services=[VALIDATOR], + bls_key=None): _, submitter_did = sdk_submitter_wallet # filling node request node_request = looper.loop.run_until_complete( @@ -435,6 +437,7 @@ def sdk_send_update_node(looper, sdk_submitter_wallet, sdk_pool_handle, clientPort=client_port, nodeIp=node_ip, nodePort=node_port, + blsKey=bls_key, destination=destination, services=services)) From 9394543e12c1d6aa0bf202320a9124c56e6a752b Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 12 Mar 2018 09:01:20 +0300 Subject: [PATCH 02/73] Two more files done Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 5 +-- .../bls/test_bls_key_registry_pool_manager.py | 15 ++++++--- plenum/test/bls/test_no_state_proof.py | 31 ++++++++++--------- 3 files changed, 31 insertions(+), 20 deletions(-) diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index e30b249227..be469f708e 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -222,10 +222,11 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, node, sdk_pool_handle, sdk_wallet_steward, - add_wrong=False): + add_wrong=False, + new_bls=None): new_blspk = init_bls_keys(node.keys_dir, node.name) - key_in_txn = new_blspk \ + key_in_txn = new_bls or new_blspk \ if not add_wrong \ else ''.join(random_from_alphabet(32, base58.alphabet)) diff --git a/plenum/test/bls/test_bls_key_registry_pool_manager.py b/plenum/test/bls/test_bls_key_registry_pool_manager.py index 4bfcd01539..091c2e1d38 100644 --- a/plenum/test/bls/test_bls_key_registry_pool_manager.py +++ b/plenum/test/bls/test_bls_key_registry_pool_manager.py @@ -1,6 +1,7 @@ import pytest from plenum.bls.bls_key_register_pool_manager import BlsKeyRegisterPoolManager from plenum.common.constants import NODE, TXN_TYPE, BLS_KEY, DATA, ALIAS +from plenum.test.bls.helper import sdk_change_bls_key from plenum.test.pool_transactions.helper import updateNodeData from plenum.test.primary_selection.conftest import stewardAndWalletForMasterNode, txnPoolMasterNodes from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ @@ -52,17 +53,23 @@ def test_get_key_for_current_root_explicitly(bls_key_register_ledger, txnPoolNod def test_get_key_for_old_root_keys_changed(bls_key_register_ledger, pool_node_txns, + txnPoolNodeSet, node, looper, - stewardAndWalletForMasterNode): + sdk_wallet_steward, + sdk_pool_handle): old_bls_key = pool_node_txns[0][DATA][BLS_KEY] new_bls_key = pool_node_txns[0][DATA][BLS_KEY] + "Changed" old_pool_root_hash = node.poolManager.state.committedHeadHash # change BLS keys - client, wallet = stewardAndWalletForMasterNode - change_bls_keys(new_bls_key, node, - looper, client, wallet) + + sdk_change_bls_key(looper, txnPoolNodeSet, + node, + sdk_pool_handle, + sdk_wallet_steward, + add_wrong=False, + new_bls=new_bls_key) new_pool_root_hash = node.poolManager.state.committedHeadHash assert old_pool_root_hash != new_pool_root_hash diff --git a/plenum/test/bls/test_no_state_proof.py b/plenum/test/bls/test_no_state_proof.py index e35cbe1732..3c2f09aff3 100644 --- a/plenum/test/bls/test_no_state_proof.py +++ b/plenum/test/bls/test_no_state_proof.py @@ -1,37 +1,40 @@ from plenum.common.constants import TXN_TYPE, STATE_PROOF, DOMAIN_LEDGER_ID from plenum.common.util import get_utc_epoch -from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected +from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, sdk_send_random_and_check, \ + sdk_json_to_request_object nodeCount = 4 nodes_wth_bls = 0 def test_make_proof_bls_disabled(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): - reqs = sendRandomRequests(wallet1, client1, 1) - waitForSufficientRepliesForRequests(looper, client1, requests=reqs) + sdk_pool_handle, sdk_wallet_client): + reqs = sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) - req = reqs[0] + req = reqs[0][1]['result'] for node in txnPoolNodeSet: req_handler = node.get_req_handler(DOMAIN_LEDGER_ID) - key = req_handler.prepare_buy_key(req.identifier, req.reqId) + key = req_handler.prepare_buy_key(req['identifier'], req['reqId']) proof = req_handler.make_proof(key) assert not proof def test_make_result_bls_disabled(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): - reqs = sendRandomRequests(wallet1, client1, 1) - waitForSufficientRepliesForRequests(looper, client1, requests=reqs) + sdk_pool_handle, sdk_wallet_client): + reqs = sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) - req = reqs[0] + req = reqs[0][1]['result'] for node in txnPoolNodeSet: req_handler = node.get_req_handler(DOMAIN_LEDGER_ID) - key = req_handler.prepare_buy_key(req.identifier, req.reqId) + key = req_handler.prepare_buy_key(req['identifier'], req['reqId']) proof = req_handler.make_proof(key) - result = req_handler.make_result(req, + result = req_handler.make_result(sdk_json_to_request_object(reqs[0][0]), {TXN_TYPE: "buy"}, 2, get_utc_epoch(), From e11af80a1e135262803445cd1c8dee2ab0d0cd93 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 12 Mar 2018 16:29:49 +0300 Subject: [PATCH 03/73] new test Signed-off-by: ArtObr --- plenum/test/bls/test_bls_key_registry_pool_manager.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/plenum/test/bls/test_bls_key_registry_pool_manager.py b/plenum/test/bls/test_bls_key_registry_pool_manager.py index 091c2e1d38..004c673c7a 100644 --- a/plenum/test/bls/test_bls_key_registry_pool_manager.py +++ b/plenum/test/bls/test_bls_key_registry_pool_manager.py @@ -3,9 +3,6 @@ from plenum.common.constants import NODE, TXN_TYPE, BLS_KEY, DATA, ALIAS from plenum.test.bls.helper import sdk_change_bls_key from plenum.test.pool_transactions.helper import updateNodeData -from plenum.test.primary_selection.conftest import stewardAndWalletForMasterNode, txnPoolMasterNodes -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 4 From 873bbb1a661d083a9e48bc4945d148116ac1a249 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 12 Mar 2018 17:45:00 +0300 Subject: [PATCH 04/73] 4 more folders done Signed-off-by: ArtObr --- plenum/test/bls/test_send_txns_bls_consensus.py | 13 ++++++------- .../bls/test_send_txns_bls_less_than_consensus.py | 13 ++++++------- plenum/test/bls/test_send_txns_full_bls.py | 14 +++++++------- plenum/test/bls/test_send_txns_no_bls.py | 13 ++++++------- 4 files changed, 25 insertions(+), 28 deletions(-) diff --git a/plenum/test/bls/test_send_txns_bls_consensus.py b/plenum/test/bls/test_send_txns_bls_consensus.py index 3eb7f4cd45..7409ceba72 100644 --- a/plenum/test/bls/test_send_txns_bls_consensus.py +++ b/plenum/test/bls/test_send_txns_bls_consensus.py @@ -1,7 +1,6 @@ from plenum.server.quorums import Quorum -from plenum.test.bls.helper import check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected +from plenum.test.bls.helper import sdk_check_bls_multi_sig_after_send +from plenum.test.pool_transactions.conftest import looper nodeCount = 7 nodes_wth_bls = 5 @@ -14,13 +13,13 @@ def test_each_node_has_bls(txnPoolNodeSet): def test_send_txns_bls_consensus(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): + sdk_pool_handle, sdk_wallet_client): # make sure that we have commits from all nodes, and have 5 of 7 (n-f) BLS sigs there is enough # otherwise we may have 3 commits, but 1 of them may be without BLS, so we will Order this txn, but without multi-sig for node in txnPoolNodeSet: node.quorums.commit = Quorum(nodeCount) # we expect that although not all nodes can sign with BLS (because not all nodes have BLS keys), # we get multi-sig on all nodes (since all nodes can verify signatures) - check_bls_multi_sig_after_send(looper, txnPoolNodeSet, - client1, wallet1, - saved_multi_sigs_count=nodeCount) + sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, + saved_multi_sigs_count=nodeCount) diff --git a/plenum/test/bls/test_send_txns_bls_less_than_consensus.py b/plenum/test/bls/test_send_txns_bls_less_than_consensus.py index 96630ab5a4..370d6c3993 100644 --- a/plenum/test/bls/test_send_txns_bls_less_than_consensus.py +++ b/plenum/test/bls/test_send_txns_bls_less_than_consensus.py @@ -1,7 +1,6 @@ from plenum.server.quorums import Quorum -from plenum.test.bls.helper import check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected +from plenum.test.bls.helper import check_bls_multi_sig_after_send, sdk_check_bls_multi_sig_after_send +from plenum.test.pool_transactions.conftest import looper nodeCount = 7 nodes_wth_bls = 4 @@ -14,10 +13,10 @@ def test_each_node_has_bls(txnPoolNodeSet): def test_send_txns_bls_less_than_consensus(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): + sdk_pool_handle, sdk_wallet_client): # make sure that we have commits from all nodes, and have 4 of 7 ( < n-f) BLS sigs there is not enough for node in txnPoolNodeSet: node.quorums.commit = Quorum(nodeCount) - check_bls_multi_sig_after_send(looper, txnPoolNodeSet, - client1, wallet1, - saved_multi_sigs_count=0) + sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, + saved_multi_sigs_count=0) diff --git a/plenum/test/bls/test_send_txns_full_bls.py b/plenum/test/bls/test_send_txns_full_bls.py index 74d32b2da3..e5dbf74f16 100644 --- a/plenum/test/bls/test_send_txns_full_bls.py +++ b/plenum/test/bls/test_send_txns_full_bls.py @@ -1,10 +1,10 @@ -from plenum.test.bls.helper import check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected +from plenum.test.bls.helper import sdk_check_bls_multi_sig_after_send +from plenum.test.pool_transactions.conftest import looper nodeCount = 7 nodes_wth_bls = 7 + def test_each_node_has_bls(txnPoolNodeSet): for node in txnPoolNodeSet: assert node.bls_bft @@ -12,7 +12,7 @@ def test_each_node_has_bls(txnPoolNodeSet): def test_send_txns_full_bls(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): - check_bls_multi_sig_after_send(looper, txnPoolNodeSet, - client1, wallet1, - saved_multi_sigs_count=nodeCount) \ No newline at end of file + sdk_pool_handle, sdk_wallet_client): + sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, + saved_multi_sigs_count=nodeCount) diff --git a/plenum/test/bls/test_send_txns_no_bls.py b/plenum/test/bls/test_send_txns_no_bls.py index aced99ea25..613326faee 100644 --- a/plenum/test/bls/test_send_txns_no_bls.py +++ b/plenum/test/bls/test_send_txns_no_bls.py @@ -1,6 +1,5 @@ -from plenum.test.bls.helper import check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected +from plenum.test.bls.helper import sdk_check_bls_multi_sig_after_send +from plenum.test.pool_transactions.conftest import looper nodeCount = 4 nodes_wth_bls = 0 @@ -13,7 +12,7 @@ def test_each_node_has_bls(txnPoolNodeSet): def test_send_txns_no_bls(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): - check_bls_multi_sig_after_send(looper, txnPoolNodeSet, - client1, wallet1, - saved_multi_sigs_count=0) + sdk_pool_handle, sdk_wallet_client,): + sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, + saved_multi_sigs_count=0) From cff29c62c322f7c24b3534a035c6b3881b2d1fe6 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 12 Mar 2018 18:17:29 +0300 Subject: [PATCH 05/73] two more files Signed-off-by: ArtObr --- plenum/test/bls/test_update_bls_key.py | 52 +++++++++---------- .../test/bls/test_update_incorrect_bls_key.py | 52 +++++++++---------- 2 files changed, 48 insertions(+), 56 deletions(-) diff --git a/plenum/test/bls/test_update_bls_key.py b/plenum/test/bls/test_update_bls_key.py index 8173beba2a..d5e5a8b50a 100644 --- a/plenum/test/bls/test_update_bls_key.py +++ b/plenum/test/bls/test_update_bls_key.py @@ -9,57 +9,53 @@ # As we use tests with Module scope, results from previous tests are accumulated, so # rotating BLS keys one by one, eventually we will have all keys changed -def test_update_bls_one_node(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_update_bls_one_node(looper, txnPoolNodeSet, + sdk_wallet_stewards, + sdk_pool_handle): ''' Rotated BLS key for 1st node; BLS multi-signatures must be calculated for all Nodes. ''' - check_update_bls_key(node_num=0, - saved_multi_sigs_count=4, + check_update_bls_key(node_num=0, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle) -def test_update_bls_two_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_update_bls_two_nodes(looper, txnPoolNodeSet, + sdk_wallet_stewards, + sdk_pool_handle): ''' Rotated BLS key for 1st and 2d nodes; BLS multi-signatures must be calculated for all Nodes. ''' - check_update_bls_key(node_num=1, - saved_multi_sigs_count=4, + check_update_bls_key(node_num=1, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle) -def test_update_bls_three_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_update_bls_three_nodes(looper, txnPoolNodeSet, + sdk_wallet_stewards, + sdk_pool_handle): ''' Rotated BLS key for 1-3 Nodes; BLS multi-signatures must be calculated for all Nodes. ''' - check_update_bls_key(node_num=2, - saved_multi_sigs_count=4, + check_update_bls_key(node_num=2, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle) -def test_update_bls_all_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_update_bls_all_nodes(looper, txnPoolNodeSet, + sdk_wallet_stewards, + sdk_pool_handle): ''' Rotated BLS key for all Nodes; BLS multi-signatures must be calculated for all Nodes. ''' - check_update_bls_key(node_num=3, - saved_multi_sigs_count=4, + check_update_bls_key(node_num=3, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle) diff --git a/plenum/test/bls/test_update_incorrect_bls_key.py b/plenum/test/bls/test_update_incorrect_bls_key.py index 29694fe732..309a58a52b 100644 --- a/plenum/test/bls/test_update_incorrect_bls_key.py +++ b/plenum/test/bls/test_update_incorrect_bls_key.py @@ -10,8 +10,9 @@ # As we use tests with Module scope, results from previous tests are accumulated, so # rotating BLS keys one by one, eventually we will have all keys changed -def test_update_incorrect_bls_one_node(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_update_incorrect_bls_one_node(looper, txnPoolNodeSet, + sdk_wallet_stewards, + sdk_pool_handle): ''' Updated with wrong BLS key for 1st Node; Expect that BLS multi-sigs are applied since we have 3 correct signatures @@ -20,55 +21,50 @@ def test_update_incorrect_bls_one_node(looper, txnPoolNodeSet, client_tdir, # otherwise we may have 3 commits, but 1 of them may be without BLS, so we will Order this txn, but without multi-sig for node in txnPoolNodeSet: node.quorums.commit = Quorum(nodeCount) - check_update_bls_key(node_num=0, - saved_multi_sigs_count=4, + check_update_bls_key(node_num=0, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets, + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle, add_wrong=True) -def test_update_incorrect_bls_two_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_update_incorrect_bls_two_nodes(looper, txnPoolNodeSet, + sdk_wallet_stewards, + sdk_pool_handle): ''' Updated with wrong BLS key for 1st and 2d Nodes; do not expect that BLS multi-sigs are applied (we have less than n-f correct BLS sigs) ''' - check_update_bls_key(node_num=1, - saved_multi_sigs_count=0, + check_update_bls_key(node_num=1, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets, + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle, add_wrong=True) -def test_update_incorrect_bls_three_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_update_incorrect_bls_three_nodes(looper, txnPoolNodeSet, + sdk_wallet_stewards, + sdk_pool_handle): ''' Updated with wrong BLS keys 1-3 Nodes; do not expect that BLS multi-sigs are applied (we have less than n-f correct BLS sigs) ''' - check_update_bls_key(node_num=2, - saved_multi_sigs_count=0, + check_update_bls_key(node_num=2, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets, + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle, add_wrong=True) -def test_update_incorrect_bls_all_nodes(looper, txnPoolNodeSet, client_tdir, - poolTxnClientData, stewards_and_wallets): +def test_update_incorrect_bls_all_nodes(looper, txnPoolNodeSet, + sdk_wallet_stewards, + sdk_pool_handle): ''' Updated with wrong BLS keys all Nodes; do not expect that BLS multi-sigs are applied (we have less than n-f correct BLS sigs) ''' - check_update_bls_key(node_num=3, - saved_multi_sigs_count=0, + check_update_bls_key(node_num=3, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, - client_tdir=client_tdir, - poolTxnClientData=poolTxnClientData, - stewards_and_wallets=stewards_and_wallets, + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_pool_handle=sdk_pool_handle, add_wrong=True) From b070c365539a3d831c87da642bb84d048c293268 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 12 Mar 2018 19:02:53 +0300 Subject: [PATCH 06/73] some changes Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 11 ++++++----- plenum/test/bls/test_add_bls_key.py | 15 +++++++++++---- plenum/test/bls/test_add_incorrect_bls_key.py | 16 ++++++++++++---- 3 files changed, 29 insertions(+), 13 deletions(-) diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index be469f708e..22405e9ce7 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -236,10 +236,12 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, node_dest, node.name, None, None, None, None, - bls_key=key_in_txn) + bls_key=key_in_txn, + services=None) sdk_pool_refresh(looper, sdk_pool_handle) waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) - sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) + if not add_wrong: + sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) return new_blspk @@ -265,6 +267,7 @@ def check_bls_key(blskey, node, nodes, add_wrong=False): def check_update_bls_key(node_num, saved_multi_sigs_count, looper, txnPoolNodeSet, sdk_wallet_stewards, + sdk_wallet_client, sdk_pool_handle, add_wrong=False): # 1. Change BLS key for a specified NODE @@ -280,8 +283,6 @@ def check_update_bls_key(node_num, saved_multi_sigs_count, check_bls_key(new_blspk, node, txnPoolNodeSet, add_wrong) # 3. Check that we can send new requests and have correct multisigs - sdk_wallet_new_client = sdk_add_new_nym(looper, sdk_pool_handle, - sdk_wallet_steward) sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, - sdk_pool_handle, sdk_wallet_new_client, + sdk_pool_handle, sdk_wallet_client, saved_multi_sigs_count) diff --git a/plenum/test/bls/test_add_bls_key.py b/plenum/test/bls/test_add_bls_key.py index a503d55b9a..a2942e5e6d 100644 --- a/plenum/test/bls/test_add_bls_key.py +++ b/plenum/test/bls/test_add_bls_key.py @@ -11,7 +11,7 @@ def test_add_bls_one_node(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_stewards): + sdk_wallet_stewards, sdk_wallet_client): ''' Added BLS key for 1st Node; do not expect that BLS multi-sigs are applied since no consensus (n-f) @@ -20,13 +20,15 @@ def test_add_bls_one_node(looper, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) def test_add_bls_two_nodes(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_stewards): + sdk_wallet_stewards, + sdk_wallet_client): ''' Added BLS key for 1st and 2d Nodes; do not expect that BLS multi-sigs are applied since no consensus (n-f) @@ -35,13 +37,15 @@ def test_add_bls_two_nodes(looper, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client= sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) def test_add_bls_three_nodes(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_stewards): + sdk_wallet_stewards, + sdk_wallet_client): ''' Added BLS key for 1st, 2d and 3d Nodes; expect that BLS multi-sigs are applied since we have consensus now (3=n-f) @@ -54,13 +58,15 @@ def test_add_bls_three_nodes(looper, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) def test_add_bls_all_nodes(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_stewards): + sdk_wallet_stewards, + sdk_wallet_client): ''' Eventually added BLS key for all Nodes; expect that BLS multi-sigs are applied since we have consensus now (4 > n-f) @@ -69,4 +75,5 @@ def test_add_bls_all_nodes(looper, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) diff --git a/plenum/test/bls/test_add_incorrect_bls_key.py b/plenum/test/bls/test_add_incorrect_bls_key.py index a9bd70a08c..74b56f7752 100644 --- a/plenum/test/bls/test_add_incorrect_bls_key.py +++ b/plenum/test/bls/test_add_incorrect_bls_key.py @@ -11,7 +11,8 @@ def test_add_incorrect_bls_one_node(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_stewards): + sdk_wallet_stewards, + sdk_wallet_client): ''' Added wrong BLS key for 1st Node; do not expect that BLS multi-sigs are applied @@ -20,6 +21,7 @@ def test_add_incorrect_bls_one_node(looper, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle, add_wrong=True) @@ -27,7 +29,8 @@ def test_add_incorrect_bls_one_node(looper, def test_add_incorrect_bls_two_nodes(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_stewards): + sdk_wallet_stewards, + sdk_wallet_client): ''' Added wrong BLS key for 1st and 2d Nodes; do not expect that BLS multi-sigs are applied @@ -36,6 +39,7 @@ def test_add_incorrect_bls_two_nodes(looper, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle, add_wrong=True) @@ -43,7 +47,8 @@ def test_add_incorrect_bls_two_nodes(looper, def test_add_incorrect_bls_three_nodes(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_stewards): + sdk_wallet_stewards, + sdk_wallet_client): ''' Added wrong BLS key for 1-3 Nodes; do not expect that BLS multi-sigs are applied @@ -56,6 +61,7 @@ def test_add_incorrect_bls_three_nodes(looper, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle, add_wrong=True) @@ -63,7 +69,8 @@ def test_add_incorrect_bls_three_nodes(looper, def test_add_incorrect_bls_all_nodes(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_stewards): + sdk_wallet_stewards, + sdk_wallet_client): ''' Added wrong BLS key for all Nodes; Still do not expect that BLS multi-sigs are applied @@ -72,5 +79,6 @@ def test_add_incorrect_bls_all_nodes(looper, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle, add_wrong=True) From a0f75588eebc374b0b8a24cf215b00676824a4a5 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 13 Mar 2018 15:07:07 +0300 Subject: [PATCH 07/73] More tests integrated. Problems: validator functions of client in test_state_proof; problem with wrong bls keys Signed-off-by: ArtObr --- plenum/test/bls/conftest.py | 1 - plenum/test/bls/helper.py | 5 ++--- plenum/test/bls/test_add_bls_key.py | 2 +- plenum/test/bls/test_bls_bft_replica.py | 1 + plenum/test/bls/test_multi_signature_verifier.py | 3 +-- plenum/test/bls/test_send_txns_no_bls.py | 2 +- plenum/test/bls/test_state_proof.py | 3 ++- plenum/test/bls/test_update_bls_key.py | 15 +++++++++++---- plenum/test/bls/test_update_incorrect_bls_key.py | 11 +++++++++-- 9 files changed, 28 insertions(+), 15 deletions(-) diff --git a/plenum/test/bls/conftest.py b/plenum/test/bls/conftest.py index 3c03dd4061..a5d496e4d7 100644 --- a/plenum/test/bls/conftest.py +++ b/plenum/test/bls/conftest.py @@ -6,7 +6,6 @@ from plenum.test.bls.helper import generate_state_root from plenum.test.pool_transactions.conftest import looper - participants = ["Node1", "Node2", "Node3"] signature = "somefakesignaturesomefakesignaturesomefakesignature" diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 22405e9ce7..84236b4cd8 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -238,10 +238,9 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, None, None, bls_key=key_in_txn, services=None) - sdk_pool_refresh(looper, sdk_pool_handle) waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) - if not add_wrong: - sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) + sdk_pool_refresh(looper, sdk_pool_handle) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) return new_blspk diff --git a/plenum/test/bls/test_add_bls_key.py b/plenum/test/bls/test_add_bls_key.py index a2942e5e6d..019da1a34a 100644 --- a/plenum/test/bls/test_add_bls_key.py +++ b/plenum/test/bls/test_add_bls_key.py @@ -37,7 +37,7 @@ def test_add_bls_two_nodes(looper, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, - sdk_wallet_client= sdk_wallet_client, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) diff --git a/plenum/test/bls/test_bls_bft_replica.py b/plenum/test/bls/test_bls_bft_replica.py index 8709401b28..b133dfa2c2 100644 --- a/plenum/test/bls/test_bls_bft_replica.py +++ b/plenum/test/bls/test_bls_bft_replica.py @@ -18,6 +18,7 @@ whitelist = ['Indy Crypto error'] + @pytest.fixture() def bls_bft_replicas(txnPoolNodeSet): bls_bft_replicas = [] diff --git a/plenum/test/bls/test_multi_signature_verifier.py b/plenum/test/bls/test_multi_signature_verifier.py index 464e06c225..9191350ae9 100644 --- a/plenum/test/bls/test_multi_signature_verifier.py +++ b/plenum/test/bls/test_multi_signature_verifier.py @@ -18,7 +18,7 @@ def create_crypto_signer(bls_crypto_factory): def test_bls_crypto_verifier(bls_crypto_factory): - bls_crypto_factory = bls_crypto_factory # type: BlsFactoryIndyCrypto + bls_crypto_factory = bls_crypto_factory # type: BlsFactoryIndyCrypto bls_crypto_verifier = bls_crypto_factory.create_bls_crypto_verifier() assert bls_crypto_verifier @@ -32,4 +32,3 @@ def test_bls_crypto_verifier(bls_crypto_factory): multi_signature = bls_crypto_verifier.create_multi_sig(sigs) assert bls_crypto_verifier.verify_multi_sig(multi_signature, message, keys) - diff --git a/plenum/test/bls/test_send_txns_no_bls.py b/plenum/test/bls/test_send_txns_no_bls.py index 613326faee..e0ed9bc53e 100644 --- a/plenum/test/bls/test_send_txns_no_bls.py +++ b/plenum/test/bls/test_send_txns_no_bls.py @@ -12,7 +12,7 @@ def test_each_node_has_bls(txnPoolNodeSet): def test_send_txns_no_bls(looper, txnPoolNodeSet, - sdk_pool_handle, sdk_wallet_client,): + sdk_pool_handle, sdk_wallet_client): sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, saved_multi_sigs_count=0) diff --git a/plenum/test/bls/test_state_proof.py b/plenum/test/bls/test_state_proof.py index d003482a9d..ef1de21b32 100644 --- a/plenum/test/bls/test_state_proof.py +++ b/plenum/test/bls/test_state_proof.py @@ -1,5 +1,6 @@ from plenum.common.constants import ROOT_HASH, MULTI_SIGNATURE, PROOF_NODES, TXN_TYPE, DATA, TXN_TIME, STATE_PROOF, \ - DOMAIN_LEDGER_ID, MULTI_SIGNATURE_VALUE, MULTI_SIGNATURE_PARTICIPANTS, MULTI_SIGNATURE_SIGNATURE, MULTI_SIGNATURE_VALUE_LEDGER_ID, \ + DOMAIN_LEDGER_ID, MULTI_SIGNATURE_VALUE, MULTI_SIGNATURE_PARTICIPANTS, MULTI_SIGNATURE_SIGNATURE, \ + MULTI_SIGNATURE_VALUE_LEDGER_ID, \ MULTI_SIGNATURE_VALUE_STATE_ROOT, MULTI_SIGNATURE_VALUE_TXN_ROOT, MULTI_SIGNATURE_VALUE_POOL_STATE_ROOT, \ MULTI_SIGNATURE_VALUE_TIMESTAMP, DOMAIN_LEDGER_ID from plenum.common.plenum_protocol_version import PlenumProtocolVersion diff --git a/plenum/test/bls/test_update_bls_key.py b/plenum/test/bls/test_update_bls_key.py index d5e5a8b50a..2dcb70d28c 100644 --- a/plenum/test/bls/test_update_bls_key.py +++ b/plenum/test/bls/test_update_bls_key.py @@ -1,6 +1,5 @@ from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected +from plenum.test.pool_transactions.conftest import looper nodeCount = 4 nodes_wth_bls = 4 @@ -11,6 +10,7 @@ def test_update_bls_one_node(looper, txnPoolNodeSet, sdk_wallet_stewards, + sdk_wallet_client, sdk_pool_handle): ''' Rotated BLS key for 1st node; @@ -19,11 +19,13 @@ def test_update_bls_one_node(looper, txnPoolNodeSet, check_update_bls_key(node_num=0, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) def test_update_bls_two_nodes(looper, txnPoolNodeSet, sdk_wallet_stewards, + sdk_wallet_client, sdk_pool_handle): ''' Rotated BLS key for 1st and 2d nodes; @@ -32,11 +34,13 @@ def test_update_bls_two_nodes(looper, txnPoolNodeSet, check_update_bls_key(node_num=1, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) def test_update_bls_three_nodes(looper, txnPoolNodeSet, sdk_wallet_stewards, + sdk_wallet_client, sdk_pool_handle): ''' Rotated BLS key for 1-3 Nodes; @@ -45,12 +49,14 @@ def test_update_bls_three_nodes(looper, txnPoolNodeSet, check_update_bls_key(node_num=2, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) def test_update_bls_all_nodes(looper, txnPoolNodeSet, - sdk_wallet_stewards, - sdk_pool_handle): + sdk_wallet_stewards, + sdk_wallet_client, + sdk_pool_handle): ''' Rotated BLS key for all Nodes; BLS multi-signatures must be calculated for all Nodes. @@ -58,4 +64,5 @@ def test_update_bls_all_nodes(looper, txnPoolNodeSet, check_update_bls_key(node_num=3, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle) diff --git a/plenum/test/bls/test_update_incorrect_bls_key.py b/plenum/test/bls/test_update_incorrect_bls_key.py index 309a58a52b..026d3cff22 100644 --- a/plenum/test/bls/test_update_incorrect_bls_key.py +++ b/plenum/test/bls/test_update_incorrect_bls_key.py @@ -1,7 +1,6 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected +from plenum.test.pool_transactions.conftest import looper nodeCount = 4 nodes_wth_bls = 4 @@ -12,6 +11,7 @@ def test_update_incorrect_bls_one_node(looper, txnPoolNodeSet, sdk_wallet_stewards, + sdk_wallet_client, sdk_pool_handle): ''' Updated with wrong BLS key for 1st Node; @@ -24,12 +24,14 @@ def test_update_incorrect_bls_one_node(looper, txnPoolNodeSet, check_update_bls_key(node_num=0, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle, add_wrong=True) def test_update_incorrect_bls_two_nodes(looper, txnPoolNodeSet, sdk_wallet_stewards, + sdk_wallet_client, sdk_pool_handle): ''' Updated with wrong BLS key for 1st and 2d Nodes; @@ -38,12 +40,14 @@ def test_update_incorrect_bls_two_nodes(looper, txnPoolNodeSet, check_update_bls_key(node_num=1, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle, add_wrong=True) def test_update_incorrect_bls_three_nodes(looper, txnPoolNodeSet, sdk_wallet_stewards, + sdk_wallet_client, sdk_pool_handle): ''' Updated with wrong BLS keys 1-3 Nodes; @@ -52,12 +56,14 @@ def test_update_incorrect_bls_three_nodes(looper, txnPoolNodeSet, check_update_bls_key(node_num=2, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle, add_wrong=True) def test_update_incorrect_bls_all_nodes(looper, txnPoolNodeSet, sdk_wallet_stewards, + sdk_wallet_client, sdk_pool_handle): ''' Updated with wrong BLS keys all Nodes; @@ -66,5 +72,6 @@ def test_update_incorrect_bls_all_nodes(looper, txnPoolNodeSet, check_update_bls_key(node_num=3, saved_multi_sigs_count=4, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client, sdk_pool_handle=sdk_pool_handle, add_wrong=True) From b924ff58416c7e74efea44f20eb8c0e30bbab751 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 13 Mar 2018 18:47:59 +0300 Subject: [PATCH 08/73] Initial commit Signed-off-by: ArtObr --- .../test_3pc_paused_during_catch_up.py | 1 + .../test_state_reverted_before_catchup.py | 3 +- plenum/test/batching_3pc/conftest.py | 2 - .../batching_3pc/test_batching_scenarios.py | 2 +- .../test/blacklist/test_blacklist_client.py | 6 +- plenum/test/bls/conftest.py | 1 - plenum/test/bls/helper.py | 30 +++-- plenum/test/bls/test_add_bls_key.py | 3 - plenum/test/bls/test_add_incorrect_bls_key.py | 2 - plenum/test/bls/test_bls_bft_replica.py | 1 + .../bls/test_bls_key_registry_pool_manager.py | 2 - .../test/bls/test_multi_signature_verifier.py | 3 +- plenum/test/bls/test_send_txns_full_bls.py | 3 +- plenum/test/bls/test_state_proof.py | 3 +- plenum/test/checkpoints/conftest.py | 3 - plenum/test/checkpoints/helper.py | 4 +- .../test_checkpoint_bounds_after_catchup.py | 1 - ..._checkpoint_stabilization_after_catchup.py | 1 - .../test_message_outside_watermark.py | 1 + .../test_message_outside_watermark1.py | 1 - .../test_ordering_after_catchup.py | 1 - .../test_view_change_after_checkpoint.py | 1 - plenum/test/cli/conftest.py | 3 + plenum/test/cli/helper.py | 41 +++--- plenum/test/cli/test_basic_node_commands.py | 10 +- plenum/test/cli/test_cli_startup.py | 4 +- plenum/test/cli/test_keyring.py | 2 +- plenum/test/cli/test_prompt.py | 2 - plenum/test/client/test_client.py | 18 +-- plenum/test/client/test_client_authn.py | 1 - plenum/test/client/test_client_can_send.py | 2 - plenum/test/client/test_client_observer.py | 4 +- .../test/client/test_client_request_nack.py | 1 - ...st_client_resends_not_confirmed_request.py | 2 +- plenum/test/client/test_client_retry.py | 28 ++-- ...st_client_sends_get_request_to_one_node.py | 7 +- .../test_client_sends_to_f_plus_one_nodes.py | 9 +- .../client/test_client_sends_to_less_nodes.py | 2 +- plenum/test/client/test_core_authn.py | 1 - .../test_faulty_client_with_faulty_node.py | 10 +- plenum/test/client/test_protocol_version.py | 18 ++- .../test/client/test_state_proof_verified.py | 6 +- plenum/test/common/test_config_util.py | 1 - plenum/test/common/test_hook_mananger.py | 2 +- plenum/test/common/test_parse_ledger.py | 3 +- ...test_pool_file_raises_descriptive_error.py | 3 +- plenum/test/common/test_prepare_batch.py | 3 +- plenum/test/conftest.py | 126 ++++++++---------- plenum/test/delayers.py | 21 +-- plenum/test/exceptions.py | 1 + plenum/test/grouped_load_scheduling.py | 1 + plenum/test/helper.py | 6 +- .../fields_validation/test_base58_field.py | 2 +- .../fields_validation/test_ledger_id_field.py | 1 - .../test_time_among_field.py | 2 +- .../fields_validation/test_version_field.py | 1 - plenum/test/input_validation/helper.py | 22 +-- .../test_catchuprep_message.py | 1 - .../test_catchupreq_message.py | 1 - .../test_checkpoint_message.py | 1 - .../test_threepcstate_message.py | 1 - plenum/test/instances/test_prepare_digest.py | 1 - plenum/test/logging/conftest.py | 16 +-- plenum/test/logging/test_logging_txn_state.py | 1 - plenum/test/malicious_behaviors_client.py | 1 + plenum/test/malicious_behaviors_node.py | 11 +- plenum/test/monitoring/test_avg_latency.py | 1 - .../test_instance_change_with_Delta.py | 4 +- .../test_instance_change_with_req_Lambda.py | 7 +- .../test_no_check_if_no_new_requests.py | 2 +- .../monitoring/test_post_monitoring_stats.py | 17 ++- .../test/monitoring/test_stats_publisher.py | 2 +- plenum/test/monitoring/test_system_stats.py | 2 +- plenum/test/monitoring/test_throughput.py | 1 - .../monitoring/test_warn_unordered_log_msg.py | 3 +- plenum/test/node_catchup/conftest.py | 2 +- plenum/test/node_catchup/helper.py | 2 - .../test/node_catchup/test_catchup_demoted.py | 2 +- .../test_catchup_inlcuding_3PC.py | 1 - .../node_catchup/test_catchup_scenarios.py | 1 - .../test/node_catchup/test_config_ledger.py | 6 +- .../test/node_catchup/test_discard_view_no.py | 1 - .../test/node_catchup/test_large_catchup.py | 1 - .../node_catchup/test_new_node_catchup.py | 2 +- .../node_catchup/test_new_node_catchup2.py | 4 +- .../test_node_catchup_after_checkpoints.py | 1 - ...t_node_catchup_after_restart_after_txns.py | 2 +- ...test_node_catchup_after_restart_no_txns.py | 1 - .../test_node_catchup_causes_no_desync.py | 1 - ...tchup_when_3_not_primary_node_restarted.py | 3 +- .../node_catchup/test_node_ledger_manager.py | 2 +- ..._node_reject_invalid_txn_during_catchup.py | 4 +- .../test_node_request_consistency_proof.py | 1 - .../test_node_request_missing_transactions.py | 5 +- .../test_remove_request_keys_post_catchup.py | 4 +- .../node_catchup/test_req_id_key_error.py | 4 +- .../test_revert_during_catchup.py | 3 +- .../node_request/message_request/conftest.py | 1 - ...de_request_missing_three_phase_messages.py | 1 - .../test_node_requests_missing_preprepare.py | 15 +-- .../test_preprepare_request.py | 2 +- .../test_valid_message_request.py | 5 +- .../test/node_request/node_request_helper.py | 18 +-- .../test_belated_request_not_processed.py | 9 -- .../test_discard_3pc_for_ordered.py | 3 +- ..._ordering_when_pre_prepare_not_received.py | 2 +- .../test_order/test_request_ordering_2.py | 6 +- .../test_num_prepare_with_2_of_6_faulty.py | 1 - .../test_node_lacks_finalised_requests.py | 4 +- ...plus_2_nodes_but_not_primary_off_and_on.py | 1 - ...us_2_nodes_including_primary_off_and_on.py | 1 - .../node_request/test_request_forwarding.py | 4 +- .../test_timestamp/test_clock_disruption.py | 7 +- .../test_timestamp_post_view_change.py | 7 +- plenum/test/plugin/demo_plugin/__init__.py | 5 +- .../plugin/demo_plugin/test_plugin_basic.py | 2 +- .../test_plugin_request_handling.py | 4 +- .../test/plugin/demo_plugin/transactions.py | 1 - .../plugin/test_notifier_plugin_manager.py | 17 +-- plenum/test/pool_transactions/conftest.py | 53 -------- .../test_nodes_with_pool_txns.py | 3 +- plenum/test/primary_election/helpers.py | 2 +- .../test_primary_election_case1.py | 3 +- .../test_primary_election_case2.py | 2 +- .../test_primary_election_case4.py | 2 +- .../test_primary_election_case5.py | 8 +- .../test_primary_election_contested.py | 1 - .../test_primary_election_with_tie.py | 2 - ...est_add_node_to_pool_with_large_ppseqno.py | 8 +- .../test_catchup_after_view_change.py | 1 - .../test_catchup_multiple_rounds.py | 5 +- .../test_catchup_needed_check.py | 1 - .../test_primary_selection.py | 6 +- ...after_primary_demotion_and_pool_restart.py | 6 +- ...after_primary_demotion_and_view_changes.py | 7 +- .../test_primary_selection_pool_txn.py | 5 +- .../test_primary_selector.py | 6 +- .../test_recover_more_than_f_failure.py | 8 +- .../test_recover_primary_no_view_change.py | 5 +- .../test_selection_f_plus_one_quorum.py | 1 - .../test_propagate_recvd_after_request.py | 2 +- .../test_propagate_recvd_before_request.py | 1 - plenum/test/replica/test_buffers_cleaning.py | 14 +- ...y_marked_suspicious_for_sending_prepare.py | 1 - .../test_replica_reject_same_pre_prepare.py | 2 - plenum/test/script/helper.py | 4 +- .../script/test_change_non_primary_node_ha.py | 2 - .../script/test_change_primary_node_ha.py | 1 - .../test/storage/test_client_req_rep_store.py | 2 +- plenum/test/test_action_queue.py | 4 +- plenum/test/test_bootstrapping.py | 1 - plenum/test/test_client.py | 5 +- plenum/test/test_crypto.py | 2 +- plenum/test/test_delay.py | 2 +- plenum/test/test_dirty_read.py | 4 +- plenum/test/test_memory_consumpion.py | 1 - plenum/test/test_node.py | 42 +++--- plenum/test/test_node_basic.py | 2 +- plenum/test/test_node_request.py | 1 + plenum/test/test_performance.py | 8 +- plenum/test/test_req_authenticator.py | 1 - .../test_round_trip_with_one_faulty_node.py | 1 - plenum/test/test_stack.py | 5 +- plenum/test/test_testable.py | 1 - plenum/test/testable.py | 6 +- .../validator_info/test_validator_info.py | 4 +- plenum/test/view_change/helper.py | 81 ++++++----- ...test_view_change_all_nodes_random_delay.py | 1 - .../slow_nodes/test_view_change_complex.py | 1 - ...in_after_view_change_by_primary_restart.py | 37 +++-- ...st_discard_inst_chng_msg_from_past_view.py | 2 +- ..._master_primary_different_from_previous.py | 1 + .../test_new_node_joins_after_view_change.py | 1 + ...detecting_lag_from_view_change_messages.py | 1 - .../test_queueing_req_from_future_view.py | 1 - .../view_change/test_reverted_unordered.py | 1 - ...er_the_same_after_restart_for_all_nodes.py | 20 ++- ...ack_to_quorum_with_disconnected_primary.py | 2 +- .../test_view_change_n_minus_f_quorum.py | 1 - .../test_view_change_on_master_degraded.py | 5 +- .../test_view_change_start_without_primary.py | 3 +- .../test_view_change_without_any_reqs.py | 3 +- ..._changes_if_master_primary_disconnected.py | 1 - ...ew_not_changed_when_short_disconnection.py | 6 +- plenum/test/waits.py | 13 +- .../test/wallet/test_wallet_storage_helper.py | 4 - .../zstack_tests/test_zstack_reconnection.py | 1 - 187 files changed, 475 insertions(+), 648 deletions(-) diff --git a/plenum/test/batching_3pc/catch-up/test_3pc_paused_during_catch_up.py b/plenum/test/batching_3pc/catch-up/test_3pc_paused_during_catch_up.py index 3658677801..e8bcf2b782 100644 --- a/plenum/test/batching_3pc/catch-up/test_3pc_paused_during_catch_up.py +++ b/plenum/test/batching_3pc/catch-up/test_3pc_paused_during_catch_up.py @@ -3,6 +3,7 @@ from plenum.test.test_node import getNonPrimaryReplicas from plenum.test.helper import sdk_send_random_requests + def test_sdk_no_ordering_during_syncup(tconf, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): non_primary_replica = getNonPrimaryReplicas(txnPoolNodeSet, instId=0)[0] diff --git a/plenum/test/batching_3pc/catch-up/test_state_reverted_before_catchup.py b/plenum/test/batching_3pc/catch-up/test_state_reverted_before_catchup.py index 7a490ab93b..a25370a1bc 100644 --- a/plenum/test/batching_3pc/catch-up/test_state_reverted_before_catchup.py +++ b/plenum/test/batching_3pc/catch-up/test_state_reverted_before_catchup.py @@ -2,7 +2,7 @@ from plenum.test.delayers import cDelay from plenum.test.test_node import getNonPrimaryReplicas from plenum.test.batching_3pc.helper import checkNodesHaveSameRoots -from plenum.test.helper import sdk_signed_random_requests, sdk_send_and_check,\ +from plenum.test.helper import sdk_signed_random_requests, sdk_send_and_check, \ sdk_send_random_requests, sdk_get_replies @@ -46,7 +46,6 @@ def test_unordered_state_reverted_before_catchup( reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, tconf.Max3PCBatchSize) sdk_get_replies(looper, reqs, timeout=40) - committed_ledger_during_3pc = non_primary_node.getLedger( ledger_id).tree.root_hash uncommitted_ledger_during_3pc = non_primary_node.getLedger( diff --git a/plenum/test/batching_3pc/conftest.py b/plenum/test/batching_3pc/conftest.py index fdafc6bcb8..4f382c489c 100644 --- a/plenum/test/batching_3pc/conftest.py +++ b/plenum/test/batching_3pc/conftest.py @@ -1,8 +1,6 @@ import pytest from plenum.test.conftest import getValueFromModule -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected @pytest.fixture(scope="module") diff --git a/plenum/test/batching_3pc/test_batching_scenarios.py b/plenum/test/batching_3pc/test_batching_scenarios.py index 513f8211dd..3a7c11482d 100644 --- a/plenum/test/batching_3pc/test_batching_scenarios.py +++ b/plenum/test/batching_3pc/test_batching_scenarios.py @@ -22,7 +22,7 @@ def testPrePrepareProcessedInOrder(perf_chk_patched, looper, txnPoolNodeSet, """ tconf = perf_chk_patched pr, otherR = getPrimaryReplica(txnPoolNodeSet, instId=0), \ - getNonPrimaryReplicas(txnPoolNodeSet, instId=0) + getNonPrimaryReplicas(txnPoolNodeSet, instId=0) otherNodes = [r.node for r in otherR] ppsToDelay = 2 delayeds = 0 diff --git a/plenum/test/blacklist/test_blacklist_client.py b/plenum/test/blacklist/test_blacklist_client.py index f98ab7a18d..b35d3ce784 100644 --- a/plenum/test/blacklist/test_blacklist_client.py +++ b/plenum/test/blacklist/test_blacklist_client.py @@ -12,15 +12,15 @@ def setup(client1): # noinspection PyIncorrectDocstring,PyUnusedLocal,PyShadowingNames -def testDoNotBlacklistClient(setup, looper, nodeSet, up, client1, sent1): +def testDoNotBlacklistClient(setup, looper, txnPoolNodeSet, client1, sent1): """ Client should be not be blacklisted by node on sending an unsigned request """ # No node should blacklist the client def chk(): - for node in nodeSet: + for node in txnPoolNodeSet: assert not node.isClientBlacklisted(client1.name) - timeout = waits.expectedClientToPoolConnectionTimeout(len(nodeSet)) + timeout = waits.expectedClientToPoolConnectionTimeout(len(txnPoolNodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) diff --git a/plenum/test/bls/conftest.py b/plenum/test/bls/conftest.py index c95eb40a93..5c96c4ac2c 100644 --- a/plenum/test/bls/conftest.py +++ b/plenum/test/bls/conftest.py @@ -5,7 +5,6 @@ from plenum.common.util import get_utc_epoch from plenum.test.bls.helper import generate_state_root - participants = ["Node1", "Node2", "Node3"] signature = "somefakesignaturesomefakesignaturesomefakesignature" diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 96298fa419..2f87de3b80 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -45,7 +45,7 @@ def check_bls_multi_sig_after_send(looper, txnPoolNodeSet, # 3. check how many multi-sigs are saved for multi_sigs in multi_sigs_for_batch: - assert len(multi_sigs) == saved_multi_sigs_count,\ + assert len(multi_sigs) == saved_multi_sigs_count, \ "{} != {}".format(len(multi_sigs), saved_multi_sigs_count) # 3. check that bls multi-sig is the same for all nodes we get PrePrepare for (that is for all expect the last one) @@ -89,20 +89,20 @@ def calculate_multi_sig(creator, bls_bft_with_commits, quorums, pre_prepare): def create_pre_prepare_params(state_root, - ledger_id = DOMAIN_LEDGER_ID, + ledger_id=DOMAIN_LEDGER_ID, txn_root=None, timestamp=None, bls_multi_sig=None): - params= [0, - 0, - 0, - timestamp or get_utc_epoch(), - [('1' * 16, 1)], - 0, - "random digest", - ledger_id, - state_root, - txn_root or '1' * 32] + params = [0, + 0, + 0, + timestamp or get_utc_epoch(), + [('1' * 16, 1)], + 0, + "random digest", + ledger_id, + state_root, + txn_root or '1' * 32] if bls_multi_sig: params.append(bls_multi_sig.as_list()) return params @@ -122,12 +122,14 @@ def create_commit_no_bls_sig(req_key): params = create_commit_params(view_no, pp_seq_no) return Commit(*params) + def create_commit_with_bls_sig(req_key, bls_sig): view_no, pp_seq_no = req_key params = create_commit_params(view_no, pp_seq_no) params.append(bls_sig) return Commit(*params) + def create_commit_bls_sig(bls_bft, req_key, pre_prepare): view_no, pp_seq_no = req_key params = create_commit_params(view_no, pp_seq_no) @@ -159,8 +161,8 @@ def change_bls_key(looper, txnPoolNodeSet, key_in_txn = \ new_blspk \ - if not add_wrong \ - else ''.join(random_from_alphabet(32, base58.alphabet)) + if not add_wrong \ + else ''.join(random_from_alphabet(32, base58.alphabet)) node_data = { ALIAS: node.name, diff --git a/plenum/test/bls/test_add_bls_key.py b/plenum/test/bls/test_add_bls_key.py index 8d2ec7bca4..0991aad199 100644 --- a/plenum/test/bls/test_add_bls_key.py +++ b/plenum/test/bls/test_add_bls_key.py @@ -1,7 +1,5 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 4 nodes_wth_bls = 0 @@ -24,7 +22,6 @@ def test_add_bls_one_node(looper, txnPoolNodeSet, client_tdir, stewards_and_wallets=stewards_and_wallets) - def test_add_bls_two_nodes(looper, txnPoolNodeSet, client_tdir, poolTxnClientData, stewards_and_wallets): ''' diff --git a/plenum/test/bls/test_add_incorrect_bls_key.py b/plenum/test/bls/test_add_incorrect_bls_key.py index 1e520746e2..ab0716757d 100644 --- a/plenum/test/bls/test_add_incorrect_bls_key.py +++ b/plenum/test/bls/test_add_incorrect_bls_key.py @@ -1,7 +1,5 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 4 nodes_wth_bls = 0 diff --git a/plenum/test/bls/test_bls_bft_replica.py b/plenum/test/bls/test_bls_bft_replica.py index 8709401b28..b133dfa2c2 100644 --- a/plenum/test/bls/test_bls_bft_replica.py +++ b/plenum/test/bls/test_bls_bft_replica.py @@ -18,6 +18,7 @@ whitelist = ['Indy Crypto error'] + @pytest.fixture() def bls_bft_replicas(txnPoolNodeSet): bls_bft_replicas = [] diff --git a/plenum/test/bls/test_bls_key_registry_pool_manager.py b/plenum/test/bls/test_bls_key_registry_pool_manager.py index 4bfcd01539..5b03ce532d 100644 --- a/plenum/test/bls/test_bls_key_registry_pool_manager.py +++ b/plenum/test/bls/test_bls_key_registry_pool_manager.py @@ -3,8 +3,6 @@ from plenum.common.constants import NODE, TXN_TYPE, BLS_KEY, DATA, ALIAS from plenum.test.pool_transactions.helper import updateNodeData from plenum.test.primary_selection.conftest import stewardAndWalletForMasterNode, txnPoolMasterNodes -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 4 diff --git a/plenum/test/bls/test_multi_signature_verifier.py b/plenum/test/bls/test_multi_signature_verifier.py index 464e06c225..9191350ae9 100644 --- a/plenum/test/bls/test_multi_signature_verifier.py +++ b/plenum/test/bls/test_multi_signature_verifier.py @@ -18,7 +18,7 @@ def create_crypto_signer(bls_crypto_factory): def test_bls_crypto_verifier(bls_crypto_factory): - bls_crypto_factory = bls_crypto_factory # type: BlsFactoryIndyCrypto + bls_crypto_factory = bls_crypto_factory # type: BlsFactoryIndyCrypto bls_crypto_verifier = bls_crypto_factory.create_bls_crypto_verifier() assert bls_crypto_verifier @@ -32,4 +32,3 @@ def test_bls_crypto_verifier(bls_crypto_factory): multi_signature = bls_crypto_verifier.create_multi_sig(sigs) assert bls_crypto_verifier.verify_multi_sig(multi_signature, message, keys) - diff --git a/plenum/test/bls/test_send_txns_full_bls.py b/plenum/test/bls/test_send_txns_full_bls.py index 74d32b2da3..fbfa936ebb 100644 --- a/plenum/test/bls/test_send_txns_full_bls.py +++ b/plenum/test/bls/test_send_txns_full_bls.py @@ -5,6 +5,7 @@ nodeCount = 7 nodes_wth_bls = 7 + def test_each_node_has_bls(txnPoolNodeSet): for node in txnPoolNodeSet: assert node.bls_bft @@ -15,4 +16,4 @@ def test_send_txns_full_bls(looper, txnPoolNodeSet, client1, client1Connected, wallet1): check_bls_multi_sig_after_send(looper, txnPoolNodeSet, client1, wallet1, - saved_multi_sigs_count=nodeCount) \ No newline at end of file + saved_multi_sigs_count=nodeCount) diff --git a/plenum/test/bls/test_state_proof.py b/plenum/test/bls/test_state_proof.py index d003482a9d..ef1de21b32 100644 --- a/plenum/test/bls/test_state_proof.py +++ b/plenum/test/bls/test_state_proof.py @@ -1,5 +1,6 @@ from plenum.common.constants import ROOT_HASH, MULTI_SIGNATURE, PROOF_NODES, TXN_TYPE, DATA, TXN_TIME, STATE_PROOF, \ - DOMAIN_LEDGER_ID, MULTI_SIGNATURE_VALUE, MULTI_SIGNATURE_PARTICIPANTS, MULTI_SIGNATURE_SIGNATURE, MULTI_SIGNATURE_VALUE_LEDGER_ID, \ + DOMAIN_LEDGER_ID, MULTI_SIGNATURE_VALUE, MULTI_SIGNATURE_PARTICIPANTS, MULTI_SIGNATURE_SIGNATURE, \ + MULTI_SIGNATURE_VALUE_LEDGER_ID, \ MULTI_SIGNATURE_VALUE_STATE_ROOT, MULTI_SIGNATURE_VALUE_TXN_ROOT, MULTI_SIGNATURE_VALUE_POOL_STATE_ROOT, \ MULTI_SIGNATURE_VALUE_TIMESTAMP, DOMAIN_LEDGER_ID from plenum.common.plenum_protocol_version import PlenumProtocolVersion diff --git a/plenum/test/checkpoints/conftest.py b/plenum/test/checkpoints/conftest.py index 7ff4584dd8..f0545bc5a2 100644 --- a/plenum/test/checkpoints/conftest.py +++ b/plenum/test/checkpoints/conftest.py @@ -1,9 +1,6 @@ import pytest from plenum.test.conftest import getValueFromModule -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper, nodeThetaAdded, \ - stewardAndWallet1, steward1, stewardWallet from plenum.test.batching_3pc.conftest import tconf diff --git a/plenum/test/checkpoints/helper.py b/plenum/test/checkpoints/helper.py index 771949b501..3acf349aad 100644 --- a/plenum/test/checkpoints/helper.py +++ b/plenum/test/checkpoints/helper.py @@ -1,10 +1,10 @@ from plenum.test.helper import assertEquality -def chkChkpoints(nodes, total: int, stableIndex: int=None): +def chkChkpoints(nodes, total: int, stableIndex: int = None): for node in nodes: for r in node.replicas: - assert len(r.checkpoints) == total, '{} checkpoints {}, whereas total {}'.\ + assert len(r.checkpoints) == total, '{} checkpoints {}, whereas total {}'. \ format(r, len(r.checkpoints), total) if stableIndex is not None: assert r.checkpoints.values()[stableIndex].isStable, r.name diff --git a/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py index ec4a5237c6..872b7827f5 100644 --- a/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py @@ -10,7 +10,6 @@ def test_upper_bound_of_checkpoint_after_catchup_is_divisible_by_chk_freq( chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet, client1, wallet1, client1Connected, tdir, client_tdir, tconf, allPluginsPath): - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) _, _, epsilon = addNewStewardAndNode(looper, steward1, stewardWallet, diff --git a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py index 5209eb52b5..8a50eaf2e1 100644 --- a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py @@ -12,7 +12,6 @@ def test_second_checkpoint_after_catchup_can_be_stabilized( chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet, client1, wallet1, client1Connected, tdir, client_tdir, tconf, allPluginsPath): - _, _, epsilon = addNewStewardAndNode(looper, steward1, stewardWallet, 'EpsilonSteward', 'Epsilon', tdir, client_tdir, tconf, diff --git a/plenum/test/checkpoints/test_message_outside_watermark.py b/plenum/test/checkpoints/test_message_outside_watermark.py index 8011b38964..f1e7201409 100644 --- a/plenum/test/checkpoints/test_message_outside_watermark.py +++ b/plenum/test/checkpoints/test_message_outside_watermark.py @@ -9,6 +9,7 @@ CHK_FREQ = 5 LOG_SIZE = 3 * CHK_FREQ + def test_non_primary_recvs_3phase_message_outside_watermarks(chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, reqs_for_logsize): """ diff --git a/plenum/test/checkpoints/test_message_outside_watermark1.py b/plenum/test/checkpoints/test_message_outside_watermark1.py index a0b871ec7b..75db9acd0b 100644 --- a/plenum/test/checkpoints/test_message_outside_watermark1.py +++ b/plenum/test/checkpoints/test_message_outside_watermark1.py @@ -9,7 +9,6 @@ from plenum.test.view_change.conftest import perf_chk_patched from plenum.test.helper import sdk_send_random_and_check - TestRunningTimeLimitSec = 300 PerfCheckFreq = 30 diff --git a/plenum/test/checkpoints/test_ordering_after_catchup.py b/plenum/test/checkpoints/test_ordering_after_catchup.py index 21f0b62a85..09ad59d2dc 100644 --- a/plenum/test/checkpoints/test_ordering_after_catchup.py +++ b/plenum/test/checkpoints/test_ordering_after_catchup.py @@ -14,7 +14,6 @@ def add_new_node(looper, pool_nodes, steward, steward_wallet, tdir, client_tdir, tconf, all_plugins_path): - name = randomString(6) node_name = "Node-" + name new_steward_name = "Steward-" + name diff --git a/plenum/test/checkpoints/test_view_change_after_checkpoint.py b/plenum/test/checkpoints/test_view_change_after_checkpoint.py index 1b76cdb9e2..d0d57e8936 100644 --- a/plenum/test/checkpoints/test_view_change_after_checkpoint.py +++ b/plenum/test/checkpoints/test_view_change_after_checkpoint.py @@ -7,7 +7,6 @@ from stp_core.loop.eventually import eventually from plenum.test.helper import sdk_send_batches_of_random_and_check - CHK_FREQ = 5 diff --git a/plenum/test/cli/conftest.py b/plenum/test/cli/conftest.py index af0ad4303a..525351bf33 100644 --- a/plenum/test/cli/conftest.py +++ b/plenum/test/cli/conftest.py @@ -63,6 +63,7 @@ def createAllNodes(request, cli): def stopNodes(): for node in cli.nodes.values(): node.stop() + request.addfinalizer(stopNodes) @@ -85,8 +86,10 @@ def be(ctx): Fixture that is a 'be' function that closes over the test context. 'be' allows to change the current cli in the context. """ + def _(cli): ctx['current_cli'] = cli + return _ diff --git a/plenum/test/cli/helper.py b/plenum/test/cli/helper.py index 7c4da66266..c5b7140b77 100644 --- a/plenum/test/cli/helper.py +++ b/plenum/test/cli/helper.py @@ -7,7 +7,6 @@ import time - import plenum.cli.cli as cli from plenum.client.wallet import Wallet from plenum.common.constants import PRIMARY_SELECTION_PREFIX, CURRENT_PROTOCOL_VERSION @@ -97,7 +96,7 @@ def lastCmdOutput(self): self.printeds[: (len(self.printeds) - self.lastPrintIndex)])] printedTokens = [token[1] for tokens in reversed(self.printedTokens[:( - len(self.printedTokens) - self.lastPrintedTokenIndex)]) + len(self.printedTokens) - self.lastPrintedTokenIndex)]) for token in tokens.get('tokens', []) if len(token) > 1] pt = ''.join(printedTokens) return '\n'.join(printeds + [pt]).strip() @@ -169,12 +168,12 @@ def chk(): print("checking for {}".format(nodeName)) print(msgs) assert "{} added replica {}:0 to instance 0 (master)" \ - .format(nodeName, nodeName) in msgs + .format(nodeName, nodeName) in msgs assert "{} added replica {}:1 to instance 1 (backup)" \ - .format(nodeName, nodeName) in msgs + .format(nodeName, nodeName) in msgs assert "{}{} listening for other nodes at {}:{}" \ - .format(CONNECTION_PREFIX, nodeName, - *cli.nodes[nodeName].nodestack.ha) \ + .format(CONNECTION_PREFIX, nodeName, + *cli.nodes[nodeName].nodestack.ha) \ in msgs startUpTimeout = waits.expectedNodeStartUpTimeout() @@ -192,7 +191,7 @@ def checkAllNodesUp(cli): msgs = {stmt['msg'] for stmt in cli.printeds} expected = PRIMARY_SELECTION_PREFIX + \ - "{nm}:{inst} selected primary {pri}" " for instance {inst} (view 0)" + "{nm}:{inst} selected primary {pri}" " for instance {inst} (view 0)" assert len(cli.nodes) > 0 for nm, node in cli.nodes.items(): assert node @@ -304,12 +303,12 @@ def newCLI(looper, basedir, ledger_base_dir, nodeClass=TestNode, clientClass=TestClient, config=None, - partition: str=None, + partition: str = None, unique_name=None, logFileName=None, name=None, agentCreator=None, - nodes_chroot: str=None): + nodes_chroot: str = None): if partition: recorder = Recorder(partition) else: @@ -348,7 +347,7 @@ def checkCmdValid(cli, cmd): assert 'Invalid command' not in cli.lastCmdOutput -def newKeyPair(cli: TestCli, alias: str=None): +def newKeyPair(cli: TestCli, alias: str = None): cmd = "new key {}".format(alias) if alias else "new key" idrs = set() if cli.activeWallet: @@ -471,7 +470,7 @@ def checkBalance(balance, data): def waitForReply(cli, nodeCount, replyChecker, customTimeout=None): timeout = customTimeout or \ - waits.expectedTransactionExecutionTime(nodeCount) + waits.expectedTransactionExecutionTime(nodeCount) cli.looper.run(eventually(checkReply, cli, nodeCount, replyChecker, timeout=timeout)) @@ -500,8 +499,8 @@ def assertCliTokens(matchedVars, tokens): if expectedValue is not None: assert matchedValue is not None, \ - "Key '{}' not found in machedVars (matchedValue={})".\ - format(key, matchedValue) + "Key '{}' not found in machedVars (matchedValue={})". \ + format(key, matchedValue) expectedValueLen = len(expectedValue) if expectedValue else 0 matchedValueLen = len(matchedValue) if matchedValue else 0 @@ -509,9 +508,9 @@ def assertCliTokens(matchedVars, tokens): assert matchedValue == expectedValue, \ "Value not matched for key '{}', " \ "\nexpectedValue (length: {}): {}, " \ - "\nactualValue (length: {}): {}".\ - format(key, expectedValueLen, expectedValue, - matchedValueLen, matchedValue) + "\nactualValue (length: {}): {}". \ + format(key, expectedValueLen, expectedValue, + matchedValueLen, matchedValue) def doByCtx(ctx): @@ -594,12 +593,15 @@ def chk(obj, parity=True): raise AttributeError("only str, callable, or " "collections of str and callable " "are allowed") + chk(expect) chk(not_expect, False) + if within: cli.looper.run(eventually(check, timeout=within)) else: check() + return _ @@ -613,12 +615,11 @@ def checkPermissions(path, mode): def checkWalletRestored(cli, expectedWalletKeyName, expectedIdentifiers): - cli.lastCmdOutput == "Saved wallet {} restored".format( expectedWalletKeyName) assert cli._activeWallet.name == expectedWalletKeyName assert len(cli._activeWallet.identifiers) == \ - expectedIdentifiers + expectedIdentifiers def getOldIdentifiersForActiveWallet(cli): @@ -646,7 +647,7 @@ def createAndAssertNewKeyringCreation(do, name, expectedMsgs=None): def useAndAssertKeyring(do, name, expectedName=None, expectedMsgs=None): keyringName = expectedName or name finalExpectedMsgs = expectedMsgs or \ - ['Active wallet set to "{}"'.format(keyringName)] + ['Active wallet set to "{}"'.format(keyringName)] do('use wallet {}'.format(name), expect=finalExpectedMsgs ) @@ -655,7 +656,7 @@ def useAndAssertKeyring(do, name, expectedName=None, expectedMsgs=None): def saveAndAssertKeyring(do, name, expectedName=None, expectedMsgs=None): keyringName = expectedName or name finalExpectedMsgs = expectedMsgs or \ - ['Active wallet "{}" saved'.format(keyringName)] + ['Active wallet "{}" saved'.format(keyringName)] do('save wallet'.format(name), expect=finalExpectedMsgs ) diff --git a/plenum/test/cli/test_basic_node_commands.py b/plenum/test/cli/test_basic_node_commands.py index 24322227e4..eb03f49f75 100644 --- a/plenum/test/cli/test_basic_node_commands.py +++ b/plenum/test/cli/test_basic_node_commands.py @@ -23,13 +23,13 @@ def testNodeNames(be, do, cli, validNodeNames): # Create a node with a name of an already created node be(cli) do("new node {}".format(lastNodeName), expect=[ - "Node {} already exists.".format(lastNodeName)]) + "Node {} already exists.".format(lastNodeName)]) assert len(cli.nodes) == 4 # Create a node with invalid name randName = randomString(10) do("new node {}".format(randName), expect=[ - "Invalid node name '{}'. ".format(randName)]) + "Invalid node name '{}'. ".format(randName)]) args = cli.printedTokens[-1] token, _ = args['tokens'][0] # An error token should be printed @@ -44,15 +44,15 @@ def testCreateNodeWhenClientExistsWithoutKey(be, do, cli, validNodeNames): clientName = "testc1" be(cli) do("new client {}".format(clientName), expect=[ - "Active client set to {}".format(clientName)]) + "Active client set to {}".format(clientName)]) do("new node {}".format(validNodeNames[0]), expect=[ - "No key present in wallet"], within=2) + "No key present in wallet"], within=2) def testCreateNodeWhenClientExistsWithKey(be, do, cli, validNodeNames): clientName = "testc2" be(cli) do("new client {}".format(clientName), expect=[ - "Active client set to {}".format(clientName)]) + "Active client set to {}".format(clientName)]) do("new key", expect=["Current DID set to "]) addNodes(be, do, cli, validNodeNames) diff --git a/plenum/test/cli/test_cli_startup.py b/plenum/test/cli/test_cli_startup.py index 0c9595e106..ae7e9722cc 100644 --- a/plenum/test/cli/test_cli_startup.py +++ b/plenum/test/cli/test_cli_startup.py @@ -11,8 +11,8 @@ def assertPrintsDefaultClientAndIdentifier(cli): assert cli.printeds[1]['msg'] == "Current wallet set to {walletName}". \ format(walletName=dc.name) assert cli.printeds[0]['msg'] == \ - "Current identifier set to {alias} ({cryptonym})". \ - format(alias=dc.name, cryptonym=verstr) + "Current identifier set to {alias} ({cryptonym})". \ + format(alias=dc.name, cryptonym=verstr) def printedMessages(cli): diff --git a/plenum/test/cli/test_keyring.py b/plenum/test/cli/test_keyring.py index d709d3069e..467886976a 100644 --- a/plenum/test/cli/test_keyring.py +++ b/plenum/test/cli/test_keyring.py @@ -4,7 +4,7 @@ def createNewKeyring(name, cli): assert 'Active wallet set to "{}"'.format(name) in cli.lastCmdOutput assert 'New wallet {} created'.format(name) in cli.lastCmdOutput assert not oldKeyring or ( - oldKeyring and oldKeyring.name != cli._activeWallet.name) + oldKeyring and oldKeyring.name != cli._activeWallet.name) assert cli.activeWallet.name == name assert len(cli._activeWallet.identifiers) == 0 diff --git a/plenum/test/cli/test_prompt.py b/plenum/test/cli/test_prompt.py index 969fe7bbcb..dd4622e21b 100644 --- a/plenum/test/cli/test_prompt.py +++ b/plenum/test/cli/test_prompt.py @@ -1,5 +1,3 @@ - - def checkPrompt(cli, checkWith: str): promptTokens = cli.cli.application.layout.children[1].children[ 0].content.content.get_tokens("") diff --git a/plenum/test/client/test_client.py b/plenum/test/client/test_client.py index ba319f948d..040caeffa8 100644 --- a/plenum/test/client/test_client.py +++ b/plenum/test/client/test_client.py @@ -12,7 +12,7 @@ from plenum.test import waits from plenum.test.helper import checkResponseCorrectnessFromNodes, \ randomOperation, checkLastClientReqForNode, getRepliesFromClientInbox, \ - sendRandomRequest, waitForSufficientRepliesForRequests, assertLength, \ + sendRandomRequest, waitForSufficientRepliesForRequests, assertLength, \ sendReqsToNodesAndVerifySuffReplies from plenum.test.test_client import genTestClient @@ -29,7 +29,6 @@ 'verification key from disk', 'got error while verifying message'] # warnings - logger = getlogger() @@ -60,6 +59,7 @@ def testClientShouldNotBeAbleToConnectToNodesNodeStack(pool): """ Client should not be able to connect to nodes in the node's nodestack """ + async def go(ctx): nodestacksVersion = {k: v.ha for k, v in ctx.nodeset.nodeReg.items()} client1, _ = genTestClient( @@ -195,7 +195,7 @@ def testReplyWhenRepliesFromExactlyFPlusOneNodesAreSame(looper, # noinspection PyIncorrectDocstring -def testReplyWhenRequestAlreadyExecuted(looper, nodeSet, client1, sent1): +def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, client1, sent1): """ When a request has already been executed the previously executed reply will be sent again to the client. An acknowledgement will not be sent @@ -211,7 +211,7 @@ def testReplyWhenRequestAlreadyExecuted(looper, nodeSet, client1, sent1): for part in message_parts: client1.nodestack._enqueueIntoAllRemotes(part, None) - + def chk(): assertLength([response for response in client1.inBox if (response[0].get(f.RESULT.nm) and @@ -225,7 +225,8 @@ def chk(): # noinspection PyIncorrectDocstring -def testReplyMatchesRequest(looper, nodeSet, client_tdir, up): +@pytest.mark.skip('Unskip and complete this test while sdk integration') +def testReplyMatchesRequest(looper, txnPoolNodeSet, client_tdir): ''' This tests does check following things: - wallet works correctly when used by multiple clients @@ -233,7 +234,7 @@ def testReplyMatchesRequest(looper, nodeSet, client_tdir, up): ''' def makeClient(id): - client, wallet = genTestClient(nodeSet, + client, wallet = genTestClient(txnPoolNodeSet, tmpdir=client_tdir, name="client-{}".format(id)) looper.add(client) @@ -293,9 +294,10 @@ def makeClient(id): assert replies[0] == sentAmount -def testReplyReceivedOnlyByClientWhoSentRequest(looper, nodeSet, client_tdir, +@pytest.mark.skip('Unskip and complete this test while sdk integration') +def testReplyReceivedOnlyByClientWhoSentRequest(looper, txnPoolNodeSet, client_tdir, client1, wallet1): - newClient, _ = genTestClient(nodeSet, tmpdir=client_tdir) + newClient, _ = genTestClient(txnPoolNodeSet, tmpdir=client_tdir) looper.add(newClient) looper.run(newClient.ensureConnectedToNodes()) client1InboxSize = len(client1.inBox) diff --git a/plenum/test/client/test_client_authn.py b/plenum/test/client/test_client_authn.py index ea94178abd..1bb6bc882f 100644 --- a/plenum/test/client/test_client_authn.py +++ b/plenum/test/client/test_client_authn.py @@ -5,7 +5,6 @@ from plenum.common.signer_simple import SimpleSigner from plenum.server.client_authn import SimpleAuthNr - idr = '5G72199XZB7wREviUbQma7' msg_str = "42 (forty-two) is the natural number that succeeds 41 and precedes 43." diff --git a/plenum/test/client/test_client_can_send.py b/plenum/test/client/test_client_can_send.py index 68a8404174..e7f2bdb6ff 100644 --- a/plenum/test/client/test_client_can_send.py +++ b/plenum/test/client/test_client_can_send.py @@ -6,8 +6,6 @@ from plenum.test.helper import random_requests from plenum.test.pool_transactions.helper import buildPoolClientAndWallet from stp_core.loop.eventually import eventually -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected, steward1, stewardWallet, stewardAndWallet1 def new_client(poolTxnClientData, tdirWithPoolTxns): diff --git a/plenum/test/client/test_client_observer.py b/plenum/test/client/test_client_observer.py index a5f2073284..08cab0549e 100644 --- a/plenum/test/client/test_client_observer.py +++ b/plenum/test/client/test_client_observer.py @@ -3,7 +3,7 @@ from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies -def test_observer_registration(looper, nodeSet, up, client1): +def test_observer_registration(looper, txnPoolNodeSet, client1): def callable1(*args, **kwargs): print(1) print(args) @@ -48,7 +48,7 @@ def callable2(*args, **kwargs): assert len(client1._observers) == 1 -def test_observer_execution(looper, nodeSet, up, client1, wallet1): +def test_observer_execution(looper, txnPoolNodeSet, client1, wallet1): resp1 = [] resp2 = [] diff --git a/plenum/test/client/test_client_request_nack.py b/plenum/test/client/test_client_request_nack.py index 53643bd92e..0d43f2d05b 100644 --- a/plenum/test/client/test_client_request_nack.py +++ b/plenum/test/client/test_client_request_nack.py @@ -36,7 +36,6 @@ def testRequestFullRoundTrip(restrictiveVerifier, sent1, looper, nodeSet): - update = {'reason': 'client request invalid: InvalidClientRequest() ' '[caused by amount too high\nassert 999 <= 100]'} diff --git a/plenum/test/client/test_client_resends_not_confirmed_request.py b/plenum/test/client/test_client_resends_not_confirmed_request.py index 0a08874bb7..b5c09ac2f0 100644 --- a/plenum/test/client/test_client_resends_not_confirmed_request.py +++ b/plenum/test/client/test_client_resends_not_confirmed_request.py @@ -18,7 +18,7 @@ def test_client_resends_not_confirmed_request(looper, client1, wallet1, - nodeSet): + txnPoolNodeSet): """ Check that client resends request to all nodes if it was previously sent to one node but reply cannot be verified diff --git a/plenum/test/client/test_client_retry.py b/plenum/test/client/test_client_retry.py index f38fc85de6..128fb12900 100644 --- a/plenum/test/client/test_client_retry.py +++ b/plenum/test/client/test_client_retry.py @@ -13,14 +13,14 @@ whitelist = ['AlphaC unable to send message', ] -def testClientRetryRequestWhenAckNotReceived(looper, nodeSet, client1, wallet1): +def testClientRetryRequestWhenAckNotReceived(looper, txnPoolNodeSet, client1, wallet1): """ The client gets disconnected from node say Alpha but does not know it. It sends request to all nodes including Alpha, expects ACK and REPLY from Alpha too, does not get it, so reconnects to Alpha and sends request again and gets REPLY """ - alpha = nodeSet.Alpha + alpha = txnPoolNodeSet[0] skipped = False origPr = alpha.processRequest @@ -37,7 +37,7 @@ def skipReqOnce(msg, remoteName): req = sendRandomRequest(wallet1, client1) def chkAcks(): - for node in nodeSet: + for node in txnPoolNodeSet: if node != alpha: checkReqAck(client1, node, *req.key) else: @@ -50,14 +50,14 @@ def chkAcks(): wait_for_replies(looper, client1, idr, reqId, 4) -def testClientRetryRequestWhenReplyNotReceived(looper, nodeSet, client1, +def testClientRetryRequestWhenReplyNotReceived(looper, txnPoolNodeSet, client1, wallet1, tconf): """ A node say Alpha sends ACK but doesn't send REPLY. The client resends the request and gets REPLY """ - alpha = nodeSet.Alpha + alpha = txnPoolNodeSet[0] skipped = False origTrans = alpha.transmitToClient @@ -70,7 +70,7 @@ def skipReplyOnce(msg, remoteName): alpha.transmitToClient = skipReplyOnce req = sendRandomRequest(wallet1, client1) - coros = [partial(checkReqAck, client1, node, *req.key) for node in nodeSet] + coros = [partial(checkReqAck, client1, node, *req.key) for node in txnPoolNodeSet] timeout = waits.expectedReqAckQuorumTime() start = time.perf_counter() looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=timeout)) @@ -86,14 +86,14 @@ def skipReplyOnce(msg, remoteName): wait_for_replies(looper, client1, idr, reqId, 4) -def testClientNotRetryRequestWhenReqnackReceived(looper, nodeSet, client1, wallet1): +def testClientNotRetryRequestWhenReqnackReceived(looper, txnPoolNodeSet, client1, wallet1): """ A node sends REQNACK. The client does not resend Request. """ - numOfNodes = len(nodeSet) + numOfNodes = len(txnPoolNodeSet) - alpha = nodeSet.Alpha + alpha = txnPoolNodeSet[0] origProcReq = alpha.processRequest origTrans = alpha.transmitToClient @@ -148,7 +148,7 @@ def reset(): def testClientNotRetryingRequestAfterMaxTriesDone(looper, - nodeSet, + txnPoolNodeSet, client1, wallet1, withFewerRetryReq): @@ -158,7 +158,7 @@ def testClientNotRetryingRequestAfterMaxTriesDone(looper, configuration and no more """ - alpha = nodeSet.Alpha + alpha = txnPoolNodeSet[0] origTrans = alpha.transmitToClient def dontTransmitReply(msg, remoteName): @@ -175,8 +175,8 @@ def dontTransmitReply(msg, remoteName): # +1 because we have to wait one more retry timeout to make sure what # client cleaned his buffers (expectingAcksFor, expectingRepliesFor) retryTime = withFewerRetryReq.CLIENT_REPLY_TIMEOUT * \ - (withFewerRetryReq.CLIENT_MAX_RETRY_REPLY + 1) - timeout = waits.expectedTransactionExecutionTime(len(nodeSet)) + retryTime + (withFewerRetryReq.CLIENT_MAX_RETRY_REPLY + 1) + timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + retryTime looper.runFor(timeout) @@ -184,7 +184,7 @@ def dontTransmitReply(msg, remoteName): wait_for_replies(looper, client1, idr, reqId, 3) assert client1.spylog.count(client1.resendRequests.__name__) == \ - (totalResends + withFewerRetryReq.CLIENT_MAX_RETRY_REPLY) + (totalResends + withFewerRetryReq.CLIENT_MAX_RETRY_REPLY) assert req.key not in client1.expectingAcksFor assert req.key not in client1.expectingRepliesFor alpha.transmitToClient = origTrans diff --git a/plenum/test/client/test_client_sends_get_request_to_one_node.py b/plenum/test/client/test_client_sends_get_request_to_one_node.py index 57db7c093c..0ea6a1d61e 100644 --- a/plenum/test/client/test_client_sends_get_request_to_one_node.py +++ b/plenum/test/client/test_client_sends_get_request_to_one_node.py @@ -4,7 +4,6 @@ from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually from plenum.test.client.conftest import passThroughReqAcked1 -from plenum.test.helper import stopNodes, send_signed_requests from plenum.test.helper import stopNodes, send_signed_requests from plenum.test.malicious_behaviors_client import \ @@ -20,7 +19,7 @@ def test_client_sends_get_request_to_one_node(looper, client1, wallet1, - nodeSet): + txnPoolNodeSet): """ Check that client sends read only request to one node only """ @@ -52,7 +51,7 @@ def sign_and_send(op): def test_client_can_send_get_request_to_one_node(looper, client1, wallet1, - nodeSet): + txnPoolNodeSet): """ Check that read only request can be sent without having connection to all nodes @@ -61,7 +60,7 @@ def test_client_can_send_get_request_to_one_node(looper, wallet = wallet1 logger.info("Stopping nodes") - nodes_to_stop = list(nodeSet)[1:] + nodes_to_stop = list(txnPoolNodeSet)[1:] stopNodes(nodes_to_stop, looper) def sign_and_send(op): diff --git a/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py b/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py index a6d7ca3656..b94b227189 100644 --- a/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py +++ b/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py @@ -5,24 +5,23 @@ genDoesntSendRequestToSomeNodes from plenum.test.node_catchup.helper import waitNodeDataEquality - nodeCount = 4 clientFault = genDoesntSendRequestToSomeNodes("AlphaC") reqAcked1 = passThroughReqAcked1 -def testReplyWhenRequestSentToMoreThanFPlusOneNodes(looper, nodeSet, +def testReplyWhenRequestSentToMoreThanFPlusOneNodes(looper, txnPoolNodeSet, fClient, replied1, wallet1): """ Alpha would not be sent request but other nodes will be, so Alpha will just rely on propagates from other nodes """ - alpha = nodeSet.Alpha - other_nodes = [n for n in nodeSet if n != alpha] + alpha = txnPoolNodeSet[0] + other_nodes = [n for n in txnPoolNodeSet if n != alpha] def chk(req_count=1): - for node in nodeSet: + for node in txnPoolNodeSet: prc_req = node.processRequest.__name__ prc_ppg = node.processPropagate.__name__ if node != alpha: diff --git a/plenum/test/client/test_client_sends_to_less_nodes.py b/plenum/test/client/test_client_sends_to_less_nodes.py index 74d149edb5..5facd46649 100644 --- a/plenum/test/client/test_client_sends_to_less_nodes.py +++ b/plenum/test/client/test_client_sends_to_less_nodes.py @@ -10,7 +10,7 @@ # noinspection PyIncorrectDocstring -def testReplyWhenRequestSentToLessThanFPlusOneNodes(looper, nodeSet, +def testReplyWhenRequestSentToLessThanFPlusOneNodes(looper, txnPoolNodeSet, fClient: Client, replied1): """ In a system with no faulty nodes, even if the client sends the request to diff --git a/plenum/test/client/test_core_authn.py b/plenum/test/client/test_core_authn.py index 1cdb645150..7875348fd5 100644 --- a/plenum/test/client/test_core_authn.py +++ b/plenum/test/client/test_core_authn.py @@ -7,7 +7,6 @@ from plenum.common.types import f from plenum.server.client_authn import CoreAuthNr - idr = '5G72199XZB7wREviUbQma7' msg_str = "42 (forty-two) is the natural number that succeeds 41 and precedes 43." diff --git a/plenum/test/client/test_faulty_client_with_faulty_node.py b/plenum/test/client/test_faulty_client_with_faulty_node.py index 47f3138d9e..c0cc8e766c 100644 --- a/plenum/test/client/test_faulty_client_with_faulty_node.py +++ b/plenum/test/client/test_faulty_client_with_faulty_node.py @@ -17,18 +17,18 @@ @pytest.fixture(scope="module") -def nodeChangesRequest(nodeSet): - alpha = nodeSet.Alpha +def nodeChangesRequest(txnPoolNodeSet): + alpha = txnPoolNodeSet[0] # Alpha should not be blacklisted for Invalid Signature by all other nodes whitelistNode(alpha.name, - [node for node in nodeSet if node != alpha], + [node for node in txnPoolNodeSet if node != alpha], InvalidSignature.code) - makeNodeFaulty(alpha, changesRequest,) + makeNodeFaulty(alpha, changesRequest, ) # noinspection PyIncorrectDocstring,PyUnusedLocal,PyShadowingNames -def testReplyUnaffectedByFaultyNode(looper, nodeSet, nodeChangesRequest, +def testReplyUnaffectedByFaultyNode(looper, txnPoolNodeSet, nodeChangesRequest, fClient, replied1): """ Client is malicious - sends requests to Alpha and Beta only diff --git a/plenum/test/client/test_protocol_version.py b/plenum/test/client/test_protocol_version.py index ad37a07c60..da3b89426e 100644 --- a/plenum/test/client/test_protocol_version.py +++ b/plenum/test/client/test_protocol_version.py @@ -3,10 +3,7 @@ from plenum.common.request import Request from plenum.test.helper import waitForSufficientRepliesForRequests, \ send_signed_requests, checkReqNackWithReason, random_request_objects, \ - sign_request_objects, signed_random_requests, random_requests -# noinspection PyUnresolvedReferences -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected + sign_request_objects, random_requests from stp_core.loop.eventually import eventually @@ -14,8 +11,9 @@ def request_num(request): return int(request.param) + def test_request_no_protocol_version(looper, txnPoolNodeSet, - client1, client1Connected, + client1, wallet1, request_num): reqs = random_request_objects(request_num, protocol_version=None) @@ -28,9 +26,9 @@ def test_request_no_protocol_version(looper, txnPoolNodeSet, def test_version_not_set_by_default(looper, txnPoolNodeSet, - client1, client1Connected, - wallet1, - request_num): + client1, + wallet1, + request_num): req_dicts = random_requests(request_num) reqs = [Request(operation=op) for op in req_dicts] for req in reqs: @@ -44,7 +42,7 @@ def test_version_not_set_by_default(looper, txnPoolNodeSet, def test_request_with_correct_version(looper, - txnPoolNodeSet, client1, client1Connected, + txnPoolNodeSet, client1, wallet1, request_num): reqs = random_request_objects(request_num, protocol_version=CURRENT_PROTOCOL_VERSION) @@ -57,7 +55,7 @@ def test_request_with_correct_version(looper, def test_request_with_invalid_version(looper, txnPoolNodeSet, - client1, client1Connected, + client1, wallet1, request_num): reqs = random_request_objects(request_num, protocol_version=-1) diff --git a/plenum/test/client/test_state_proof_verified.py b/plenum/test/client/test_state_proof_verified.py index adfd93ca43..3f3568bb4a 100644 --- a/plenum/test/client/test_state_proof_verified.py +++ b/plenum/test/client/test_state_proof_verified.py @@ -3,10 +3,6 @@ from plenum.test.helper import sendRandomRequest, \ checkResponseCorrectnessFromNodes from stp_core.loop.eventually import eventually -# noinspection PyUnresolvedReferences -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected - nodeCount = 4 nodes_wth_bls = 4 @@ -21,7 +17,7 @@ def check_proved_reply_received(client, identifier, request_id): def test_state_proof_checked_in_client_request(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): + client1, wallet1): """ Checks that client cat use state proofs instead of quorum for replies. diff --git a/plenum/test/common/test_config_util.py b/plenum/test/common/test_config_util.py index 5bbd58a161..fb0ceb33a9 100644 --- a/plenum/test/common/test_config_util.py +++ b/plenum/test/common/test_config_util.py @@ -4,7 +4,6 @@ from plenum.common.config_util import extend_with_external_config, \ extend_with_default_external_config - TEST_NETWORK_NAME = 'test_network' GENERAL_CONFIG_FILE_NAME = 'test_config.py' USER_CONFIG_FILE_NAME = 'user_config.py' diff --git a/plenum/test/common/test_hook_mananger.py b/plenum/test/common/test_hook_mananger.py index 9443fc8e91..2fd1da65fa 100644 --- a/plenum/test/common/test_hook_mananger.py +++ b/plenum/test/common/test_hook_mananger.py @@ -12,7 +12,7 @@ def test_hook_registration(): manager.register_hook(9, lambda x, y: print(x, y)) for i in hook_ids: assert len(manager.hooks[i]) == 0 - manager.register_hook(i, lambda x, y: print(+y+i)) + manager.register_hook(i, lambda x, y: print(+y + i)) assert len(manager.hooks[i]) == 1 diff --git a/plenum/test/common/test_parse_ledger.py b/plenum/test/common/test_parse_ledger.py index 0f35fc3ca5..d451a4a779 100644 --- a/plenum/test/common/test_parse_ledger.py +++ b/plenum/test/common/test_parse_ledger.py @@ -3,7 +3,8 @@ from ledger.compact_merkle_tree import CompactMerkleTree from ledger.ledger import Ledger -from plenum.common.constants import TXN_TYPE, TARGET_NYM, DATA, NAME, ALIAS, SERVICES, VALIDATOR, IDENTIFIER, NODE_PORT, CLIENT_PORT, NODE_IP +from plenum.common.constants import TXN_TYPE, TARGET_NYM, DATA, NAME, ALIAS, SERVICES, VALIDATOR, IDENTIFIER, NODE_PORT, \ + CLIENT_PORT, NODE_IP from plenum.common.stack_manager import TxnStackManager errMsg1 = 'Invalid verkey. Rebuild pool transactions.' diff --git a/plenum/test/common/test_pool_file_raises_descriptive_error.py b/plenum/test/common/test_pool_file_raises_descriptive_error.py index 6aa9425eac..cbc601a69b 100644 --- a/plenum/test/common/test_pool_file_raises_descriptive_error.py +++ b/plenum/test/common/test_pool_file_raises_descriptive_error.py @@ -5,13 +5,12 @@ from plenum.common.stack_manager import TxnStackManager from json.decoder import JSONDecodeError - errMsg = 'Pool transaction file corrupted. Rebuild pool transactions.' whitelist = [errMsg] class DummyLedger(Ledger): - def getAllTxn(self, frm: int=None, to: int=None): + def getAllTxn(self, frm: int = None, to: int = None): raise JSONDecodeError('', '', 0) diff --git a/plenum/test/common/test_prepare_batch.py b/plenum/test/common/test_prepare_batch.py index db0f226a52..cf2be2860e 100644 --- a/plenum/test/common/test_prepare_batch.py +++ b/plenum/test/common/test_prepare_batch.py @@ -1,6 +1,5 @@ from plenum.common.prepare_batch import split_messages_on_batches, SPLIT_STEPS_LIMIT - LEN_LIMIT_BYTES = 100 SERIALIZATION_OTHER_HEAD_BYTES = 10 MAX_ONE_MSG_LEN = LEN_LIMIT_BYTES - SERIALIZATION_OTHER_HEAD_BYTES @@ -55,5 +54,5 @@ def test_one_msg_almost_excesses_limit_split_fails(): def test_excesses_limit_of_split_steps_split_fails(): - msgs = [b'1' * MAX_ONE_MSG_LEN] * 2**(SPLIT_STEPS_LIMIT + 1) + msgs = [b'1' * MAX_ONE_MSG_LEN] * 2 ** (SPLIT_STEPS_LIMIT + 1) assert split_ut(msgs) is None diff --git a/plenum/test/conftest.py b/plenum/test/conftest.py index 0f6b871e16..a021fd413d 100644 --- a/plenum/test/conftest.py +++ b/plenum/test/conftest.py @@ -229,18 +229,6 @@ def allPluginsPath(): return [getPluginPath('stats_consumer')] -@pytest.fixture(scope="module") -def keySharedNodes(startedNodes): - return startedNodes - - -@pytest.fixture(scope="module") -def startedNodes(nodeSet, looper): - for n in nodeSet: - n.start(looper.loop) - return nodeSet - - @pytest.fixture(scope="module") def whitelist(request): return getValueFromModule(request, "whitelist", []) @@ -326,15 +314,6 @@ def node_config_helper_class(): return PNodeConfigHelper -@pytest.yield_fixture(scope="module") -def nodeSet(request, tdir, tconf, nodeReg, allPluginsPath, patchPluginManager): - primaryDecider = getValueFromModule(request, "PrimaryDecider", None) - with TestNodeSet(tconf, nodeReg=nodeReg, tmpdir=tdir, - primaryDecider=primaryDecider, - pluginPaths=allPluginsPath) as ns: - yield ns - - def _tdir(tdir_fact): return tdir_fact.mktemp('').strpath @@ -419,16 +398,8 @@ def nodeReg(request) -> Dict[str, HA]: @pytest.yield_fixture(scope="module") -def unstartedLooper(nodeSet): - with Looper(nodeSet, autoStart=False) as l: - yield l - - -@pytest.fixture(scope="module") -def looper(unstartedLooper): - unstartedLooper.autoStart = True - unstartedLooper.startall() - return unstartedLooper +def looper(txnPoolNodesLooper): + yield txnPoolNodesLooper @pytest.fixture(scope="function") @@ -436,44 +407,57 @@ def pool(tdir_for_func, tconf_for_func): return Pool(tmpdir=tdir_for_func, config=tconf_for_func) -@pytest.fixture(scope="module") -def ready(looper, keySharedNodes): - looper.run(checkNodesConnected(keySharedNodes)) - return keySharedNodes - - -@pytest.fixture(scope="module") -def up(looper, ready): - ensureElectionsDone(looper=looper, nodes=ready) - - # noinspection PyIncorrectDocstring @pytest.fixture(scope="module") -def ensureView(nodeSet, looper, up): +def ensureView(txnPoolNodeSet, looper): """ - Ensure that all the nodes in the nodeSet are in the same view. + Ensure that all the nodes in the txnPoolNodeSet are in the same view. """ - return waitForViewChange(looper, nodeSet) + return waitForViewChange(looper, txnPoolNodeSet) @pytest.fixture("module") -def delayed_perf_chk(nodeSet): +def delayed_perf_chk(txnPoolNodeSet): d = 20 - for node in nodeSet: + for node in txnPoolNodeSet: node.delayCheckPerformance(d) return d @pytest.fixture(scope="module") -def clientAndWallet1(looper, nodeSet, client_tdir, up): - client, wallet = genTestClient(nodeSet, tmpdir=client_tdir) +def stewardWallet(stewardAndWallet1): + return stewardAndWallet1[1] + + +@pytest.fixture(scope="module") +def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithClientPoolTxns, client_tdir): + client, wallet = buildPoolClientAndWallet(poolTxnClientData, + client_tdir) yield client, wallet client.stop() @pytest.fixture(scope="module") -def client1(clientAndWallet1, looper): - client, _ = clientAndWallet1 +def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData, + tdirWithClientPoolTxns, client_tdir): + client, wallet = buildPoolClientAndWallet(poolTxnStewardData, + client_tdir) + yield client, wallet + client.stop() + + +@pytest.fixture(scope="module") +def steward1(looper, txnPoolNodeSet, stewardAndWallet1): + steward, wallet = stewardAndWallet1 + looper.add(steward) + ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward, + *txnPoolNodeSet) + return steward + + +@pytest.fixture(scope="module") +def client1(looper, clientAndWallet1): + client = clientAndWallet1[0] looper.add(client) looper.run(client.ensureConnectedToNodes()) return client @@ -481,8 +465,7 @@ def client1(clientAndWallet1, looper): @pytest.fixture(scope="module") def wallet1(clientAndWallet1): - _, wallet = clientAndWallet1 - return wallet + return clientAndWallet1[1] @pytest.fixture(scope="module") @@ -498,13 +481,13 @@ def sent1(client1, request1): @pytest.fixture(scope="module") -def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes): - numerOfNodes = len(nodeSet) +def reqAcked1(looper, txnPoolNodeSet, client1, sent1, faultyNodes): + numerOfNodes = len(txnPoolNodeSet) # Wait until request received by all nodes propTimeout = waits.expectedClientToPoolRequestDeliveryTime(numerOfNodes) coros = [partial(checkLastClientReqForNode, node, sent1) - for node in nodeSet] + for node in txnPoolNodeSet] # looper.run(eventuallyAll(*coros, # totalTimeout=propTimeout, # acceptableFails=faultyNodes)) @@ -518,7 +501,7 @@ def reqAcked1(looper, nodeSet, client1, sent1, faultyNodes): client1, node, sent1.identifier, - sent1.reqId) for node in nodeSet] + sent1.reqId) for node in txnPoolNodeSet] ackTimeout = waits.expectedReqAckQuorumTime() # looper.run(eventuallyAll(*coros2, # totalTimeout=ackTimeout, @@ -550,47 +533,46 @@ def faultyNodes(request): @pytest.fixture(scope="module") def propagated1(looper, - nodeSet, - up, + txnPoolNodeSet, reqAcked1, faultyNodes): - checkPropagated(looper, nodeSet, reqAcked1, faultyNodes) + checkPropagated(looper, txnPoolNodeSet, reqAcked1, faultyNodes) return reqAcked1 @pytest.fixture(scope="module") -def preprepared1(looper, nodeSet, propagated1, faultyNodes): +def preprepared1(looper, txnPoolNodeSet, propagated1, faultyNodes): checkPrePrepared(looper, - nodeSet, + txnPoolNodeSet, propagated1, - range(getNoInstances(len(nodeSet))), + range(getNoInstances(len(txnPoolNodeSet))), faultyNodes) return propagated1 @pytest.fixture(scope="module") -def prepared1(looper, nodeSet, client1, preprepared1, faultyNodes): +def prepared1(looper, txnPoolNodeSet, client1, preprepared1, faultyNodes): checkPrepared(looper, - nodeSet, + txnPoolNodeSet, preprepared1, - range(getNoInstances(len(nodeSet))), + range(getNoInstances(len(txnPoolNodeSet))), faultyNodes) return preprepared1 @pytest.fixture(scope="module") -def committed1(looper, nodeSet, client1, prepared1, faultyNodes): +def committed1(looper, txnPoolNodeSet, client1, prepared1, faultyNodes): checkCommitted(looper, - nodeSet, + txnPoolNodeSet, prepared1, - range(getNoInstances(len(nodeSet))), + range(getNoInstances(len(txnPoolNodeSet))), faultyNodes) return prepared1 @pytest.fixture(scope="module") -def replied1(looper, nodeSet, client1, committed1, wallet1, faultyNodes): - numOfNodes = len(nodeSet) +def replied1(looper, txnPoolNodeSet, client1, committed1, wallet1, faultyNodes): + numOfNodes = len(txnPoolNodeSet) numOfInstances = getNoInstances(numOfNodes) quorum = numOfInstances * (numOfNodes - faultyNodes) @@ -599,7 +581,7 @@ def checkOrderedCount(): wallet1.defaultId, committed1.reqId, instId) - for node in nodeSet for instId in range(numOfInstances)] + for node in txnPoolNodeSet for instId in range(numOfInstances)] assert resp.count(True) >= quorum orderingTimeout = waits.expectedOrderingTime(numOfInstances) diff --git a/plenum/test/delayers.py b/plenum/test/delayers.py index f1da56fa9d..6e9292e6b3 100644 --- a/plenum/test/delayers.py +++ b/plenum/test/delayers.py @@ -15,6 +15,7 @@ DEFAULT_DELAY = 600 + def delayer(seconds, op, senderFilter=None, instFilter: int = None): def inner(rx): msg, frm = rx @@ -75,42 +76,42 @@ def inner(action_pair): return inner -def nom_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str=None): +def nom_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str = None): # Delayer of NOMINATE requests return delayerMsgTuple( delay, Nomination, instFilter=inst_id, senderFilter=sender_filter) -def prim_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str=None): +def prim_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str = None): # Delayer of PRIMARY requests return delayerMsgTuple( delay, Primary, instFilter=inst_id, senderFilter=sender_filter) -def rel_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str=None): +def rel_delay(delay: float = DEFAULT_DELAY, inst_id=None, sender_filter: str = None): # Delayer of REELECTION requests return delayerMsgTuple( delay, Reelection, instFilter=inst_id, senderFilter=sender_filter) -def ppgDelay(delay: float = DEFAULT_DELAY, sender_filter: str=None): +def ppgDelay(delay: float = DEFAULT_DELAY, sender_filter: str = None): # Delayer of PROPAGATE requests return delayerMsgTuple(delay, Propagate, senderFilter=sender_filter) -def ppDelay(delay: float = DEFAULT_DELAY, instId: int=None, sender_filter: str=None): +def ppDelay(delay: float = DEFAULT_DELAY, instId: int = None, sender_filter: str = None): # Delayer of PRE-PREPARE requests from a particular instance return delayerMsgTuple(delay, PrePrepare, instFilter=instId, senderFilter=sender_filter) -def pDelay(delay: float = DEFAULT_DELAY, instId: int=None, sender_filter: str=None): +def pDelay(delay: float = DEFAULT_DELAY, instId: int = None, sender_filter: str = None): # Delayer of PREPARE requests from a particular instance return delayerMsgTuple( delay, Prepare, instFilter=instId, senderFilter=sender_filter) -def cDelay(delay: float = DEFAULT_DELAY, instId: int=None, sender_filter: str=None): +def cDelay(delay: float = DEFAULT_DELAY, instId: int = None, sender_filter: str = None): # Delayer of COMMIT requests from a particular instance return delayerMsgTuple( delay, Commit, instFilter=instId, senderFilter=sender_filter) @@ -151,7 +152,7 @@ def req_delay(delay: float = DEFAULT_DELAY): return delayerMsgTuple(delay, Request) -def msg_req_delay(delay: float = DEFAULT_DELAY, types_to_delay: List=None): +def msg_req_delay(delay: float = DEFAULT_DELAY, types_to_delay: List = None): # Delayer of MessageReq messages def specific_msgs(msg): if isinstance( @@ -163,7 +164,7 @@ def specific_msgs(msg): return specific_msgs -def msg_rep_delay(delay: float = DEFAULT_DELAY, types_to_delay: List=None): +def msg_rep_delay(delay: float = DEFAULT_DELAY, types_to_delay: List = None): # Delayer of MessageRep messages def specific_msgs(msg): if isinstance( @@ -216,7 +217,7 @@ def delay_messages(typ, nodes, inst_id, delay=None, else: RuntimeError('Unknown type') assert delay is not None or ( - min_delay is not None and max_delay is not None) + min_delay is not None and max_delay is not None) for node in nodes: if delay: d = delay diff --git a/plenum/test/exceptions.py b/plenum/test/exceptions.py index 133a432035..263a925454 100644 --- a/plenum/test/exceptions.py +++ b/plenum/test/exceptions.py @@ -1,5 +1,6 @@ class NotFullyConnected(Exception): pass + class TestException(Exception): pass diff --git a/plenum/test/grouped_load_scheduling.py b/plenum/test/grouped_load_scheduling.py index 382473817f..7664267cb9 100644 --- a/plenum/test/grouped_load_scheduling.py +++ b/plenum/test/grouped_load_scheduling.py @@ -54,4 +54,5 @@ def batch_generator(self): def grouper(p): name = self.collection[p] return '::'.join(name.split('::')[:-1]) + return itertools.groupby(self.pending, key=grouper) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index bf7ebf6aea..6b07d98ab5 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -645,16 +645,16 @@ def checkViewNoForNodes(nodes: Iterable[TestNode], expectedViewNo: int = None): return vNo -def waitForViewChange(looper, nodeSet, expectedViewNo=None, +def waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=None, customTimeout=None): """ Waits for nodes to come to same view. Raises exception when time is out """ - timeout = customTimeout or waits.expectedPoolElectionTimeout(len(nodeSet)) + timeout = customTimeout or waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) return looper.run(eventually(checkViewNoForNodes, - nodeSet, + txnPoolNodeSet, expectedViewNo, timeout=timeout)) diff --git a/plenum/test/input_validation/fields_validation/test_base58_field.py b/plenum/test/input_validation/fields_validation/test_base58_field.py index cf169d87ba..1985d34dec 100644 --- a/plenum/test/input_validation/fields_validation/test_base58_field.py +++ b/plenum/test/input_validation/fields_validation/test_base58_field.py @@ -48,4 +48,4 @@ def test_invalid_symbols_truncated_output(): b58_by_len(20)[slice(0, -len(INVALID_CHARS))] + INVALID_CHARS) assert res assert (res == 'should not contain the following chars ' - '{} (truncated)'.format(sorted(set(INVALID_CHARS))[:10])) + '{} (truncated)'.format(sorted(set(INVALID_CHARS))[:10])) diff --git a/plenum/test/input_validation/fields_validation/test_ledger_id_field.py b/plenum/test/input_validation/fields_validation/test_ledger_id_field.py index 105001633b..478512d930 100644 --- a/plenum/test/input_validation/fields_validation/test_ledger_id_field.py +++ b/plenum/test/input_validation/fields_validation/test_ledger_id_field.py @@ -2,7 +2,6 @@ from plenum.common.constants import POOL_LEDGER_ID, DOMAIN_LEDGER_ID from plenum import PLUGIN_LEDGER_IDS - validator = LedgerIdField() diff --git a/plenum/test/input_validation/fields_validation/test_time_among_field.py b/plenum/test/input_validation/fields_validation/test_time_among_field.py index 587d75dd0c..9f7320ec42 100644 --- a/plenum/test/input_validation/fields_validation/test_time_among_field.py +++ b/plenum/test/input_validation/fields_validation/test_time_among_field.py @@ -18,4 +18,4 @@ def test_empty_node_id(): def test_long_id(): - assert validator.validate(("NNooddee11::00", 1)) \ No newline at end of file + assert validator.validate(("NNooddee11::00", 1)) diff --git a/plenum/test/input_validation/fields_validation/test_version_field.py b/plenum/test/input_validation/fields_validation/test_version_field.py index 9b1a9e0e16..0b65c22860 100644 --- a/plenum/test/input_validation/fields_validation/test_version_field.py +++ b/plenum/test/input_validation/fields_validation/test_version_field.py @@ -3,7 +3,6 @@ from plenum.common.messages.fields import VersionField from plenum.config import VERSION_FIELD_LIMIT - validator = VersionField(components_number=(2, 3,), max_length=VERSION_FIELD_LIMIT) diff --git a/plenum/test/input_validation/helper.py b/plenum/test/input_validation/helper.py index c8f253adea..b256309694 100644 --- a/plenum/test/input_validation/helper.py +++ b/plenum/test/input_validation/helper.py @@ -56,10 +56,10 @@ class NonEmptyStringField(TestFieldBase): class HexString64Field(TestFieldBase): # TODO implement negative_test_cases = ( - #'', - #'fba333c13994f63edd900cdc625b88d0dcee6dda7df2c6e9b5bcd5c1072c04f', # 63 characters - #'77fba333c13994f63edd900cdc625b88d0dcee6dda7df2c6e9b5bcd5c1072c04f', # 65 characters - #'xfba333c13994f63edd900cdc625b88d0dcee6dda7df2c6e9b5bcd5c1072c04f', # first char is 'x' + # '', + # 'fba333c13994f63edd900cdc625b88d0dcee6dda7df2c6e9b5bcd5c1072c04f', # 63 characters + # '77fba333c13994f63edd900cdc625b88d0dcee6dda7df2c6e9b5bcd5c1072c04f', # 65 characters + # 'xfba333c13994f63edd900cdc625b88d0dcee6dda7df2c6e9b5bcd5c1072c04f', # first char is 'x' ) positive_test_cases = ( '7fba333c13994f63edd900cdc625b88d0dcee6dda7df2c6e9b5bcd5c1072c04f', # lower case @@ -138,16 +138,16 @@ class RequestIdrField(TestFieldBase): def negative_test_cases(self): return [ [[self.idr_field.positive_test_cases[0], - self.ts_field.negative_test_cases[0]]], + self.ts_field.negative_test_cases[0]]], [[self.idr_field.negative_test_cases[0], - self.ts_field.positive_test_cases[0]]], + self.ts_field.positive_test_cases[0]]], ] @property def positive_test_cases(self): return [ [[self.idr_field.positive_test_cases[0], - self.ts_field.positive_test_cases[0]]], + self.ts_field.positive_test_cases[0]]], ] @@ -160,16 +160,16 @@ class TieAmongField(TestFieldBase): def negative_test_cases(self): return [ [self.name_field.positive_test_cases[0], - self.ts_field.negative_test_cases[0]], + self.ts_field.negative_test_cases[0]], [self.name_field.negative_test_cases[0], - self.ts_field.positive_test_cases[0]], + self.ts_field.positive_test_cases[0]], ] @property def positive_test_cases(self): return [ [self.name_field.positive_test_cases[0], - self.ts_field.positive_test_cases[0]], + self.ts_field.positive_test_cases[0]], ] @@ -247,7 +247,7 @@ def positive_test_cases(self): class MessageDescriptor(TestFieldBase): - field_types = (dict, ) + field_types = (dict,) def __init__(self, klass, fields, optional_fields=None, name=None): self.klass = klass diff --git a/plenum/test/input_validation/message_validation/test_catchuprep_message.py b/plenum/test/input_validation/message_validation/test_catchuprep_message.py index 51229e92ad..96b8560ed7 100644 --- a/plenum/test/input_validation/message_validation/test_catchuprep_message.py +++ b/plenum/test/input_validation/message_validation/test_catchuprep_message.py @@ -3,7 +3,6 @@ from plenum.common.messages.fields import \ IterableField, LedgerIdField, MapField - EXPECTED_ORDERED_FIELDS = OrderedDict([ ("ledgerId", LedgerIdField), ("txns", AnyValueField), diff --git a/plenum/test/input_validation/message_validation/test_catchupreq_message.py b/plenum/test/input_validation/message_validation/test_catchupreq_message.py index c49482a5f7..f69c7b8529 100644 --- a/plenum/test/input_validation/message_validation/test_catchupreq_message.py +++ b/plenum/test/input_validation/message_validation/test_catchupreq_message.py @@ -4,7 +4,6 @@ from plenum.common.messages.fields import \ NonNegativeNumberField, LedgerIdField - EXPECTED_ORDERED_FIELDS = OrderedDict([ ("ledgerId", LedgerIdField), ("seqNoStart", NonNegativeNumberField), diff --git a/plenum/test/input_validation/message_validation/test_checkpoint_message.py b/plenum/test/input_validation/message_validation/test_checkpoint_message.py index 18e2d6a499..f886fc31a9 100644 --- a/plenum/test/input_validation/message_validation/test_checkpoint_message.py +++ b/plenum/test/input_validation/message_validation/test_checkpoint_message.py @@ -4,7 +4,6 @@ from plenum.common.messages.fields import \ NonNegativeNumberField, LimitedLengthStringField - EXPECTED_ORDERED_FIELDS = OrderedDict([ ("instId", NonNegativeNumberField), ("viewNo", NonNegativeNumberField), diff --git a/plenum/test/input_validation/message_validation/test_threepcstate_message.py b/plenum/test/input_validation/message_validation/test_threepcstate_message.py index ba24fe1178..f26669fc2f 100644 --- a/plenum/test/input_validation/message_validation/test_threepcstate_message.py +++ b/plenum/test/input_validation/message_validation/test_threepcstate_message.py @@ -3,7 +3,6 @@ from plenum.common.messages.fields import \ NonNegativeNumberField, IterableField - EXPECTED_ORDERED_FIELDS = OrderedDict([ ("instId", NonNegativeNumberField), ("messages", IterableField), diff --git a/plenum/test/instances/test_prepare_digest.py b/plenum/test/instances/test_prepare_digest.py index a726c91bfe..b9692a8629 100644 --- a/plenum/test/instances/test_prepare_digest.py +++ b/plenum/test/instances/test_prepare_digest.py @@ -11,7 +11,6 @@ send3PhaseMsgWithIncorrectDigest from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.pool_transactions.conftest import looper from plenum.test.node_request.conftest import committed1, \ prepared1, preprepared1, propagated1, reqAcked1, \ sent1, noRetryReq, faultyNodes diff --git a/plenum/test/logging/conftest.py b/plenum/test/logging/conftest.py index 534a8a73d5..24c7b5f642 100644 --- a/plenum/test/logging/conftest.py +++ b/plenum/test/logging/conftest.py @@ -39,14 +39,14 @@ def wrapper(levels=None, files=None, funcs=None, msgs=None): class TestingFilter(logging.Filter): def filter(self, record): return ( - (levels is None or - record.levelname in levels) and - (files is None or - record.filename in files) and - (funcs is None or - record.funcName in funcs) and - (reMsgs is None or - reMsgs.search(record.getMessage()) is not None) + (levels is None or + record.levelname in levels) and + (files is None or + record.filename in files) and + (funcs is None or + record.funcName in funcs) and + (reMsgs is None or + reMsgs.search(record.getMessage()) is not None) ) def tester(record): diff --git a/plenum/test/logging/test_logging_txn_state.py b/plenum/test/logging/test_logging_txn_state.py index b3ee33dd23..b28a02c2ad 100644 --- a/plenum/test/logging/test_logging_txn_state.py +++ b/plenum/test/logging/test_logging_txn_state.py @@ -15,7 +15,6 @@ from plenum.test import waits from plenum.test.helper import sdk_send_random_and_check, sdk_get_and_check_replies - ERORR_MSG = "something went wrong" diff --git a/plenum/test/malicious_behaviors_client.py b/plenum/test/malicious_behaviors_client.py index 6e9395342d..a926458b8b 100644 --- a/plenum/test/malicious_behaviors_client.py +++ b/plenum/test/malicious_behaviors_client.py @@ -82,5 +82,6 @@ def sendsUnsignedRequest(client) -> Client: def evilSign(self, msg, signer) -> Mapping: logger.debug("EVIL: client doesn't sign any of the requests") return msg + client.nodestack.sign = types.MethodType(evilSign, client) return client diff --git a/plenum/test/malicious_behaviors_node.py b/plenum/test/malicious_behaviors_node.py index a5ca4e0888..d9fcd2c3c9 100644 --- a/plenum/test/malicious_behaviors_node.py +++ b/plenum/test/malicious_behaviors_node.py @@ -44,11 +44,11 @@ def evilCreatePropagate(self, return node -def delaysPrePrepareProcessing(node, delay: float=30, instId: int=None): +def delaysPrePrepareProcessing(node, delay: float = 30, instId: int = None): node.nodeIbStasher.delay(ppDelay(delay=delay, instId=instId)) -def delaysCommitProcessing(node, delay: float=30, instId: int=None): +def delaysCommitProcessing(node, delay: float = 30, instId: int = None): node.nodeIbStasher.delay(cDelay(delay=delay, instId=instId)) @@ -57,7 +57,7 @@ def delaysCommitProcessing(node, delay: float=30, instId: int=None): def sendDuplicate3PhaseMsg( node: TestNode, msgType: ThreePhaseMsg, - count: int=2, + count: int = 2, instId=None): def evilSendPrePrepareRequest(self, ppReq: PrePrepare): logger.debug("EVIL: Sending duplicate pre-prepare message: {}". @@ -118,7 +118,7 @@ def malign3PhaseSendingMethod(replica: TestReplica, msgType: ThreePhaseMsg, common.error.error("Not a 3 phase message") -def malignInstancesOfNode(node: TestNode, malignMethod, instId: int=None): +def malignInstancesOfNode(node: TestNode, malignMethod, instId: int = None): if instId is not None: malignMethod(replica=node.replicas[instId]) else: @@ -129,7 +129,7 @@ def malignInstancesOfNode(node: TestNode, malignMethod, instId: int=None): def send3PhaseMsgWithIncorrectDigest(node: TestNode, msgType: ThreePhaseMsg, - instId: int=None): + instId: int = None): def evilSendPrePrepareRequest(self, ppReq: PrePrepare): logger.debug("EVIL: Creating pre-prepare message for request : {}". format(ppReq)) @@ -183,6 +183,7 @@ def newGenerateReply(self, viewNo: int, req: Request) -> Reply: reply.result[f.SIG.nm] = "incorrect signature" reply.result["declaration"] = "All your base are belong to us." return reply + node.generateReply = types.MethodType(newGenerateReply, node) diff --git a/plenum/test/monitoring/test_avg_latency.py b/plenum/test/monitoring/test_avg_latency.py index 045cd12eeb..b17137822d 100644 --- a/plenum/test/monitoring/test_avg_latency.py +++ b/plenum/test/monitoring/test_avg_latency.py @@ -2,7 +2,6 @@ from plenum.test.helper import sdk_send_random_and_check from plenum.test.pool_transactions.conftest import looper - nodeCount = 4 logger = getlogger() diff --git a/plenum/test/monitoring/test_instance_change_with_Delta.py b/plenum/test/monitoring/test_instance_change_with_Delta.py index 4e4b1b52b6..f7895cee38 100644 --- a/plenum/test/monitoring/test_instance_change_with_Delta.py +++ b/plenum/test/monitoring/test_instance_change_with_Delta.py @@ -12,7 +12,6 @@ from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually - nodeCount = 7 whitelist = ["discarding message"] @@ -24,6 +23,7 @@ verify a view change happens """ + @pytest.fixture def logger(): logger = getlogger() @@ -32,9 +32,11 @@ def logger(): yield logger logger.root.setLevel(old_value) + # autouse and inject before others in all tests pytestmark = pytest.mark.usefixtures("logger") + def latestPerfChecks(nodes): """ Returns spylog entry for most recent checkPerformance executions for a set diff --git a/plenum/test/monitoring/test_instance_change_with_req_Lambda.py b/plenum/test/monitoring/test_instance_change_with_req_Lambda.py index b51a7a58c5..dced47af86 100644 --- a/plenum/test/monitoring/test_instance_change_with_req_Lambda.py +++ b/plenum/test/monitoring/test_instance_change_with_req_Lambda.py @@ -26,9 +26,8 @@ @pytest.fixture() def setup(looper, tconf, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): - sdk_send_random_and_check(looper, txnPoolNodeSet, - sdk_pool_handle, sdk_wallet_client, 5) + sdk_pool_handle, sdk_wallet_client, 5) P = getPrimaryReplica(txnPoolNodeSet) # set LAMBDA smaller than the production config to make the test faster @@ -55,8 +54,8 @@ def specificPrePrepare(msg): P.outBoxTestStasher.delay(specificPrePrepare) # TODO select or create a timeout for this case in 'waits' sdk_send_random_and_check(looper, txnPoolNodeSet, - sdk_pool_handle, sdk_wallet_client, 5, - customTimeoutPerReq=tconf.TestRunningTimeLimitSec) + sdk_pool_handle, sdk_wallet_client, 5, + customTimeoutPerReq=tconf.TestRunningTimeLimitSec) return adict(nodes=txnPoolNodeSet) diff --git a/plenum/test/monitoring/test_no_check_if_no_new_requests.py b/plenum/test/monitoring/test_no_check_if_no_new_requests.py index 0ef44b7f3a..54755d4f53 100644 --- a/plenum/test/monitoring/test_no_check_if_no_new_requests.py +++ b/plenum/test/monitoring/test_no_check_if_no_new_requests.py @@ -15,7 +15,7 @@ def test_not_check_if_no_new_requests(perf_chk_patched, looper, txnPoolNodeSet, Checks that node does not do performance check if there were no new requests since previous check """ - + # Ensure that nodes participating, because otherwise they do not do check for node in txnPoolNodeSet: assert node.isParticipating diff --git a/plenum/test/monitoring/test_post_monitoring_stats.py b/plenum/test/monitoring/test_post_monitoring_stats.py index c5776fc364..5d33b84b23 100644 --- a/plenum/test/monitoring/test_post_monitoring_stats.py +++ b/plenum/test/monitoring/test_post_monitoring_stats.py @@ -5,7 +5,6 @@ from plenum.test.helper import sdk_send_random_and_check - def testPostingThroughput(postingStatsEnabled, decreasedMonitoringTimeouts, looper, @@ -32,10 +31,10 @@ def testPostingThroughput(postingStatsEnabled, assert node.monitor.totalRequests == 0 sdk_send_random_and_check(looper, - txnPoolNodeSet, - sdk_pool_handle, - sdk_wallet_client, - reqCount) + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + reqCount) for node in txnPoolNodeSet: assert len(node.monitor.orderedRequestsInLast) == reqCount @@ -88,10 +87,10 @@ def testPostingLatency(postingStatsEnabled, assert node.monitor.avgBackupLatency == 0 sdk_send_random_and_check(looper, - txnPoolNodeSet, - sdk_pool_handle, - sdk_wallet_client, - reqCount) + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + reqCount) for node in txnPoolNodeSet: assert node.monitor.masterLatency > 0 diff --git a/plenum/test/monitoring/test_stats_publisher.py b/plenum/test/monitoring/test_stats_publisher.py index 792d8559e9..8d2aeff8fe 100644 --- a/plenum/test/monitoring/test_stats_publisher.py +++ b/plenum/test/monitoring/test_stats_publisher.py @@ -128,7 +128,7 @@ def testSendManyNoExceptionsIfDestPortFromSourceRange(): statsPublisher.send(message="testMessage{}".format(i)) assert N == len(statsPublisher.refused) + \ - len(statsPublisher.unexpected) + len(statsPublisher.sent) + len(statsPublisher.unexpected) + len(statsPublisher.sent) class TestStatsPublisher(StatsPublisher): diff --git a/plenum/test/monitoring/test_system_stats.py b/plenum/test/monitoring/test_system_stats.py index be3ec66332..996c1af71b 100644 --- a/plenum/test/monitoring/test_system_stats.py +++ b/plenum/test/monitoring/test_system_stats.py @@ -50,4 +50,4 @@ def test_traffic(): assert data2['cpu']['value'] == cpu assert data2['ram']['value'] == ram assert data2['traffic']['value'] == bytes / \ - 1024 - data1['traffic']['value'] + 1024 - data1['traffic']['value'] diff --git a/plenum/test/monitoring/test_throughput.py b/plenum/test/monitoring/test_throughput.py index 463fec02d6..f2f9005601 100644 --- a/plenum/test/monitoring/test_throughput.py +++ b/plenum/test/monitoring/test_throughput.py @@ -3,7 +3,6 @@ from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_random_and_check - nodeCount = 4 logger = getlogger() diff --git a/plenum/test/monitoring/test_warn_unordered_log_msg.py b/plenum/test/monitoring/test_warn_unordered_log_msg.py index 54513e3e11..e4ccb93584 100644 --- a/plenum/test/monitoring/test_warn_unordered_log_msg.py +++ b/plenum/test/monitoring/test_warn_unordered_log_msg.py @@ -6,7 +6,6 @@ from plenum.test.helper import sendRandomRequest, \ waitForSufficientRepliesForRequests - nodeCount = 4 logger = getlogger() @@ -36,7 +35,7 @@ def test_slow_node_has_warn_unordered_log_msg(looper, monitor = nodeSet[0].monitor delay = monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC * \ - monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM + 10 + monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM + 10 delaysCommitProcessing(slow_node, delay=delay) assert no_any_warn(*nodeSet), \ diff --git a/plenum/test/node_catchup/conftest.py b/plenum/test/node_catchup/conftest.py index 2ebeb03ede..3ca7b18a07 100644 --- a/plenum/test/node_catchup/conftest.py +++ b/plenum/test/node_catchup/conftest.py @@ -51,7 +51,7 @@ def nodeCreatedAfterSomeTxns(looper, testNodeClass, do_post_node_creation, allPluginsPath=allPluginsPath, autoStart=True, do_post_node_creation=do_post_node_creation) yield looper, newNode, client, wallet, newStewardClient, \ - newStewardWallet + newStewardWallet @pytest.fixture("module") diff --git a/plenum/test/node_catchup/helper.py b/plenum/test/node_catchup/helper.py index 59b19e5ea9..1ed9b2cf13 100644 --- a/plenum/test/node_catchup/helper.py +++ b/plenum/test/node_catchup/helper.py @@ -16,7 +16,6 @@ from plenum.test import waits import pytest - logger = getlogger() @@ -25,7 +24,6 @@ def checkNodeDataForEquality(node: TestNode, *otherNodes: TestNode, exclude_from_check=None): - def chk_ledger_and_state(first_node, second_node, ledger_id): checkLedgerEquality(first_node.getLedger(ledger_id), second_node.getLedger(ledger_id)) diff --git a/plenum/test/node_catchup/test_catchup_demoted.py b/plenum/test/node_catchup/test_catchup_demoted.py index 1fcc1d2e2b..9f6734421a 100644 --- a/plenum/test/node_catchup/test_catchup_demoted.py +++ b/plenum/test/node_catchup/test_catchup_demoted.py @@ -17,7 +17,7 @@ def test_catch_up_after_demoted( "1. add a new node after sending some txns and check that catch-up " "is done (the new node is up to date)") looper, newNode, client, wallet, newStewardClient, \ - newStewardWallet = nodeSetWithNodeAddedAfterSomeTxns + newStewardWallet = nodeSetWithNodeAddedAfterSomeTxns waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:4]) logger.info("2. turn the new node off (demote)") diff --git a/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py b/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py index 2f3565f7ac..b64478dd7c 100644 --- a/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py +++ b/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py @@ -9,7 +9,6 @@ from plenum.test.test_client import TestClient from stp_core.loop.eventually import eventually - TestRunningTimeLimitSec = 125 diff --git a/plenum/test/node_catchup/test_catchup_scenarios.py b/plenum/test/node_catchup/test_catchup_scenarios.py index f0b1b4f700..e6409e3497 100644 --- a/plenum/test/node_catchup/test_catchup_scenarios.py +++ b/plenum/test/node_catchup/test_catchup_scenarios.py @@ -10,7 +10,6 @@ from plenum.test.test_node import checkNodesConnected from plenum.test import waits - logger = getlogger() txnCount = 10 diff --git a/plenum/test/node_catchup/test_config_ledger.py b/plenum/test/node_catchup/test_config_ledger.py index 07b03f0841..aab21bb1fc 100644 --- a/plenum/test/node_catchup/test_config_ledger.py +++ b/plenum/test/node_catchup/test_config_ledger.py @@ -39,13 +39,13 @@ def read(key, looper, sdk_pool_handle, sdk_wallet): for op in [read_conf_op(key)]] reqs = sdk_sign_request_objects(looper, sdk_wallet, reqs_obj) sent_reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) - (req, resp), = sdk_get_replies(looper, sent_reqs, timeout=10) + (req, resp), = sdk_get_replies(looper, sent_reqs, timeout=10) return json.loads(resp['result'][DATA])[key] def send_some_config_txns(looper, sdk_pool_handle, sdk_wallet_client, keys): for i in range(5): - key, val = 'key_{}'.format(i+1), randomString() + key, val = 'key_{}'.format(i + 1), randomString() write(key, val, looper, sdk_pool_handle, sdk_wallet_client) keys[key] = val return keys @@ -119,7 +119,7 @@ def test_new_node_catchup_config_ledger(looper, some_config_txns_done, A new node catches up the config ledger too """ assert len(newNodeCaughtUp.getLedger(CONFIG_LEDGER_ID)) >= \ - len(some_config_txns_done) + len(some_config_txns_done) def test_disconnected_node_catchup_config_ledger_txns(looper, diff --git a/plenum/test/node_catchup/test_discard_view_no.py b/plenum/test/node_catchup/test_discard_view_no.py index 2549da57c1..658ac45c99 100644 --- a/plenum/test/node_catchup/test_discard_view_no.py +++ b/plenum/test/node_catchup/test_discard_view_no.py @@ -11,7 +11,6 @@ checkProtocolInstanceSetup, getPrimaryReplica from plenum.test import waits - whitelist = ['found legacy entry'] # warnings diff --git a/plenum/test/node_catchup/test_large_catchup.py b/plenum/test/node_catchup/test_large_catchup.py index 6665547c8f..067394f0bb 100644 --- a/plenum/test/node_catchup/test_large_catchup.py +++ b/plenum/test/node_catchup/test_large_catchup.py @@ -9,7 +9,6 @@ from stp_core.validators.message_length_validator import MessageLenValidator - TestRunningTimeLimitSec = 125 diff --git a/plenum/test/node_catchup/test_new_node_catchup.py b/plenum/test/node_catchup/test_new_node_catchup.py index 331a408597..d10480ea50 100644 --- a/plenum/test/node_catchup/test_new_node_catchup.py +++ b/plenum/test/node_catchup/test_new_node_catchup.py @@ -43,7 +43,7 @@ def testPoolLegerCatchupBeforeDomainLedgerCatchup(txnPoolNodeSet, for comp in completes: completionTimes[comp.params.get('ledgerId')] = comp.endtime assert startTimes[0] < completionTimes[0] < \ - startTimes[1] < completionTimes[1] + startTimes[1] < completionTimes[1] @pytest.mark.skip(reason="SOV-554. " diff --git a/plenum/test/node_catchup/test_new_node_catchup2.py b/plenum/test/node_catchup/test_new_node_catchup2.py index e6a0b5bfb0..47770cb6b6 100644 --- a/plenum/test/node_catchup/test_new_node_catchup2.py +++ b/plenum/test/node_catchup/test_new_node_catchup2.py @@ -46,8 +46,8 @@ def testNodeDoesNotParticipateUntilCaughtUp(txnPoolNodeSet, node.reset_delays_and_process_delayeds() timeout = waits.expectedPoolCatchupTime(len(txnPoolNodeSet)) + \ - catchup_delay + \ - waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + catchup_delay + \ + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=timeout) waitNodeDataEquality(looper, new_node, *old_nodes) diff --git a/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py b/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py index 55688551ca..e78dcbbf8d 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py +++ b/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py @@ -10,7 +10,6 @@ from plenum.test.node_catchup.helper import waitNodeDataInequality, waitNodeDataEquality from plenum.test.test_node import getNonPrimaryReplicas - logger = getLogger() TestRunningTimeLimitSec = 200 diff --git a/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py b/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py index ab8e414f21..0f56311ea5 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py +++ b/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py @@ -118,7 +118,7 @@ def send_and_chk(ledger_state): # Not accurate timeout but a conservative one timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \ - 2 * delay_catchup_reply + 2 * delay_catchup_reply waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1], customTimeout=timeout) assert new_node_ledger.num_txns_caught_up == more_requests diff --git a/plenum/test/node_catchup/test_node_catchup_after_restart_no_txns.py b/plenum/test/node_catchup/test_node_catchup_after_restart_no_txns.py index 2f41225732..805024236c 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_restart_no_txns.py +++ b/plenum/test/node_catchup/test_node_catchup_after_restart_no_txns.py @@ -1,4 +1,3 @@ - import pytest from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState diff --git a/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py b/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py index ce50d98176..76734adf2c 100644 --- a/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py +++ b/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py @@ -21,7 +21,6 @@ def make_master_replica_lag(node): - node.nodeIbStasher.delay(ppDelay(1200, 0)) node.nodeIbStasher.delay(pDelay(1200, 0)) node.nodeIbStasher.delay(cDelay(1200, 0)) diff --git a/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py b/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py index 1774bae58d..5ec0825698 100644 --- a/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py +++ b/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py @@ -16,7 +16,6 @@ from plenum.common.startable import Mode from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected - logger = getlogger() @@ -26,7 +25,7 @@ def catchuped(node): def test_node_catchup_when_3_not_primary_node_restarted( looper, txnPoolNodeSet, tdir, tconf, - allPluginsPath, steward1, stewardWallet): + allPluginsPath, steward1, stewardWallet): """ Test case: 1. Create pool of 4 nodes diff --git a/plenum/test/node_catchup/test_node_ledger_manager.py b/plenum/test/node_catchup/test_node_ledger_manager.py index 8b2358c105..a9e586169f 100644 --- a/plenum/test/node_catchup/test_node_ledger_manager.py +++ b/plenum/test/node_catchup/test_node_ledger_manager.py @@ -9,6 +9,6 @@ def test_ledger_sync_order(looper, txnPoolNodeSet): for idx, lid in enumerate(ledger_ids): next_ledger_id = node.ledgerManager.ledger_to_sync_after(lid) if idx != (len(ledger_ids) - 1): - assert next_ledger_id == ledger_ids[idx+1] + assert next_ledger_id == ledger_ids[idx + 1] else: assert next_ledger_id is None diff --git a/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py b/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py index 02c698ae64..9488298023 100644 --- a/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py +++ b/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py @@ -10,13 +10,11 @@ from plenum.test.test_node import checkNodesConnected, getNonPrimaryReplicas from plenum.test import waits - # Do not remove the next import from plenum.test.node_catchup.conftest import whitelist logger = getlogger() - txnCount = 10 @@ -81,7 +79,7 @@ def _sendIncorrectTxns(self, req, frm): " for catchup request {} from {}". format(self, req, frm)) start, end = getattr(req, f.SEQ_NO_START.nm), \ - getattr(req, f.SEQ_NO_END.nm) + getattr(req, f.SEQ_NO_END.nm) ledger = self.getLedgerForMsg(req) txns = {} for seqNo, txn in ledger.getAllTxn(start, end): diff --git a/plenum/test/node_catchup/test_node_request_consistency_proof.py b/plenum/test/node_catchup/test_node_request_consistency_proof.py index 3a08a636b2..8f22946715 100644 --- a/plenum/test/node_catchup/test_node_request_consistency_proof.py +++ b/plenum/test/node_catchup/test_node_request_consistency_proof.py @@ -14,7 +14,6 @@ from plenum.test.node_catchup.conftest import whitelist from plenum.test.batching_3pc.conftest import tconf - logger = getlogger() # So that `three_phase_key_for_txn_seq_no` always works, it makes the test # easy as the requesting node selects a random size for the ledger diff --git a/plenum/test/node_catchup/test_node_request_missing_transactions.py b/plenum/test/node_catchup/test_node_request_missing_transactions.py index 70d194af65..8190d3ac55 100644 --- a/plenum/test/node_catchup/test_node_request_missing_transactions.py +++ b/plenum/test/node_catchup/test_node_request_missing_transactions.py @@ -13,7 +13,6 @@ # Do not remove the next import from plenum.test.node_catchup.conftest import whitelist - logger = getlogger() TestRunningTimeLimitSec = 180 @@ -68,7 +67,7 @@ def ignoreCatchupReq(self, req, frm): # Since one of the nodes does not reply, this new node will experience a # timeout and retry catchup requests, hence a long test timeout. timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \ - reduced_catchup_timeout_conf.CatchupTransactionsTimeout + reduced_catchup_timeout_conf.CatchupTransactionsTimeout waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1], customTimeout=timeout) new_size = len(new_node_ledger.ledger) @@ -76,7 +75,7 @@ def ignoreCatchupReq(self, req, frm): # The new node ledger might catchup some transactions from the batch of # `more_request` transactions assert old_size_others - \ - old_size <= new_node_ledger.num_txns_caught_up <= new_size - old_size + old_size <= new_node_ledger.num_txns_caught_up <= new_size - old_size sendRandomRequests(wallet, client, 2) waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1], customTimeout=timeout) diff --git a/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py b/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py index 95589ef066..786f6fc1d2 100644 --- a/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py +++ b/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py @@ -38,12 +38,12 @@ def test_nodes_removes_request_keys_for_ordered(setup, looper, txnPoolNodeSet, looper, wallet1, client1, 10, 5) ensure_all_nodes_have_same_data(looper, fast_nodes) assert slow_node.master_replica.last_ordered_3pc != \ - fast_nodes[0].master_replica.last_ordered_3pc + fast_nodes[0].master_replica.last_ordered_3pc def chk(key, nodes, present): for node in nodes: assert ( - key in node.master_replica.requestQueues[DOMAIN_LEDGER_ID]) == present + key in node.master_replica.requestQueues[DOMAIN_LEDGER_ID]) == present for req in reqs: chk(req.key, fast_nodes, False) diff --git a/plenum/test/node_catchup/test_req_id_key_error.py b/plenum/test/node_catchup/test_req_id_key_error.py index ee09f952c8..b1651ae759 100644 --- a/plenum/test/node_catchup/test_req_id_key_error.py +++ b/plenum/test/node_catchup/test_req_id_key_error.py @@ -7,11 +7,11 @@ def test_req_id_key_error(testNode, wallet1): - #create random transactions + # create random transactions count_of_txn = 3 reqs = signed_random_requests(wallet1, count_of_txn) txns = [] - #prepare transactions and remove reqId from + # prepare transactions and remove reqId from for i, req in enumerate(reqs): txnreq = reqToTxn(req) txnreq[f.SEQ_NO.nm] = i diff --git a/plenum/test/node_catchup/test_revert_during_catchup.py b/plenum/test/node_catchup/test_revert_during_catchup.py index 7814491adb..c18a4530e9 100644 --- a/plenum/test/node_catchup/test_revert_during_catchup.py +++ b/plenum/test/node_catchup/test_revert_during_catchup.py @@ -17,7 +17,7 @@ TestRunningTimeLimitSec = 125 # Do not remove the next imports -from plenum.test.batching_3pc.conftest import tconf # noqa +from plenum.test.batching_3pc.conftest import tconf # noqa def test_slow_node_reverts_unordered_state_during_catchup(looper, @@ -62,6 +62,7 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, def is_catchup_needed_count(): return len(getAllReturnVals(slow_node, slow_node.is_catchup_needed, compare_val_to=True)) + old_lcu_count = slow_node.spylog.count(slow_node.allLedgersCaughtUp) old_cn_count = is_catchup_needed_count() diff --git a/plenum/test/node_request/message_request/conftest.py b/plenum/test/node_request/message_request/conftest.py index 72e6240ce6..7d793b407c 100644 --- a/plenum/test/node_request/message_request/conftest.py +++ b/plenum/test/node_request/message_request/conftest.py @@ -7,7 +7,6 @@ @pytest.fixture(scope="module") def teardown(request, looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): - def tear(): # Repair any broken network for node in txnPoolNodeSet: diff --git a/plenum/test/node_request/message_request/test_node_request_missing_three_phase_messages.py b/plenum/test/node_request/message_request/test_node_request_missing_three_phase_messages.py index ff8399ff6f..b6d76ba112 100644 --- a/plenum/test/node_request/message_request/test_node_request_missing_three_phase_messages.py +++ b/plenum/test/node_request/message_request/test_node_request_missing_three_phase_messages.py @@ -7,7 +7,6 @@ from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_random_requests, sdk_send_random_and_check - logger = getlogger() diff --git a/plenum/test/node_request/message_request/test_node_requests_missing_preprepare.py b/plenum/test/node_request/message_request/test_node_requests_missing_preprepare.py index 5d41f2c021..2c9fa11ce1 100644 --- a/plenum/test/node_request/message_request/test_node_requests_missing_preprepare.py +++ b/plenum/test/node_request/message_request/test_node_requests_missing_preprepare.py @@ -14,7 +14,6 @@ from plenum.test.test_node import getNonPrimaryReplicas, get_master_primary_node from plenum.test.pool_transactions.conftest import looper - whitelist = ['does not have expected state'] @@ -87,7 +86,7 @@ def do_not_send(self, msg, frm): bad_node.nodeMsgRouter.routes[MessageReq] = types.MethodType( do_not_send, bad_node) return primary_node, bad_node, good_non_primary_node, slow_node, \ - other_nodes, do_not_send, orig_method + other_nodes, do_not_send, orig_method if request.param == 'send_bad': orig_method = bad_node.nodeMsgRouter.routes[MessageReq] @@ -108,7 +107,7 @@ def send_bad(self, msg, frm): bad_node.nodeMsgRouter.routes[MessageReq] = types.MethodType(send_bad, bad_node) return primary_node, bad_node, good_non_primary_node, slow_node, \ - other_nodes, send_bad, orig_method + other_nodes, send_bad, orig_method def test_node_requests_missing_preprepare_malicious(looper, txnPoolNodeSet, @@ -127,20 +126,20 @@ def test_node_requests_missing_preprepare_malicious(looper, txnPoolNodeSet, # good_non_primary_node = [n for n in other_nodes if n != slow_node # and n != bad_node and n != primary_node][0] primary_node, bad_node, good_non_primary_node, slow_node, other_nodes, \ - bad_method, orig_method = malicious_setup + bad_method, orig_method = malicious_setup slow_node.nodeIbStasher.delay(ppDelay(300, 0)) def get_reply_count_frm(node): return sum([1 for entry in slow_node.spylog.getAll( slow_node.process_message_rep) - if entry.params['msg'].msg_type == PREPREPARE and - entry.params['frm'] == node.name]) + if entry.params['msg'].msg_type == PREPREPARE and + entry.params['frm'] == node.name]) old_reply_count_from_bad_node = get_reply_count_frm(bad_node) old_reply_count_from_good_node = get_reply_count_frm(good_non_primary_node) old_discarded = countDiscarded(slow_node.master_replica, 'does not have ' - 'expected state') + 'expected state') sdk_send_batches_of_random_and_check(looper, txnPoolNodeSet, @@ -163,7 +162,7 @@ def get_reply_count_frm(node): 'does not have expected state') > old_discarded assert get_reply_count_frm(good_non_primary_node) > \ - old_reply_count_from_good_node + old_reply_count_from_good_node slow_node.reset_delays_and_process_delayeds() bad_node.nodeMsgRouter.routes[MessageReq] = orig_method diff --git a/plenum/test/node_request/message_request/test_preprepare_request.py b/plenum/test/node_request/message_request/test_preprepare_request.py index e878d77f37..18cab89768 100644 --- a/plenum/test/node_request/message_request/test_preprepare_request.py +++ b/plenum/test/node_request/message_request/test_preprepare_request.py @@ -31,7 +31,7 @@ def test_node_request_preprepare(looper, txnPoolNodeSet, Node requests PRE-PREPARE only once after getting PREPAREs. """ slow_node, other_nodes, primary_node, \ - other_primary_nodes = split_nodes(txnPoolNodeSet) + other_primary_nodes = split_nodes(txnPoolNodeSet) # Drop PrePrepares and Prepares slow_node.nodeIbStasher.delay(ppDelay(300, 0)) slow_node.nodeIbStasher.delay(pDelay(300, 0)) diff --git a/plenum/test/node_request/message_request/test_valid_message_request.py b/plenum/test/node_request/message_request/test_valid_message_request.py index e173e53f44..53733035e1 100644 --- a/plenum/test/node_request/message_request/test_valid_message_request.py +++ b/plenum/test/node_request/message_request/test_valid_message_request.py @@ -10,15 +10,12 @@ from plenum.test.helper import countDiscarded from stp_core.loop.eventually import eventually - invalid_type_discard_log = "unknown value 'invalid_type'" invalid_req_discard_log = "cannot serve request" invalid_rep_discard_log = "cannot process requested message response" - whitelist = [invalid_type_discard_log, ] - patched_schema = ( (f.MSG_TYPE.nm, ChooseField(values={'invalid_type', LEDGER_STATUS, CONSISTENCY_PROOF, PREPREPARE, @@ -30,6 +27,7 @@ def patched_MessageReq(): class PMessageReq(MessageReq): schema = patched_schema + return PMessageReq @@ -39,6 +37,7 @@ class PMessageRep(MessageRep): *patched_schema, (f.MSG.nm, AnyField()) ) + return PMessageRep diff --git a/plenum/test/node_request/node_request_helper.py b/plenum/test/node_request/node_request_helper.py index ab258e4601..a0f24c41f7 100644 --- a/plenum/test/node_request/node_request_helper.py +++ b/plenum/test/node_request/node_request_helper.py @@ -97,7 +97,7 @@ def nonPrimarySeesCorrectNumberOfPREPREPAREs(): param['pre_prepare'][4:], param['sender']) == ( expectedPrePrepareRequest[0:3] + - expectedPrePrepareRequest[4:], + expectedPrePrepareRequest[4:], primary.name)]) numOfMsgsWithZFN = 1 @@ -257,14 +257,14 @@ def nonPrimaryReplicasReceiveCorrectNumberOfPREPAREs(): actualMsgs = len( [ param for param in getAllArgs( - npr, - npr.processPrepare) if ( - param['prepare'].instId, - param['prepare'].viewNo, - param['prepare'].ppSeqNo) == ( - primary.instId, - primary.viewNo, - primary.lastPrePrepareSeqNo)]) + npr, + npr.processPrepare) if ( + param['prepare'].instId, + param['prepare'].viewNo, + param['prepare'].ppSeqNo) == ( + primary.instId, + primary.viewNo, + primary.lastPrePrepareSeqNo)]) passes += int(msgCountOK(nodeCount, faultyNodes, diff --git a/plenum/test/node_request/test_belated_request_not_processed.py b/plenum/test/node_request/test_belated_request_not_processed.py index 59c043f9a4..1dbb51ff5d 100644 --- a/plenum/test/node_request/test_belated_request_not_processed.py +++ b/plenum/test/node_request/test_belated_request_not_processed.py @@ -9,7 +9,6 @@ def test_repeated_request_not_processed_if_already_ordered( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size @@ -25,7 +24,6 @@ def test_repeated_request_not_processed_if_already_ordered( def test_belated_request_not_processed_if_already_ordered( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) @@ -42,7 +40,6 @@ def test_belated_request_not_processed_if_already_ordered( def test_belated_propagate_not_processed_if_already_ordered( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.nodeIbStasher.delay(ppgDelay(300, 'Gamma')) @@ -59,7 +56,6 @@ def test_belated_propagate_not_processed_if_already_ordered( def test_repeated_request_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size for node in txnPoolNodeSet: @@ -88,7 +84,6 @@ def test_repeated_request_not_processed_if_already_in_3pc_process( def test_belated_request_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) @@ -118,7 +113,6 @@ def test_belated_request_not_processed_if_already_in_3pc_process( def test_belated_propagate_not_processed_if_already_in_3pc_process( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.nodeIbStasher.delay(ppgDelay(300, 'Gamma')) @@ -148,7 +142,6 @@ def test_belated_propagate_not_processed_if_already_in_3pc_process( def test_repeated_request_not_processed_after_view_change( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size @@ -167,7 +160,6 @@ def test_repeated_request_not_processed_after_view_change( def test_belated_request_not_processed_after_view_change( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.clientIbStasher.delay(req_delay(300)) @@ -187,7 +179,6 @@ def test_belated_request_not_processed_after_view_change( def test_belated_propagate_not_processed_after_view_change( looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): - delta = txnPoolNodeSet[3] initial_ledger_size = delta.domainLedger.size delta.nodeIbStasher.delay(ppgDelay(300, 'Gamma')) diff --git a/plenum/test/node_request/test_discard_3pc_for_ordered.py b/plenum/test/node_request/test_discard_3pc_for_ordered.py index 7f5732a33b..d509b01441 100644 --- a/plenum/test/node_request/test_discard_3pc_for_ordered.py +++ b/plenum/test/node_request/test_discard_3pc_for_ordered.py @@ -27,8 +27,9 @@ def test_discard_3PC_messages_for_already_ordered(looper, txnPoolNodeSet, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, - num_reqs=2*sent_batches, + num_reqs=2 * sent_batches, num_batches=sent_batches) + # send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, # 2 * sent_batches, sent_batches) diff --git a/plenum/test/node_request/test_order/test_ordering_when_pre_prepare_not_received.py b/plenum/test/node_request/test_order/test_ordering_when_pre_prepare_not_received.py index ec5725e446..d364a54df7 100644 --- a/plenum/test/node_request/test_order/test_ordering_when_pre_prepare_not_received.py +++ b/plenum/test/node_request/test_order/test_ordering_when_pre_prepare_not_received.py @@ -43,7 +43,7 @@ def patched_p(self, msg, sender): def chk1(): assert len(slow_rep.commitsWaitingForPrepare) > 0 - sdk_send_random_request(looper,sdk_pool_handle, sdk_wallet_client) + sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) + delay looper.run(eventually(chk1, retryWait=1, timeout=timeout)) diff --git a/plenum/test/node_request/test_order/test_request_ordering_2.py b/plenum/test/node_request/test_order/test_request_ordering_2.py index c26fadb205..6790f016d2 100644 --- a/plenum/test/node_request/test_order/test_request_ordering_2.py +++ b/plenum/test/node_request/test_order/test_request_ordering_2.py @@ -22,7 +22,7 @@ def testOrderingCase2(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client https://www.pivotaltracker.com/n/projects/1889887/stories/133655009 """ pr, replicas = getPrimaryReplica(txnPoolNodeSet, instId=0), \ - getNonPrimaryReplicas(txnPoolNodeSet, instId=0) + getNonPrimaryReplicas(txnPoolNodeSet, instId=0) assert len(replicas) == 6 rep0 = pr @@ -55,7 +55,7 @@ def specificCommits(wrappedMsg): delayedPpSeqNos.add(msg.ppSeqNo) logger.debug('ppSeqNo {} be delayed'.format(msg.ppSeqNo)) if isinstance(msg, Commit) and msg.instId == 0 and \ - sender in (n.name for n in (node3, node4, node5)) and \ + sender in (n.name for n in (node3, node4, node5)) and \ msg.ppSeqNo in delayedPpSeqNos: return commitDelay @@ -64,7 +64,7 @@ def specificCommits(wrappedMsg): node.nodeIbStasher.delay(specificCommits) sdk_reqs = sdk_send_random_requests(looper, sdk_pool_handle, - sdk_wallet_client, requestCount) + sdk_wallet_client, requestCount) timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) diff --git a/plenum/test/node_request/test_prepare/test_num_prepare_with_2_of_6_faulty.py b/plenum/test/node_request/test_prepare/test_num_prepare_with_2_of_6_faulty.py index b652439b4a..c630de9ee8 100644 --- a/plenum/test/node_request/test_prepare/test_num_prepare_with_2_of_6_faulty.py +++ b/plenum/test/node_request/test_prepare/test_num_prepare_with_2_of_6_faulty.py @@ -9,7 +9,6 @@ from plenum.test.node_request.helper import nodes_by_rank - nodeCount = 6 f = 1 faultyNodes = f + 1 diff --git a/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py b/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py index 54d2164478..7453182566 100644 --- a/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py +++ b/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py @@ -65,8 +65,8 @@ def sum_of_sent_batches(): # number of sent batches in both replicas since both replicas # independently request PROPAGATEs assert get_count(faulty_node, faulty_node.request_propagates) - \ - old_count_request_propagates == (sum_of_sent_batches() - - old_sum_of_sent_batches) + old_count_request_propagates == (sum_of_sent_batches() - + old_sum_of_sent_batches) requested_propagate_counts = getAllReturnVals( faulty_node, faulty_node.request_propagates) diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py index 1602a1f492..60908d2590 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py @@ -22,7 +22,6 @@ def stop_node(node_to_stop, looper, pool_nodes): def test_quorum_after_f_plus_2_nodes_but_not_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, wallet1, client1, client1Connected): - nodes = txnPoolNodeSet request1 = sendRandomRequest(wallet1, client1) diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py index aa8fc5a703..ced0a76f87 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py @@ -22,7 +22,6 @@ def stop_node(node_to_stop, looper, pool_nodes): def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, txnPoolNodeSet, wallet1, client1, client1Connected): - nodes = txnPoolNodeSet request1 = sendRandomRequest(wallet1, client1) diff --git a/plenum/test/node_request/test_request_forwarding.py b/plenum/test/node_request/test_request_forwarding.py index 0c9f9f27b0..5c17bb9166 100644 --- a/plenum/test/node_request/test_request_forwarding.py +++ b/plenum/test/node_request/test_request_forwarding.py @@ -46,7 +46,7 @@ def chk(count): # Only non primary replicas should have all request keys with them looper.run(eventually(chk, tconf.Max3PCBatchSize - 1)) sdk_get_replies(looper, req_resps, timeout=sdk_eval_timeout( - tconf.Max3PCBatchSize-1, len(txnPoolNodeSet), + tconf.Max3PCBatchSize - 1, len(txnPoolNodeSet), add_delay_to_timeout=delay_3pc)) # Replicas should have no request keys with them since they are ordered looper.run(eventually(chk, 0)) # Need to wait since one node might not @@ -66,7 +66,7 @@ def chk(count): # Since each nomination is delayed and there will be multiple nominations # so adding some extra time timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + \ - len(txnPoolNodeSet) * delay + len(txnPoolNodeSet) * delay ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=timeout) sdk_get_replies(looper, req_resps, timeout=timeout) looper.run(eventually(chk, 0)) diff --git a/plenum/test/node_request/test_timestamp/test_clock_disruption.py b/plenum/test/node_request/test_timestamp/test_clock_disruption.py index 489569a0cc..4e3c616201 100644 --- a/plenum/test/node_request/test_timestamp/test_clock_disruption.py +++ b/plenum/test/node_request/test_timestamp/test_clock_disruption.py @@ -10,7 +10,6 @@ get_timestamp_suspicion_count from plenum.test.helper import sdk_send_random_and_check, sdk_send_random_request - Max3PCBatchSize = 4 from plenum.test.batching_3pc.conftest import tconf @@ -40,9 +39,9 @@ def test_nodes_with_bad_clock(tconf, looper, txnPoolNodeSet, make_clock_faulty( node, clock_slow_by_sec=node.config.ACCEPTABLE_DEVIATION_PREPREPARE_SECS + - randint( - 5, - 15), + randint( + 5, + 15), ppr_always_wrong=False) for _ in range(5): diff --git a/plenum/test/node_request/test_timestamp/test_timestamp_post_view_change.py b/plenum/test/node_request/test_timestamp/test_timestamp_post_view_change.py index eac7b9c503..42f6c950b5 100644 --- a/plenum/test/node_request/test_timestamp/test_timestamp_post_view_change.py +++ b/plenum/test/node_request/test_timestamp/test_timestamp_post_view_change.py @@ -59,9 +59,9 @@ def test_new_primary_has_wrong_clock(tconf, looper, txnPoolNodeSet, # Requests are sent for _ in range(5): sdk_send_random_requests(looper, - sdk_pool_handle, - sdk_wallet_client, - count=2) + sdk_pool_handle, + sdk_wallet_client, + count=2) looper.runFor(2) def chk(): @@ -78,7 +78,6 @@ def chk(): looper.run(eventually(chk, retryWait=1)) - # Eventually another view change happens looper.run(eventually(checkViewNoForNodes, txnPoolNodeSet, old_view_no + 1, retryWait=1, timeout=2 * tconf.PerfCheckFreq)) diff --git a/plenum/test/plugin/demo_plugin/__init__.py b/plenum/test/plugin/demo_plugin/__init__.py index 67748643de..306c66fee7 100644 --- a/plenum/test/plugin/demo_plugin/__init__.py +++ b/plenum/test/plugin/demo_plugin/__init__.py @@ -2,12 +2,11 @@ from plenum.test.plugin.demo_plugin.transactions import DemoTransactions from plenum.test.plugin.demo_plugin.constants import AUCTION_LEDGER_ID - dummy_field_length = 10 LEDGER_IDS = {AUCTION_LEDGER_ID, } CLIENT_REQUEST_FIELDS = {'fix_length_dummy': - FixedLengthField(dummy_field_length, - optional=True, nullable=True)} + FixedLengthField(dummy_field_length, + optional=True, nullable=True)} AcceptableWriteTypes = {DemoTransactions.AUCTION_START.value, DemoTransactions.AUCTION_END.value, diff --git a/plenum/test/plugin/demo_plugin/test_plugin_basic.py b/plenum/test/plugin/demo_plugin/test_plugin_basic.py index c4ef0829a2..53b5abd289 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_basic.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_basic.py @@ -42,7 +42,7 @@ def test_plugin_client_req_fields(nodeSet, looper, stewardWallet, req = Request(operation=op, reqId=Request.gen_req_id(), protocolVersion=CURRENT_PROTOCOL_VERSION, identifier=stewardWallet.defaultId, - fix_length_dummy=randomString(dummy_field_length+1)) + fix_length_dummy=randomString(dummy_field_length + 1)) steward1.submitReqs(req) for node in nodeSet: looper.run(eventually(checkReqNackWithReason, steward1, diff --git a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py index 515b2a8a01..9103dc6783 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py @@ -88,8 +88,8 @@ def test_plugin_dynamic_validation(nodeSet, looper, stewardWallet, @pytest.fixture(scope="module") def some_requests(nodeSet, looper, stewardWallet, - steward1, client1Connected, - sdk_wallet_steward, sdk_pool_handle): + steward1, client1Connected, + sdk_wallet_steward, sdk_pool_handle): op = { TXN_TYPE: AUCTION_START, DATA: {'id': 'pqr'} diff --git a/plenum/test/plugin/demo_plugin/transactions.py b/plenum/test/plugin/demo_plugin/transactions.py index 64c73af187..e6d27dec9a 100644 --- a/plenum/test/plugin/demo_plugin/transactions.py +++ b/plenum/test/plugin/demo_plugin/transactions.py @@ -1,6 +1,5 @@ from plenum.common.transactions import Transactions - # DO NOT CHANGE ONCE CODE IS DEPLOYED ON THE LEDGER # TODO: Might need a short hash with unique entropy, plugin name being input of hash. PREFIX = '9999' diff --git a/plenum/test/plugin/test_notifier_plugin_manager.py b/plenum/test/plugin/test_notifier_plugin_manager.py index 5978d2a603..c97876f9cf 100644 --- a/plenum/test/plugin/test_notifier_plugin_manager.py +++ b/plenum/test/plugin/test_notifier_plugin_manager.py @@ -18,7 +18,7 @@ def testPluginManagerFindsPlugins(monkeypatch, pluginManager): partial( mockGetInstalledDistributions, packages=validPackages + - invalidPackages)) + invalidPackages)) assert len(pluginManager._findPlugins()) == validPackagesCnt @@ -59,9 +59,9 @@ def testPluginManagerSendMessageUponSuspiciousSpikeFailsOnMinCnt( 'minActivityThreshold': 0, 'enabled': True } - assert pluginManagerWithImportedModules\ - .sendMessageUponSuspiciousSpike(topic, historicalData, - newVal, config, name, enabled=True) is None + assert pluginManagerWithImportedModules \ + .sendMessageUponSuspiciousSpike(topic, historicalData, + newVal, config, name, enabled=True) is None def testPluginManagerSendMessageUponSuspiciousSpikeFailsOnCoefficient( @@ -79,9 +79,9 @@ def testPluginManagerSendMessageUponSuspiciousSpikeFailsOnCoefficient( 'minActivityThreshold': 0, 'enabled': True } - assert pluginManagerWithImportedModules\ - .sendMessageUponSuspiciousSpike(topic, historicalData, - newVal, config, name, enabled=True) is None + assert pluginManagerWithImportedModules \ + .sendMessageUponSuspiciousSpike(topic, historicalData, + newVal, config, name, enabled=True) is None def testPluginManagerSendMessageUponSuspiciousSpike( @@ -99,7 +99,7 @@ def testPluginManagerSendMessageUponSuspiciousSpike( 'minActivityThreshold': 0, 'enabled': True } - sent, found = pluginManagerWithImportedModules\ + sent, found = pluginManagerWithImportedModules \ .sendMessageUponSuspiciousSpike(topic, historicalData, newVal, config, name, enabled=True) assert sent == 3 @@ -108,6 +108,7 @@ def testPluginManagerSendMessageUponSuspiciousSpike( def testNodeSendNodeRequestSpike(pluginManagerWithImportedModules, testNode): def mockProcessRequest(obj, inc=1): obj.nodeRequestSpikeMonitorData['accum'] += inc + testNode.config.SpikeEventsEnabled = True testNode.config.notifierEventTriggeringConfig['nodeRequestSpike'] = { 'coefficient': 3, diff --git a/plenum/test/pool_transactions/conftest.py b/plenum/test/pool_transactions/conftest.py index 521af7e208..94930f90e5 100644 --- a/plenum/test/pool_transactions/conftest.py +++ b/plenum/test/pool_transactions/conftest.py @@ -18,34 +18,6 @@ def tconf(tconf, request): return tconf -@pytest.yield_fixture(scope="module") -def looper(txnPoolNodesLooper): - yield txnPoolNodesLooper - - -@pytest.fixture(scope="module") -def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData, - tdirWithClientPoolTxns, client_tdir): - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, - client_tdir) - yield client, wallet - client.stop() - - -@pytest.fixture(scope="module") -def steward1(looper, txnPoolNodeSet, stewardAndWallet1): - steward, wallet = stewardAndWallet1 - looper.add(steward) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward, - *txnPoolNodeSet) - return steward - - -@pytest.fixture(scope="module") -def stewardWallet(stewardAndWallet1): - return stewardAndWallet1[1] - - @pytest.fixture("module") def nodeThetaAdded(looper, txnPoolNodeSet, tdir, client_tdir, tconf, steward1, stewardWallet, allPluginsPath, testNodeClass=None, @@ -99,31 +71,6 @@ def sdk_node_theta_added(looper, return new_steward_wallet, new_node -@pytest.fixture(scope="module") -def clientAndWallet1(txnPoolNodeSet, poolTxnClientData, tdirWithClientPoolTxns, client_tdir): - client, wallet = buildPoolClientAndWallet(poolTxnClientData, - client_tdir) - yield client, wallet - client.stop() - - -@pytest.fixture(scope="module") -def client1(clientAndWallet1): - return clientAndWallet1[0] - - -@pytest.fixture(scope="module") -def wallet1(clientAndWallet1): - return clientAndWallet1[1] - - -@pytest.fixture(scope="module") -def client1Connected(looper, client1): - looper.add(client1) - looper.run(client1.ensureConnectedToNodes()) - return client1 - - @pytest.fixture(scope="function") def newAdHocSteward(looper, client_tdir, steward1, stewardWallet): newStewardName = "testClientSteward" + randomString(3) diff --git a/plenum/test/pool_transactions/test_nodes_with_pool_txns.py b/plenum/test/pool_transactions/test_nodes_with_pool_txns.py index 4409b53431..d58afac793 100644 --- a/plenum/test/pool_transactions/test_nodes_with_pool_txns.py +++ b/plenum/test/pool_transactions/test_nodes_with_pool_txns.py @@ -30,7 +30,6 @@ # reaches it - def testStewardCannotAddMoreThanOneNode(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, @@ -305,4 +304,4 @@ def test_add_node_with_not_unique_alias(looper, allPluginsPath) assert 'existing data has conflicts with request data' in \ e._excinfo[1].args[0] - sdk_pool_refresh(looper, sdk_pool_handle) \ No newline at end of file + sdk_pool_refresh(looper, sdk_pool_handle) diff --git a/plenum/test/primary_election/helpers.py b/plenum/test/primary_election/helpers.py index 18432f75b9..f7754c3259 100644 --- a/plenum/test/primary_election/helpers.py +++ b/plenum/test/primary_election/helpers.py @@ -23,7 +23,7 @@ def getSelfNominationByNode(node: TestNode) -> int: for instId, replica in enumerate(node.elector.replicas): name = Replica.generateName(node.name, instId) if node.elector.nominations.get(instId, {}).get(name, [None, ])[ - 0] == name: + 0] == name: return instId diff --git a/plenum/test/primary_election/test_primary_election_case1.py b/plenum/test/primary_election/test_primary_election_case1.py index a1bd9d194e..bb410b57a4 100644 --- a/plenum/test/primary_election/test_primary_election_case1.py +++ b/plenum/test/primary_election/test_primary_election_case1.py @@ -13,7 +13,6 @@ ensureElectionsDone from plenum.test import waits - nodeCount = 4 whitelist = ['already got nomination', 'doing nothing for now'] @@ -92,7 +91,7 @@ def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes): for node in [nodeA, nodeC, nodeD]: assert [n[0] for n in node.elector.nominations[instId].values()].count( Replica.generateName(nodeD.name, instId)) \ - <= 1 + <= 1 timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination primaryReplicas = ensureElectionsDone(looper=looper, diff --git a/plenum/test/primary_election/test_primary_election_case2.py b/plenum/test/primary_election/test_primary_election_case2.py index a0a4725f33..804ecb41b8 100644 --- a/plenum/test/primary_election/test_primary_election_case2.py +++ b/plenum/test/primary_election/test_primary_election_case2.py @@ -11,7 +11,6 @@ ensureElectionsDone from plenum.test import waits - nodeCount = 4 whitelist = ['already got nomination', 'doing nothing for now'] @@ -37,6 +36,7 @@ def case2Setup(startedNodes: TestNodeSet): for node in A, C, D: node.whitelistNode(B.name, Suspicions.DUPLICATE_NOM_SENT.code) + # noinspection PyIncorrectDocstring diff --git a/plenum/test/primary_election/test_primary_election_case4.py b/plenum/test/primary_election/test_primary_election_case4.py index 21504b0022..179e037d84 100644 --- a/plenum/test/primary_election/test_primary_election_case4.py +++ b/plenum/test/primary_election/test_primary_election_case4.py @@ -68,7 +68,7 @@ def x(): # also have to take into account the catchup procedure timeout = waits.expectedPoolNominationTimeout(len(allNodes)) + \ - waits.expectedPoolCatchupTime(len(allNodes)) + waits.expectedPoolCatchupTime(len(allNodes)) for node in (A, C, D): looper.run(eventually(x, retryWait=.5, timeout=timeout)) diff --git a/plenum/test/primary_election/test_primary_election_case5.py b/plenum/test/primary_election/test_primary_election_case5.py index 09e3d63422..c9827a27cc 100644 --- a/plenum/test/primary_election/test_primary_election_case5.py +++ b/plenum/test/primary_election/test_primary_election_case5.py @@ -1,4 +1,3 @@ - import pytest from plenum.common.messages.node_messages import Nomination, Primary from plenum.test import waits @@ -17,7 +16,6 @@ 'doing nothing for now', 'know how to handle it'] - logger = getlogger() # the total delay of election done @@ -84,8 +82,8 @@ def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes): # Ensure elections are done # also have to take into account the catchup procedure timeout = waits.expectedPoolElectionTimeout(len(nodeSet)) + \ - waits.expectedPoolCatchupTime(len(nodeSet)) + \ - delayOfElectionDone + waits.expectedPoolCatchupTime(len(nodeSet)) + \ + delayOfElectionDone ensureElectionsDone(looper=looper, nodes=nodeSet, customTimeout=timeout) # All nodes from node A, node C, node D(node B is malicious anyway so not @@ -94,5 +92,5 @@ def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes): for node in [A, C, D]: logger.debug( "node {} should have primary declaration for C from node B" - .format(node)) + .format(node)) assert node.elector.primaryDeclarations[0][BRep][0] == CRep diff --git a/plenum/test/primary_election/test_primary_election_contested.py b/plenum/test/primary_election/test_primary_election_contested.py index 0165afcc0a..2152fce73f 100644 --- a/plenum/test/primary_election/test_primary_election_contested.py +++ b/plenum/test/primary_election/test_primary_election_contested.py @@ -9,7 +9,6 @@ checkProtocolInstanceSetup from plenum.test import waits - nodeCount = 4 logger = getlogger() diff --git a/plenum/test/primary_election/test_primary_election_with_tie.py b/plenum/test/primary_election/test_primary_election_with_tie.py index 0cd18c31bb..7124baece3 100644 --- a/plenum/test/primary_election/test_primary_election_with_tie.py +++ b/plenum/test/primary_election/test_primary_election_with_tie.py @@ -9,10 +9,8 @@ checkProtocolInstanceSetup from plenum.test import waits - nodeCount = 4 - logger = getlogger() diff --git a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py index f8a0cee588..e936f1af41 100644 --- a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py +++ b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py @@ -12,7 +12,7 @@ def _get_ppseqno(nodes): for node in nodes: for repl in node.replicas: res.add(repl.lastPrePrepareSeqNo) - assert(len(res) == 1) + assert (len(res) == 1) return min(res) @@ -41,11 +41,11 @@ def test_add_node_to_pool_with_large_ppseqno_diff_views(do_view_change, looper, big_ppseqno = tconf.LOG_SIZE * 2 + 2345 cur_ppseqno = _get_ppseqno(txnPoolNodeSet) - assert(big_ppseqno > cur_ppseqno) + assert (big_ppseqno > cur_ppseqno) # ensure pool is working properly sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, numReqs=3) - assert(cur_ppseqno < _get_ppseqno(txnPoolNodeSet)) + assert (cur_ppseqno < _get_ppseqno(txnPoolNodeSet)) _set_ppseqno(txnPoolNodeSet, big_ppseqno) cur_ppseqno = _get_ppseqno(txnPoolNodeSet) @@ -55,7 +55,7 @@ def test_add_node_to_pool_with_large_ppseqno_diff_views(do_view_change, looper, new_steward_name = "testClientSteward" + randomString(4) new_node_name = "TestTheta" + randomString(4) - new_steward, new_steward_wallet, new_node =\ + new_steward, new_steward_wallet, new_node = \ addNewStewardAndNode(looper, steward1, stewardWallet, new_steward_name, new_node_name, tdir, client_tdir, tconf, allPluginsPath) txnPoolNodeSet.append(new_node) diff --git a/plenum/test/primary_selection/test_catchup_after_view_change.py b/plenum/test/primary_selection/test_catchup_after_view_change.py index 78ec458859..52405a1644 100644 --- a/plenum/test/primary_selection/test_catchup_after_view_change.py +++ b/plenum/test/primary_selection/test_catchup_after_view_change.py @@ -16,7 +16,6 @@ stewardAndWallet1, steward1, stewardWallet from plenum.test.batching_3pc.conftest import tconf - Max3PCBatchSize = 4 TestRunningTimeLimitSec = 150 diff --git a/plenum/test/primary_selection/test_catchup_multiple_rounds.py b/plenum/test/primary_selection/test_catchup_multiple_rounds.py index db54c0bab1..1bd9caaf0d 100644 --- a/plenum/test/primary_selection/test_catchup_multiple_rounds.py +++ b/plenum/test/primary_selection/test_catchup_multiple_rounds.py @@ -26,8 +26,7 @@ def test_slow_nodes_catchup_before_selecting_primary_in_new_view( txnPoolNodeSet, client1, wallet1, - one_node_added, - client1Connected): + one_node_added): """ Delay 3PC messages to one node and view change messages to some others (including primary) so the node that does not receive enough 3PC messages is @@ -54,7 +53,7 @@ def test_slow_nodes_catchup_before_selecting_primary_in_new_view( def start_count(): return sum([1 for e in slow_node.ledgerManager.spylog.getAll( slow_node.ledgerManager.startCatchUpProcess.__name__) - if e.params['ledgerId'] == DOMAIN_LEDGER_ID]) + if e.params['ledgerId'] == DOMAIN_LEDGER_ID]) s = start_count() requests = sendRandomRequests(wallet1, client1, 10 * Max3PCBatchSize) diff --git a/plenum/test/primary_selection/test_catchup_needed_check.py b/plenum/test/primary_selection/test_catchup_needed_check.py index 2588f48f0c..2a0ba2c164 100644 --- a/plenum/test/primary_selection/test_catchup_needed_check.py +++ b/plenum/test/primary_selection/test_catchup_needed_check.py @@ -13,7 +13,6 @@ # noinspection PyUnresolvedReferences from plenum.test.batching_3pc.conftest import tconf - Max3PCBatchSize = 2 diff --git a/plenum/test/primary_selection/test_primary_selection.py b/plenum/test/primary_selection/test_primary_selection.py index 2f0d7e7f9c..f87c05cf1c 100644 --- a/plenum/test/primary_selection/test_primary_selection.py +++ b/plenum/test/primary_selection/test_primary_selection.py @@ -34,6 +34,7 @@ def testPrimarySelectionAfterPoolReady( Once the pool is ready(node has connected to at least 3 other nodes), appropriate primary replicas should be selected. """ + def checkPrimaryPlacement(): # Node names sorted by rank sortedNodes = sorted(nodeSet.nodes.values(), @@ -79,15 +80,16 @@ def catchup_complete_count(nodeSet): return {n.name: n.spylog.count(n.allLedgersCaughtUp) for n in nodeSet} -@pytest.fixture(scope='module') # noqa +@pytest.fixture(scope='module') # noqa def view_change_done(looper, nodeSet): ensure_view_change(looper, nodeSet) ensureElectionsDone(looper=looper, nodes=nodeSet) + # noinspection PyIncorrectDocstring -def testPrimarySelectionAfterViewChange( # noqa +def testPrimarySelectionAfterViewChange( # noqa looper, nodeSet, ready, diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py index dd2ebdcfa3..cac930b4c7 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py @@ -14,9 +14,11 @@ logger = getlogger() + def test_primary_selection_after_primary_demotion_and_pool_restart(looper, - txnPoolNodeSet, stewardAndWalletForMasterNode, txnPoolMasterNodes, - tdir, tconf): + txnPoolNodeSet, stewardAndWalletForMasterNode, + txnPoolMasterNodes, + tdir, tconf): """ Demote primary and restart the pool. Pool should select new primary and have viewNo=0 after restart. diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py index 228af4111f..03967bf846 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py @@ -12,8 +12,9 @@ logger = getlogger() + def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPoolNodeSet, - stewardAndWalletForMasterNode, txnPoolMasterNodes): + stewardAndWalletForMasterNode, txnPoolMasterNodes): """ Demote primary and do multiple view changes forcing primaries rotation. Demoted primary should be skipped without additional view changes. @@ -32,14 +33,14 @@ def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPo updateNodeData(looper, client, wallet, master_node, node_data) restNodes = [node for node in txnPoolNodeSet \ - if node.name != master_node.name] + if node.name != master_node.name] ensureElectionsDone(looper, restNodes) viewNo1 = checkViewNoForNodes(restNodes) assert viewNo1 == viewNo0 + 1 assert master_node.viewNo == viewNo0 - assert len(restNodes[0].replicas) == 1 # only one instance left + assert len(restNodes[0].replicas) == 1 # only one instance left assert restNodes[0].replicas[0].primaryName != master_node.name # ensure pool is working properly diff --git a/plenum/test/primary_selection/test_primary_selection_pool_txn.py b/plenum/test/primary_selection/test_primary_selection_pool_txn.py index fe7c49632a..a3bc89bbcf 100644 --- a/plenum/test/primary_selection/test_primary_selection_pool_txn.py +++ b/plenum/test/primary_selection/test_primary_selection_pool_txn.py @@ -20,7 +20,7 @@ def check_accepted_view_change_sent(node, nodes): continue if other_node.name in node.view_changer._view_change_done: assert node.view_changer._view_change_done[other_node.name] == \ - node.view_changer._accepted_view_change_done_message + node.view_changer._accepted_view_change_done_message def test_primary_selection_non_genesis_node(one_node_added, looper, @@ -52,6 +52,5 @@ def test_primary_selection_increase_f( # check_accepted_view_change_sent(n, txnPoolNodeSet) ensure_pool_functional(looper, txnPoolNodeSet, stewardWallet, steward1) - # TODO: Add more tests to make one next primary crashed, malicious, ensure primary - # selection happens after catchup +# selection happens after catchup diff --git a/plenum/test/primary_selection/test_primary_selector.py b/plenum/test/primary_selection/test_primary_selector.py index c6b01ded9d..f1628008b7 100644 --- a/plenum/test/primary_selection/test_primary_selector.py +++ b/plenum/test/primary_selection/test_primary_selector.py @@ -17,7 +17,6 @@ from plenum.test.helper import create_new_test_node from plenum.test.test_node import TestNode - whitelist = ['but majority declared'] @@ -103,6 +102,7 @@ def on_view_change_start(self): def start_catchup(self): pass + def test_has_view_change_quorum_number(tconf, tdir): """ Checks method _hasViewChangeQuorum of SimpleSelector @@ -299,7 +299,7 @@ def test_get_msgs_for_lagged_nodes(tconf, tdir): viewNo=0, name='Node2', ledgerInfo=ledgerInfo), - 'Node1'), + 'Node1'), (ViewChangeDone( viewNo=0, name='Node3', @@ -483,4 +483,4 @@ def test_primaries_selection_gaps(txnPoolNodeSetWithElector): name, instance_name = node.elector.next_primary_replica_name_for_backup( 2, master_primary_rank, primaries) assert name == "Gamma" and \ - instance_name == "Gamma:2" + instance_name == "Gamma:2" diff --git a/plenum/test/primary_selection/test_recover_more_than_f_failure.py b/plenum/test/primary_selection/test_recover_more_than_f_failure.py index 4b28252845..8672065e36 100644 --- a/plenum/test/primary_selection/test_recover_more_than_f_failure.py +++ b/plenum/test/primary_selection/test_recover_more_than_f_failure.py @@ -13,11 +13,9 @@ from plenum.test.view_change.helper import ensure_view_change from plenum.test.view_change.helper import start_stopped_node - # Do not remove these imports from plenum.test.pool_transactions.conftest import client1, wallet1, client1Connected, looper - logger = getlogger() @@ -47,7 +45,7 @@ def test_recover_stop_primaries(looper, checkpoint_size, txnPoolNodeSet, logger.info("send at least one checkpoint") assert nodes_do_not_have_checkpoints(*active_nodes) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=2*checkpoint_size) + sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=2 * checkpoint_size) assert nodes_have_checkpoints(*active_nodes) ensure_all_nodes_have_same_data(looper, nodes=active_nodes) @@ -67,7 +65,7 @@ def test_recover_stop_primaries(looper, checkpoint_size, txnPoolNodeSet, ensure_all_nodes_have_same_data(looper, nodes=active_nodes) logger.info("Check if the pool is able to process requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=10*checkpoint_size) + sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=10 * checkpoint_size) ensure_all_nodes_have_same_data(looper, nodes=active_nodes) assert nodes_have_checkpoints(*active_nodes) @@ -91,7 +89,7 @@ def checkpoint_size(tconf, request): tconf.Max3PCBatchSize = 3 tconf.CHK_FREQ = getValueFromModule(request, "CHK_FREQ", 2) - tconf.LOG_SIZE = 2*tconf.CHK_FREQ + tconf.LOG_SIZE = 2 * tconf.CHK_FREQ def reset(): tconf.CHK_FREQ = oldChkFreq diff --git a/plenum/test/primary_selection/test_recover_primary_no_view_change.py b/plenum/test/primary_selection/test_recover_primary_no_view_change.py index 95f4fc9062..53a90f0440 100644 --- a/plenum/test/primary_selection/test_recover_primary_no_view_change.py +++ b/plenum/test/primary_selection/test_recover_primary_no_view_change.py @@ -19,7 +19,6 @@ # Do not remove these imports from plenum.test.pool_transactions.conftest import client1, wallet1, client1Connected, looper - logger = getlogger() @@ -48,7 +47,7 @@ def test_recover_stop_primaries_no_view_change(looper, checkpoint_size, txnPoolN logger.info("send at least one checkpoint") assert nodes_do_not_have_checkpoints(*active_nodes) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=2*checkpoint_size) + sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=2 * checkpoint_size) assert nodes_have_checkpoints(*active_nodes) ensure_all_nodes_have_same_data(looper, nodes=active_nodes) @@ -68,6 +67,6 @@ def test_recover_stop_primaries_no_view_change(looper, checkpoint_size, txnPoolN ensure_all_nodes_have_same_data(looper, nodes=active_nodes) logger.info("Check if the pool is able to process requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=10*checkpoint_size) + sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=10 * checkpoint_size) ensure_all_nodes_have_same_data(looper, nodes=active_nodes) assert nodes_have_checkpoints(*active_nodes) diff --git a/plenum/test/primary_selection/test_selection_f_plus_one_quorum.py b/plenum/test/primary_selection/test_selection_f_plus_one_quorum.py index 889cdfb1da..3e98d4c467 100644 --- a/plenum/test/primary_selection/test_selection_f_plus_one_quorum.py +++ b/plenum/test/primary_selection/test_selection_f_plus_one_quorum.py @@ -6,7 +6,6 @@ from plenum.test.view_change.helper import ensure_view_change from plenum.test.view_change.helper import start_stopped_node - # Do not remove these imports from plenum.test.pool_transactions.conftest import client1, wallet1, client1Connected, looper diff --git a/plenum/test/propagate/test_propagate_recvd_after_request.py b/plenum/test/propagate/test_propagate_recvd_after_request.py index 870ceb9309..be8b109f34 100644 --- a/plenum/test/propagate/test_propagate_recvd_after_request.py +++ b/plenum/test/propagate/test_propagate_recvd_after_request.py @@ -18,7 +18,7 @@ def setup(nodeSet): delay(Propagate, frm=[B, C, D], to=A, howlong=howlong) # Delay MessageRep by long simulating loss as if Propagate is missing, it # is requested - A.nodeIbStasher.delay(msg_rep_delay(10*howlong, [PROPAGATE, ])) + A.nodeIbStasher.delay(msg_rep_delay(10 * howlong, [PROPAGATE, ])) def testPropagateRecvdAfterRequest(setup, looper, nodeSet, up, sent1): diff --git a/plenum/test/propagate/test_propagate_recvd_before_request.py b/plenum/test/propagate/test_propagate_recvd_before_request.py index bb0d60ff2b..afca15b112 100644 --- a/plenum/test/propagate/test_propagate_recvd_before_request.py +++ b/plenum/test/propagate/test_propagate_recvd_before_request.py @@ -8,7 +8,6 @@ sentPropagate, forwardedRequest from plenum.test import waits - nodeCount = 4 howlong = 10 delaySec = 5 diff --git a/plenum/test/replica/test_buffers_cleaning.py b/plenum/test/replica/test_buffers_cleaning.py index ef0290a164..940aa8d1dc 100644 --- a/plenum/test/replica/test_buffers_cleaning.py +++ b/plenum/test/replica/test_buffers_cleaning.py @@ -3,7 +3,6 @@ def test_ordered_cleaning(tconf): - global_view_no = 2 node = FakeSomething( @@ -12,7 +11,7 @@ def test_ordered_cleaning(tconf): viewNo=global_view_no, ) bls_bft_replica = FakeSomething( - gc = lambda *args: None, + gc=lambda *args: None, ) replica = Replica(node, instId=0, config=tconf, bls_bft_replica=bls_bft_replica) @@ -34,33 +33,32 @@ def test_ordered_cleaning(tconf): def test_primary_names_cleaning(tconf): - node = FakeSomething( name="fake node", ledger_ids=[0], viewNo=0, ) bls_bft_replica = FakeSomething( - gc = lambda *args: None, + gc=lambda *args: None, ) replica = Replica(node, instId=0, config=tconf, bls_bft_replica=bls_bft_replica) replica.primaryName = "Node1:0" assert list(replica.primaryNames.items()) == \ - [(0, "Node1:0")] + [(0, "Node1:0")] node.viewNo += 1 replica.primaryName = "Node2:0" assert list(replica.primaryNames.items()) == \ - [(0, "Node1:0"), (1, "Node2:0")] + [(0, "Node1:0"), (1, "Node2:0")] node.viewNo += 1 replica.primaryName = "Node3:0" assert list(replica.primaryNames.items()) == \ - [(1, "Node2:0"), (2, "Node3:0")] + [(1, "Node2:0"), (2, "Node3:0")] node.viewNo += 1 replica.primaryName = "Node4:0" assert list(replica.primaryNames.items()) == \ - [(2, "Node3:0"), (3, "Node4:0")] + [(2, "Node3:0"), (3, "Node4:0")] diff --git a/plenum/test/replica/test_primary_marked_suspicious_for_sending_prepare.py b/plenum/test/replica/test_primary_marked_suspicious_for_sending_prepare.py index 3fd437b419..4f79c1b516 100644 --- a/plenum/test/replica/test_primary_marked_suspicious_for_sending_prepare.py +++ b/plenum/test/replica/test_primary_marked_suspicious_for_sending_prepare.py @@ -1,4 +1,3 @@ - import pytest from plenum.test.delayers import cDelay diff --git a/plenum/test/replica/test_replica_reject_same_pre_prepare.py b/plenum/test/replica/test_replica_reject_same_pre_prepare.py index 00416973f4..d02324f6ec 100644 --- a/plenum/test/replica/test_replica_reject_same_pre_prepare.py +++ b/plenum/test/replica/test_replica_reject_same_pre_prepare.py @@ -1,4 +1,3 @@ - import pytest from plenum.test.delayers import cDelay @@ -17,7 +16,6 @@ 'cannot process incoming PRE-PREPARE', 'InvalidSignature'] - logger = getlogger() diff --git a/plenum/test/script/helper.py b/plenum/test/script/helper.py index ac57a49017..0ef9fb3b7a 100644 --- a/plenum/test/script/helper.py +++ b/plenum/test/script/helper.py @@ -1,4 +1,3 @@ - import pytest from plenum.client.wallet import Wallet @@ -26,7 +25,6 @@ def looper(txnPoolNodesLooper): def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnData, poolTxnStewardNames, tconf, shouldBePrimary, tdir): - # prepare new ha for node and client stack subjectedNode = None stewardName = None @@ -40,7 +38,7 @@ def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, break nodeStackNewHA, clientStackNewHA = genHa(2) - logger.debug("change HA for node: {} to {}". format( + logger.debug("change HA for node: {} to {}".format( subjectedNode.name, (nodeStackNewHA, clientStackNewHA))) nodeSeed = poolTxnData["seeds"][subjectedNode.name].encode() diff --git a/plenum/test/script/test_change_non_primary_node_ha.py b/plenum/test/script/test_change_non_primary_node_ha.py index 5e8d5687f7..da52b4cdf5 100644 --- a/plenum/test/script/test_change_non_primary_node_ha.py +++ b/plenum/test/script/test_change_non_primary_node_ha.py @@ -4,7 +4,6 @@ from stp_core.common.log import getlogger from plenum.test.script.helper import changeNodeHa - logger = getlogger() whitelist = ['found legacy entry', "doesn't match", 'reconciling nodeReg', @@ -16,7 +15,6 @@ @pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330') def testChangeNodeHaForNonPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnData, poolTxnStewardNames, tconf, tdir): - changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, diff --git a/plenum/test/script/test_change_primary_node_ha.py b/plenum/test/script/test_change_primary_node_ha.py index 464120d604..e7289b5e87 100644 --- a/plenum/test/script/test_change_primary_node_ha.py +++ b/plenum/test/script/test_change_primary_node_ha.py @@ -4,7 +4,6 @@ from stp_core.common.log import getlogger from plenum.test.script.helper import changeNodeHa - logger = getlogger() whitelist = ['found legacy entry', "doesn't match", 'reconciling nodeReg', diff --git a/plenum/test/storage/test_client_req_rep_store.py b/plenum/test/storage/test_client_req_rep_store.py index 417e8b593b..3676e9c351 100644 --- a/plenum/test/storage/test_client_req_rep_store.py +++ b/plenum/test/storage/test_client_req_rep_store.py @@ -4,4 +4,4 @@ def testReqAcks(replied1, client1): assert len(client1.nodeReg) == len(client1.reqRepStore.getAcks(identifier, reqId)) assert set(client1.nodeReg.keys()) == \ - set(client1.reqRepStore.getAcks(identifier, reqId)) + set(client1.reqRepStore.getAcks(identifier, reqId)) diff --git a/plenum/test/test_action_queue.py b/plenum/test/test_action_queue.py index 36ee5b188b..004ae63802 100644 --- a/plenum/test/test_action_queue.py +++ b/plenum/test/test_action_queue.py @@ -5,6 +5,7 @@ from plenum.common.motor import Motor from plenum.server.has_action_queue import HasActionQueue + class Q1(Motor, HasActionQueue): def __init__(self, name): self.name = name @@ -15,7 +16,7 @@ def __init__(self, name): def start(self, loop): pass - async def prod(self, limit: int=None) -> int: + async def prod(self, limit: int = None) -> int: return self._serviceActions() def meth(self, meth_name, x): @@ -23,6 +24,7 @@ def meth(self, meth_name, x): self.results[meth_name] = [] self.results[meth_name].append((x, time.perf_counter())) + def test_action_scheduling(): with Looper() as looper: q1 = Q1('q1') diff --git a/plenum/test/test_bootstrapping.py b/plenum/test/test_bootstrapping.py index 2574be22cb..5f5bd1fb51 100644 --- a/plenum/test/test_bootstrapping.py +++ b/plenum/test/test_bootstrapping.py @@ -9,7 +9,6 @@ logger = getlogger() - whitelist = ['public key from disk', 'verification key from disk', 'doesnt have enough info to connect'] diff --git a/plenum/test/test_client.py b/plenum/test/test_client.py index 2520f3670e..8e8ef5f1d4 100644 --- a/plenum/test/test_client.py +++ b/plenum/test/test_client.py @@ -18,7 +18,6 @@ from plenum.test.testable import spyable from plenum.common.constants import OP_FIELD_NAME - logger = getlogger() client_spyables = [Client.handleOneNodeMsg, @@ -51,8 +50,8 @@ def genTestClient(nodes=None, nodeReg=None, tmpdir=None, testClientClass=TestClient, - identifier: Identifier=None, - verkey: str=None, + identifier: Identifier = None, + verkey: str = None, bootstrapKeys=True, ha=None, usePoolLedger=False, diff --git a/plenum/test/test_crypto.py b/plenum/test/test_crypto.py index 47e7266fcc..3f2c2e3a27 100644 --- a/plenum/test/test_crypto.py +++ b/plenum/test/test_crypto.py @@ -142,7 +142,7 @@ def testKeyConversionFromEd25519ToCurve25519(): publicKey = ed25519PkToCurve25519(vk) assert PrivateKey(secretKey).public_key.__bytes__() == publicKey assert ed25519PkToCurve25519(vk, toHex=True) == \ - hexlify(PrivateKey(secretKey).public_key.__bytes__()) + hexlify(PrivateKey(secretKey).public_key.__bytes__()) # Check when keys are passed as hex secretKey = ed25519SkToCurve25519(hexlify(sk)) diff --git a/plenum/test/test_delay.py b/plenum/test/test_delay.py index b40b8c379c..f78237dc98 100644 --- a/plenum/test/test_delay.py +++ b/plenum/test/test_delay.py @@ -87,4 +87,4 @@ def testSelfNominationDelay(tdir_for_func): looper.run(eventually(lambda: assertExp( len(nodeA.spylog.getAll( Node.decidePrimaries.__name__)) > 0), - retryWait=1, timeout=delay)) + retryWait=1, timeout=delay)) diff --git a/plenum/test/test_dirty_read.py b/plenum/test/test_dirty_read.py index 9b29f3cc52..9948924bc7 100644 --- a/plenum/test/test_dirty_read.py +++ b/plenum/test/test_dirty_read.py @@ -11,12 +11,14 @@ def make_node_slow(node): old = node.serviceReplicas + async def serviceReplicas(limit): for replica in node.replicas: for index, message in enumerate(list(replica.outBox)): if isinstance(message, Ordered): del replica.outBox[index] return await old(limit) + node.serviceReplicas = serviceReplicas @@ -37,7 +39,7 @@ def test_dirty_read(looper, nodeSet, client1, wallet1): numReqs=1)[0] received_replies = getRepliesFromClientInbox(inbox=client1.inBox, - reqId=set_request.reqId) + reqId=set_request.reqId) seq_no = received_replies[0]["result"]["seqNo"] get_request = [wallet1.signOp({ diff --git a/plenum/test/test_memory_consumpion.py b/plenum/test/test_memory_consumpion.py index cba0afd5be..77ba471f9d 100644 --- a/plenum/test/test_memory_consumpion.py +++ b/plenum/test/test_memory_consumpion.py @@ -7,7 +7,6 @@ ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.pool_transactions.helper import buildPoolClientAndWallet - logger = getlogger() diff --git a/plenum/test/test_node.py b/plenum/test/test_node.py index 9e33c89c64..dc546c79eb 100644 --- a/plenum/test/test_node.py +++ b/plenum/test/test_node.py @@ -62,7 +62,7 @@ class TestCoreAuthnr(CoreAuthNr): class TestDomainRequestHandler(DomainRequestHandler): - write_types = DomainRequestHandler.write_types.union({'buy', 'randombuy',}) + write_types = DomainRequestHandler.write_types.union({'buy', 'randombuy', }) query_types = DomainRequestHandler.query_types.union({'get_buy', }) @staticmethod @@ -111,13 +111,13 @@ def __init__(self, *args, **kwargs): # is among the set of suspicion codes mapped to its name. If the set of # suspicion codes is empty then the node would not be blacklisted for # any suspicion code - self.whitelistedNodes = {} # type: Dict[str, Set[int]] + self.whitelistedNodes = {} # type: Dict[str, Set[int]] # Clients that wont be blacklisted by this node if the suspicion code # is among the set of suspicion codes mapped to its name. If the set of # suspicion codes is empty then the client would not be blacklisted for # suspicion code - self.whitelistedClients = {} # type: Dict[str, Set[int]] + self.whitelistedClients = {} # type: Dict[str, Set[int]] # Reinitialize the monitor d, l, o = self.monitor.Delta, self.monitor.Lambda, self.monitor.Omega @@ -235,7 +235,7 @@ def whitelistNode(self, nodeName: str, *codes: int): logger.debug("{} whitelisting {} for codes {}" .format(self, nodeName, codes)) - def blacklistNode(self, nodeName: str, reason: str=None, code: int=None): + def blacklistNode(self, nodeName: str, reason: str = None, code: int = None): if nodeName in self.whitelistedClients: # If node whitelisted for all codes if len(self.whitelistedClients[nodeName]) == 0: @@ -253,7 +253,7 @@ def whitelistClient(self, clientName: str, *codes: int): .format(self, clientName, codes)) def blacklistClient(self, clientName: str, - reason: str=None, code: int=None): + reason: str = None, code: int = None): if clientName in self.whitelistedClients: # If node whitelisted for all codes if len(self.whitelistedClients[clientName]) == 0: @@ -388,6 +388,7 @@ def sendRepliesToClients(self, committedTxns, ppTime): txn[STATE_PROOF] = proof super().sendRepliesToClients(committedTxns, ppTime) + elector_spyables = [ PrimaryElector.discard, PrimaryElector.processPrimary, @@ -423,12 +424,12 @@ class TestPrimarySelector(PrimarySelector): ViewChanger.startViewChange ] + @spyable(methods=view_changer_spyables) class TestViewChanger(ViewChanger): pass - replica_spyables = [ replica.Replica.sendPrePrepare, replica.Replica._can_process_pre_prepare, @@ -473,13 +474,13 @@ class TestNodeSet(ExitStack): def __init__(self, config, - names: Iterable[str]=None, - count: int=None, + names: Iterable[str] = None, + count: int = None, nodeReg=None, tmpdir=None, keyshare=True, primaryDecider=None, - pluginPaths: Iterable[str]=None, + pluginPaths: Iterable[str] = None, testNodeClass=TestNode): super().__init__() @@ -500,8 +501,8 @@ def __init__(self, self.nodeReg = nodeReg else: nodeNames = (names if names is not None and count is None else - genNodeNames(count) if count is not None else - error("only one of either names or count is required")) + genNodeNames(count) if count is not None else + error("only one of either names or count is required")) self.nodeReg = genNodeReg( names=nodeNames) # type: Dict[str, NodeDetail] for name in self.nodeReg.keys(): @@ -713,12 +714,12 @@ async def checkNodesConnected(nodes: Iterable[TestNode], acceptableExceptions=[AssertionError, RemoteNotFound]) -def checkNodeRemotes(node: TestNode, states: Dict[str, RemoteState]=None, +def checkNodeRemotes(node: TestNode, states: Dict[str, RemoteState] = None, state: RemoteState = None): assert states or state, "either state or states is required" assert not ( - states and state), "only one of state or states should be provided, " \ - "but not both" + states and state), "only one of state or states should be provided, " \ + "but not both" for remote in node.nodestack.remotes.values(): try: s = states[remote.name] if states else state @@ -743,8 +744,8 @@ def checkIfSameReplicaIPrimary(looper: Looper, def checkElectionDone(): unknowns = [r for r in replicas if r.primaryName is None] assert len(unknowns) == 0, "election should be complete, " \ - "but {} out of {} ({}) don't know who the primary " \ - "is for protocol instance {}".\ + "but {} out of {} ({}) don't know who the primary " \ + "is for protocol instance {}". \ format(len(unknowns), len(replicas), unknowns, replicas[0].instId) def checkPrisAreOne(): # number of expected primaries @@ -757,6 +758,7 @@ def checkPrisAreSame(): assert len(pris) == 1, "Primary should be same for all, but were {} " \ "for protocol no {}" \ .format(pris, replicas[0].instId) + looper.run( eventuallyAll(checkElectionDone, checkPrisAreOne, checkPrisAreSame, retryWait=retryWait, totalTimeout=timeout)) @@ -767,7 +769,7 @@ def checkNodesAreReady(nodes: Sequence[TestNode]): assert node.isReady(), '{} has status {}'.format(node, node.status) -async def checkNodesParticipating(nodes: Sequence[TestNode], timeout: int=None): +async def checkNodesParticipating(nodes: Sequence[TestNode], timeout: int = None): # TODO is this used? If so - add timeout for it to plenum.test.waits if not timeout: timeout = .75 * len(nodes) @@ -784,7 +786,6 @@ def checkEveryProtocolInstanceHasOnlyOnePrimary(looper: Looper, retryWait: float = None, timeout: float = None, numInstances: int = None): - coro = eventually(instances, nodes, numInstances, retryWait=retryWait, timeout=timeout) insts, timeConsumed = timeThis(looper.run, coro) @@ -818,7 +819,6 @@ def checkProtocolInstanceSetup(looper: Looper, retryWait: float = 1, customTimeout: float = None, numInstances: int = None): - timeout = customTimeout or waits.expectedPoolElectionTimeout(len(nodes)) checkEveryProtocolInstanceHasOnlyOnePrimary(looper=looper, @@ -926,7 +926,7 @@ def timeThis(func, *args, **kwargs): def instances(nodes: Sequence[Node], numInstances: int = None) -> Dict[int, List[replica.Replica]]: numInstances = (getRequiredInstances(len(nodes)) - if numInstances is None else numInstances) + if numInstances is None else numInstances) for n in nodes: assert len(n.replicas) == numInstances return {i: [n.replicas[i] for n in nodes] for i in range(numInstances)} @@ -1020,7 +1020,7 @@ def check_node_disconnected(disconnected: TestNode, def ensure_node_disconnected(looper: Looper, disconnected: TestNode, other_nodes: Iterable[TestNode], - timeout: float=None): + timeout: float = None): timeout = timeout or (len(other_nodes) - 1) looper.run(eventually(check_node_disconnected, disconnected, other_nodes, retryWait=1, timeout=timeout)) diff --git a/plenum/test/test_node_basic.py b/plenum/test/test_node_basic.py index b2abbd2b31..4b812e4a03 100644 --- a/plenum/test/test_node_basic.py +++ b/plenum/test/test_node_basic.py @@ -63,7 +63,7 @@ def testCorrectNumOfReplicas(pool): for instId in getProtocolInstanceNums(node): # num of replicas for a instance on a node must be 1 assert len([node.replicas[instId]]) == 1 and \ - node.replicas[instId].instId == instId + node.replicas[instId].instId == instId # num of primary on every protocol instance is 1 numberOfPrimary = len([node for node in pool.nodeset if node.replicas[instId].isPrimary]) diff --git a/plenum/test/test_node_request.py b/plenum/test/test_node_request.py index d28b847f4c..cbd85c676d 100644 --- a/plenum/test/test_node_request.py +++ b/plenum/test/test_node_request.py @@ -44,6 +44,7 @@ async def chk(): assert result else: assert result is False + timeout = waits.expectedOrderingTime( nodeSet.nodes['Alpha'].instances.count) looper.run(eventually(chk, timeout=timeout)) diff --git a/plenum/test/test_performance.py b/plenum/test/test_performance.py index 1051d64116..bc6f325459 100644 --- a/plenum/test/test_performance.py +++ b/plenum/test/test_performance.py @@ -28,6 +28,7 @@ from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ client1, wallet1, client1Connected + @pytest.fixture def logger(): logger = getlogger() @@ -36,13 +37,13 @@ def logger(): yield logger logger.root.setLevel(old_value) + # autouse and inject before others in all tests pytestmark = pytest.mark.usefixtures("logger") txnCount = 5 TestRunningTimeLimitSec = math.inf - """ Since these tests expect performance to be of certain level, they can fail and for now should only be run when a perf check is required, like after a relevant @@ -92,7 +93,6 @@ def test_node_load_consistent_time(tconf, change_checkpoint_freq, disable_node_monitor_config, looper, txnPoolNodeSet, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): - # One of the reason memory grows is because spylog grows client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, @@ -253,7 +253,7 @@ def test_node_load_after_add_then_disconnect(newNodeCaughtUp, txnPoolNodeSet, # Not accurate timeout but a conservative one timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \ - 2 * delay_catchup_reply + 2 * delay_catchup_reply waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4], customTimeout=timeout) @@ -274,7 +274,6 @@ def test_nodestack_contexts_are_discrete(txnPoolNodeSet): def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) @@ -319,7 +318,6 @@ def test_node_load_after_one_node_drops_all_msgs( allPluginsPath, poolTxnStewardData, capsys): - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, tdirWithPoolTxns, clientClass=TestClient) diff --git a/plenum/test/test_req_authenticator.py b/plenum/test/test_req_authenticator.py index dd4d4c8788..ec675b3398 100644 --- a/plenum/test/test_req_authenticator.py +++ b/plenum/test/test_req_authenticator.py @@ -18,7 +18,6 @@ def pre_reqs(): return simple_authnr, core_authnr, req_authnr - @pytest.fixture(scope='module') def registration(pre_reqs): simple_authnr, core_authnr, req_authnr = pre_reqs diff --git a/plenum/test/test_round_trip_with_one_faulty_node.py b/plenum/test/test_round_trip_with_one_faulty_node.py index 2605402f9f..0d8dcf18d4 100644 --- a/plenum/test/test_round_trip_with_one_faulty_node.py +++ b/plenum/test/test_round_trip_with_one_faulty_node.py @@ -49,4 +49,3 @@ def testRequestFullRoundTrip(node_doesnt_propagate, replied1): still be able to successfully complete a full cycle. """ pass - diff --git a/plenum/test/test_stack.py b/plenum/test/test_stack.py index 12f77cba25..da55d4ab13 100644 --- a/plenum/test/test_stack.py +++ b/plenum/test/test_stack.py @@ -14,7 +14,6 @@ logger = getlogger() - BaseStackClass = ZStack @@ -62,7 +61,7 @@ def checkIfConnectedTo(self, count=None): async def ensureConnectedToNodes(self, customTimeout=None, count=None): timeout = customTimeout or \ - waits.expectedClientToPoolConnectionTimeout(len(self.nodeReg)) + waits.expectedClientToPoolConnectionTimeout(len(self.nodeReg)) logger.debug( "waiting for {} seconds to check client connections to " @@ -109,7 +108,7 @@ def getTestableStack(stack: NetworkInterface): JOINED = RemoteState(isConnected=False) -def checkState(state: RemoteState, obj: Any, details: str=None): +def checkState(state: RemoteState, obj: Any, details: str = None): if state is not None: checkedItems = {} for key, s in state._asdict().items(): diff --git a/plenum/test/test_testable.py b/plenum/test/test_testable.py index 59ae63ae7a..7093a4f21f 100644 --- a/plenum/test/test_testable.py +++ b/plenum/test/test_testable.py @@ -6,7 +6,6 @@ from plenum.server.node import Node from plenum.test.testable import spyable - pr = slice(3, 5) # params and result logger = getlogger() diff --git a/plenum/test/testable.py b/plenum/test/testable.py index a4dea73f6b..668520da3f 100644 --- a/plenum/test/testable.py +++ b/plenum/test/testable.py @@ -126,12 +126,12 @@ def decorator(clas): if callable(getattr(clas, method))]: isInit = nm == "__init__" matched = (nm if methods and nm in methods else - func if methods and func in methods else - None) + func if methods and func in methods else + None) # if method was specified to be spied on or is `__init__` method # or is does not have name starting with `__` shouldSpy = bool(matched) if methods else ( - not nm.startswith("__") or isInit) + not nm.startswith("__") or isInit) if shouldSpy or isInit: newFunc = spy(func, isInit, shouldSpy) morphed[func] = newFunc diff --git a/plenum/test/validator_info/test_validator_info.py b/plenum/test/validator_info/test_validator_info.py index 5b9d5301fe..a22f939b16 100644 --- a/plenum/test/validator_info/test_validator_info.py +++ b/plenum/test/validator_info/test_validator_info.py @@ -22,7 +22,6 @@ from stp_core.common.constants import ZMQ_NETWORK_PROTOCOL from stp_core.loop.eventually import eventually - TEST_NODE_NAME = 'Alpha' INFO_FILENAME = '{}_info.json'.format(TEST_NODE_NAME.lower()) PERIOD_SEC = 1 @@ -242,6 +241,7 @@ def read_wrapped(txn_type): retryWait=1, timeout=timeout)) txnPoolNodesLooper.runFor(patched_dump_info_period) return load_info(info_path) + return read_wrapped @@ -257,6 +257,7 @@ def write_wrapped(): waitForSufficientRepliesForRequests(txnPoolNodesLooper, client, requests=[req]) txnPoolNodesLooper.runFor(patched_dump_info_period) return load_info(info_path) + return write_wrapped @@ -265,6 +266,7 @@ def load_latest_info(txnPoolNodesLooper, patched_dump_info_period, info_path): def wrapped(): txnPoolNodesLooper.runFor(patched_dump_info_period + 1) return load_info(info_path) + return wrapped diff --git a/plenum/test/view_change/helper.py b/plenum/test/view_change/helper.py index 6258da846a..451d38763e 100644 --- a/plenum/test/view_change/helper.py +++ b/plenum/test/view_change/helper.py @@ -35,7 +35,6 @@ def start_stopped_node(stopped_node, looper, tconf, def provoke_and_check_view_change(nodes, newViewNo, wallet, client): - if {n.viewNo for n in nodes} == {newViewNo}: return True @@ -124,45 +123,45 @@ def slow_master(self): def ensure_several_view_change(looper, nodes, vc_count=1, exclude_from_check=None, custom_timeout=None): - """ - This method patches the master performance check to return False and thus - ensures that all given nodes do a view change - Also, this method can do several view change. - If you try do several view_change by calling ensure_view_change, - than monkeypatching method isMasterDegraded would work unexpectedly. - Therefore, we return isMasterDegraded only after doing view_change needed count - """ - old_meths = {} - view_changes = {} - expected_view_no = None - for node in nodes: - old_meths[node.name] = node.monitor.isMasterDegraded - - for __ in range(vc_count): - old_view_no = checkViewNoForNodes(nodes) - expected_view_no = old_view_no + 1 - - for node in nodes: - view_changes[node.name] = node.monitor.totalViewChanges - - def slow_master(self): - # Only allow one view change - rv = self.totalViewChanges == view_changes[self.name] - if rv: - logger.info('{} making master look slow'.format(self)) - return rv - - node.monitor.isMasterDegraded = types.MethodType(slow_master, node.monitor) - - perf_check_freq = next(iter(nodes)).config.PerfCheckFreq - timeout = custom_timeout or waits.expectedPoolViewChangeStartedTimeout(len(nodes)) + perf_check_freq - nodes_to_check = nodes if exclude_from_check is None else [n for n in nodes if n not in exclude_from_check] - logger.debug('Checking view no for nodes {}'.format(nodes_to_check)) - looper.run(eventually(checkViewNoForNodes, nodes_to_check, expected_view_no, retryWait=1, timeout=timeout)) - ensureElectionsDone(looper=looper, nodes=nodes, customTimeout=timeout) - ensure_all_nodes_have_same_data(looper, nodes, custom_timeout=timeout, exclude_from_check=exclude_from_check) - - return expected_view_no + """ + This method patches the master performance check to return False and thus + ensures that all given nodes do a view change + Also, this method can do several view change. + If you try do several view_change by calling ensure_view_change, + than monkeypatching method isMasterDegraded would work unexpectedly. + Therefore, we return isMasterDegraded only after doing view_change needed count + """ + old_meths = {} + view_changes = {} + expected_view_no = None + for node in nodes: + old_meths[node.name] = node.monitor.isMasterDegraded + + for __ in range(vc_count): + old_view_no = checkViewNoForNodes(nodes) + expected_view_no = old_view_no + 1 + + for node in nodes: + view_changes[node.name] = node.monitor.totalViewChanges + + def slow_master(self): + # Only allow one view change + rv = self.totalViewChanges == view_changes[self.name] + if rv: + logger.info('{} making master look slow'.format(self)) + return rv + + node.monitor.isMasterDegraded = types.MethodType(slow_master, node.monitor) + + perf_check_freq = next(iter(nodes)).config.PerfCheckFreq + timeout = custom_timeout or waits.expectedPoolViewChangeStartedTimeout(len(nodes)) + perf_check_freq + nodes_to_check = nodes if exclude_from_check is None else [n for n in nodes if n not in exclude_from_check] + logger.debug('Checking view no for nodes {}'.format(nodes_to_check)) + looper.run(eventually(checkViewNoForNodes, nodes_to_check, expected_view_no, retryWait=1, timeout=timeout)) + ensureElectionsDone(looper=looper, nodes=nodes, customTimeout=timeout) + ensure_all_nodes_have_same_data(looper, nodes, custom_timeout=timeout, exclude_from_check=exclude_from_check) + + return expected_view_no def ensure_view_change_by_primary_restart( @@ -212,7 +211,7 @@ def check_each_node_reaches_same_end_for_view(nodes, view_no): for node in nodes: params = [e.params for e in node.replicas[0].spylog.getAll( node.replicas[0].primary_changed.__name__) - if e.params['view_no'] == view_no] + if e.params['view_no'] == view_no] assert params args[node.name] = (params[0]['last_ordered_pp_seq_no'], params[0]['ledger_summary']) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py index 5e44d1e22d..f6edb486c0 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py @@ -2,7 +2,6 @@ from plenum.test.pool_transactions.conftest import clientAndWallet1, \ client1, wallet1, client1Connected, looper - TestRunningTimeLimitSec = 300 diff --git a/plenum/test/view_change/slow_nodes/test_view_change_complex.py b/plenum/test/view_change/slow_nodes/test_view_change_complex.py index f2a8e1d2a1..c04a497a47 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_complex.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_complex.py @@ -3,7 +3,6 @@ from plenum.test.pool_transactions.conftest import clientAndWallet1, \ client1, wallet1, client1Connected, looper - TestRunningTimeLimitSec = 300 diff --git a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py index b21143670b..b0aebe2d30 100644 --- a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py +++ b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py @@ -6,9 +6,9 @@ from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState, POOL_LEDGER_ID from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies -from plenum.test.pool_transactions.conftest import wallet1, client1,\ -client1Connected, looper, stewardAndWallet1, steward1, \ - stewardWallet +from plenum.test.pool_transactions.conftest import wallet1, client1, \ + client1Connected, looper, stewardAndWallet1, steward1, \ + stewardWallet from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually @@ -20,7 +20,6 @@ from plenum.test import waits from plenum.common.startable import Mode - logger = getlogger() @@ -47,7 +46,7 @@ def catchuped(node): def add_new_node(looper, nodes, steward, steward_wallet, - tdir, client_tdir, tconf, all_plugins_path, name=None): + tdir, client_tdir, tconf, all_plugins_path, name=None): node_name = name or "Psi" new_steward_name = "testClientSteward" + randomString(3) new_steward, new_steward_wallet, new_node = addNewStewardAndNode(looper, @@ -68,9 +67,9 @@ def add_new_node(looper, nodes, steward, steward_wallet, def test_6th_node_join_after_view_change_by_master_restart( - looper, txnPoolNodeSet, tdir, tconf, - allPluginsPath, steward1, stewardWallet, - client_tdir, limitTestRunningTime): + looper, txnPoolNodeSet, tdir, tconf, + allPluginsPath, steward1, stewardWallet, + client_tdir, limitTestRunningTime): """ Test steps: 1. start pool of 4 nodes @@ -84,11 +83,11 @@ def test_6th_node_join_after_view_change_by_master_restart( pool_of_nodes = txnPoolNodeSet for __ in range(4): pool_of_nodes = ensure_view_change_by_primary_restart(looper, - pool_of_nodes, - tconf, - tdir, - allPluginsPath, - customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT) + pool_of_nodes, + tconf, + tdir, + allPluginsPath, + customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT) timeout = waits.expectedPoolCatchupTime(nodeCount=len(pool_of_nodes)) for node in pool_of_nodes: looper.run(eventually(catchuped, node, timeout=2 * timeout)) @@ -116,11 +115,11 @@ def test_6th_node_join_after_view_change_by_master_restart( LedgerState.synced, retryWait=.5, timeout=timeout)) for __ in range(4): pool_of_nodes = ensure_view_change_by_primary_restart(looper, - pool_of_nodes, - tconf, - tdir, - allPluginsPath, - customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT) + pool_of_nodes, + tconf, + tdir, + allPluginsPath, + customTimeout=2 * tconf.VIEW_CHANGE_TIMEOUT) timeout = waits.expectedPoolCatchupTime(nodeCount=len(pool_of_nodes)) for node in pool_of_nodes: @@ -139,5 +138,3 @@ def test_6th_node_join_after_view_change_by_master_restart( LedgerState.synced, retryWait=.5, timeout=5)) looper.run(eventually(check_ledger_state, new_psi_node, POOL_LEDGER_ID, LedgerState.synced, retryWait=.5, timeout=5)) - - diff --git a/plenum/test/view_change/test_discard_inst_chng_msg_from_past_view.py b/plenum/test/view_change/test_discard_inst_chng_msg_from_past_view.py index 994df2e444..a8b5965fce 100644 --- a/plenum/test/view_change/test_discard_inst_chng_msg_from_past_view.py +++ b/plenum/test/view_change/test_discard_inst_chng_msg_from_past_view.py @@ -55,4 +55,4 @@ def testDoNotSendInstChngMsgIfMasterDoesntSeePerformanceProblem( # `sendInstanceChange` for n in nodeSet: assert n.spylog.count(instChngMethodName) == \ - sentInstChanges.get(n.name, 0) + sentInstChanges.get(n.name, 0) diff --git a/plenum/test/view_change/test_master_primary_different_from_previous.py b/plenum/test/view_change/test_master_primary_different_from_previous.py index 5a9ad58ba0..1ed7b96a8c 100644 --- a/plenum/test/view_change/test_master_primary_different_from_previous.py +++ b/plenum/test/view_change/test_master_primary_different_from_previous.py @@ -10,6 +10,7 @@ from plenum.test.view_change.helper import provoke_and_wait_for_view_change, ensure_view_change from stp_core.common.log import getlogger + logger = getlogger() diff --git a/plenum/test/view_change/test_new_node_joins_after_view_change.py b/plenum/test/view_change/test_new_node_joins_after_view_change.py index 513af5e2b3..e3bcedf125 100644 --- a/plenum/test/view_change/test_new_node_joins_after_view_change.py +++ b/plenum/test/view_change/test_new_node_joins_after_view_change.py @@ -18,6 +18,7 @@ one_node_added from stp_core.common.log import getlogger + logger = getlogger() diff --git a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py index 2ca91d5722..b3c62831a9 100644 --- a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py +++ b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py @@ -14,7 +14,6 @@ from plenum.test.pool_transactions.conftest import clientAndWallet1, \ client1, wallet1, client1Connected, looper - TestRunningTimeLimitSec = 150 diff --git a/plenum/test/view_change/test_queueing_req_from_future_view.py b/plenum/test/view_change/test_queueing_req_from_future_view.py index 27589db776..3537b39ca0 100644 --- a/plenum/test/view_change/test_queueing_req_from_future_view.py +++ b/plenum/test/view_change/test_queueing_req_from_future_view.py @@ -1,4 +1,3 @@ - import pytest from plenum.test.view_change.helper import provoke_and_wait_for_view_change, ensure_view_change diff --git a/plenum/test/view_change/test_reverted_unordered.py b/plenum/test/view_change/test_reverted_unordered.py index f9848a12a6..dc303dffd4 100644 --- a/plenum/test/view_change/test_reverted_unordered.py +++ b/plenum/test/view_change/test_reverted_unordered.py @@ -11,7 +11,6 @@ from plenum.test.test_node import getNonPrimaryReplicas, ensureElectionsDone from plenum.test.view_change.helper import ensure_view_change - TestRunningTimeLimitSec = 150 diff --git a/plenum/test/view_change/test_that_domain_ledger_the_same_after_restart_for_all_nodes.py b/plenum/test/view_change/test_that_domain_ledger_the_same_after_restart_for_all_nodes.py index 5eb35f3522..2f5eebe211 100644 --- a/plenum/test/view_change/test_that_domain_ledger_the_same_after_restart_for_all_nodes.py +++ b/plenum/test/view_change/test_that_domain_ledger_the_same_after_restart_for_all_nodes.py @@ -5,7 +5,6 @@ from stp_core.common.log import getlogger from plenum.common.startable import Mode - logger = getlogger() @@ -23,8 +22,8 @@ def catchuped(node): def test_that_domain_ledger_the_same_after_restart_for_all_nodes( - looper, txnPoolNodeSet, tdir, tconf, - allPluginsPath, limitTestRunningTime): + looper, txnPoolNodeSet, tdir, tconf, + allPluginsPath, limitTestRunningTime): """ Test steps: 1. Collect domainLedger data for primary node, such as: @@ -66,7 +65,7 @@ def prepare_for_compare(domain_ledger): return dict_for_compare def compare(before, after): - for k,v in before.items(): + for k, v in before.items(): if k in after: if v != after[k]: logger.debug("compare_domain_ledgers: before[{}]!=after[{}]".format(k, k)) @@ -77,20 +76,19 @@ def compare(before, after): logger.debug("compare_domain_ledgers: after_dict: {}: {}".format(k, after.get(k))) assert False - pool_of_nodes = txnPoolNodeSet for __ in range(4): p_node = [node for node in pool_of_nodes if node.has_master_primary][0] before_vc_dict = prepare_for_compare(p_node.domainLedger) pool_of_nodes = ensure_view_change_by_primary_restart(looper, - pool_of_nodes, - tconf, - tdir, - allPluginsPath, - customTimeout=tconf.VIEW_CHANGE_TIMEOUT) + pool_of_nodes, + tconf, + tdir, + allPluginsPath, + customTimeout=tconf.VIEW_CHANGE_TIMEOUT) for node in pool_of_nodes: logger.debug("compare_domain_ledgers: " "primary node before view_change: {}, " "compared node: {}".format(p_node, node)) after_vc_dict = prepare_for_compare(node.domainLedger) - compare(before_vc_dict, after_vc_dict) \ No newline at end of file + compare(before_vc_dict, after_vc_dict) diff --git a/plenum/test/view_change/test_view_change_after_back_to_quorum_with_disconnected_primary.py b/plenum/test/view_change/test_view_change_after_back_to_quorum_with_disconnected_primary.py index f120b736ea..bc30e72121 100644 --- a/plenum/test/view_change/test_view_change_after_back_to_quorum_with_disconnected_primary.py +++ b/plenum/test/view_change/test_view_change_after_back_to_quorum_with_disconnected_primary.py @@ -7,11 +7,11 @@ from plenum.test.helper import checkViewNoForNodes, waitForViewChange, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data - # We do 2 view changes during this test. Timeout for one view change is 60 sec. # Test running time will be expected near 2 * 60 = 120, so let's define it as 150 sec. TestRunningTimeLimitSec = 150 + def test_view_change_after_back_to_quorum_with_disconnected_primary(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client, diff --git a/plenum/test/view_change/test_view_change_n_minus_f_quorum.py b/plenum/test/view_change/test_view_change_n_minus_f_quorum.py index 7a852d2c21..3a854afee8 100644 --- a/plenum/test/view_change/test_view_change_n_minus_f_quorum.py +++ b/plenum/test/view_change/test_view_change_n_minus_f_quorum.py @@ -45,5 +45,4 @@ def test_view_change_n_minus_f_quorum(txnPoolNodeSet, looper): "Delta -> Ratio: None" .format(current_view_no, current_view_no + 1)) as exc_info: - ensure_view_change(looper, active) diff --git a/plenum/test/view_change/test_view_change_on_master_degraded.py b/plenum/test/view_change/test_view_change_on_master_degraded.py index 0fece1ef76..a2d18a406b 100644 --- a/plenum/test/view_change/test_view_change_on_master_degraded.py +++ b/plenum/test/view_change/test_view_change_on_master_degraded.py @@ -13,6 +13,7 @@ nodeCount = 7 + # noinspection PyIncorrectDocstring @@ -71,10 +72,10 @@ def test_view_change_on_quorum_of_master_degraded(nodeSet, looper, up, for n in nodeSet: if n.name != relucatantNode.name: assert n.view_changer.spylog.count(instChngMethodName) > \ - sentInstChanges.get(n.name, 0) + sentInstChanges.get(n.name, 0) else: assert n.view_changer.spylog.count(instChngMethodName) == \ - sentInstChanges.get(n.name, 0) + sentInstChanges.get(n.name, 0) ensureElectionsDone(looper=looper, nodes=nodeSet) new_m_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) diff --git a/plenum/test/view_change/test_view_change_start_without_primary.py b/plenum/test/view_change/test_view_change_start_without_primary.py index 8bdaf8a804..49d410a02e 100644 --- a/plenum/test/view_change/test_view_change_start_without_primary.py +++ b/plenum/test/view_change/test_view_change_start_without_primary.py @@ -10,7 +10,6 @@ def test_view_change_without_primary(nodeSet, looper, patched_view_change_timeout): - first, others = stop_nodes_and_remove_first(looper, nodeSet) start_and_connect_nodes(looper, others) @@ -28,7 +27,7 @@ def stop_nodes_and_remove_first(looper, nodes): looper.removeProdable(first_node) looper.runFor(3) # let the nodes stop return first_node, \ - list(filter(lambda x: x.name != first_node.name, nodes)) + list(filter(lambda x: x.name != first_node.name, nodes)) def start_and_connect_nodes(looper, nodes): diff --git a/plenum/test/view_change/test_view_change_without_any_reqs.py b/plenum/test/view_change/test_view_change_without_any_reqs.py index caeae77135..b4ba0f8910 100644 --- a/plenum/test/view_change/test_view_change_without_any_reqs.py +++ b/plenum/test/view_change/test_view_change_without_any_reqs.py @@ -13,7 +13,6 @@ from plenum.test.batching_3pc.conftest import tconf from plenum.test.pool_transactions.conftest import looper - TestRunningTimeLimitSec = 200 @@ -38,7 +37,7 @@ def chk1(): looper.run(eventually(chk1, retryWait=1)) timeout = tconf.PerfCheckFreq + \ - waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) waitForViewChange(looper, txnPoolNodeSet, old_view_no + 1, customTimeout=timeout) diff --git a/plenum/test/view_change/test_view_changes_if_master_primary_disconnected.py b/plenum/test/view_change/test_view_changes_if_master_primary_disconnected.py index 53273a4425..27ac2bef3d 100644 --- a/plenum/test/view_change/test_view_changes_if_master_primary_disconnected.py +++ b/plenum/test/view_change/test_view_changes_if_master_primary_disconnected.py @@ -1,4 +1,3 @@ - from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected from plenum.test.spy_helpers import getAllReturnVals from plenum.test.view_change.helper import start_stopped_node diff --git a/plenum/test/view_change/test_view_not_changed_when_short_disconnection.py b/plenum/test/view_change/test_view_not_changed_when_short_disconnection.py index db7756ba38..b97112b7ab 100644 --- a/plenum/test/view_change/test_view_not_changed_when_short_disconnection.py +++ b/plenum/test/view_change/test_view_not_changed_when_short_disconnection.py @@ -33,7 +33,7 @@ def chk1(): for node in txnPoolNodeSet: if node != pr_node: assert node.spylog.count(node.lost_master_primary.__name__) \ - > lost_pr_calls[node.name] + > lost_pr_calls[node.name] def chk2(): # Schedule an instance change but do not send it @@ -41,9 +41,9 @@ def chk2(): for node in txnPoolNodeSet: if node != pr_node: assert node.spylog.count(node.propose_view_change.__name__) \ - > prp_inst_chg_calls[node.name] + > prp_inst_chg_calls[node.name] assert node.view_changer.spylog.count(node.view_changer.process_instance_change_msg.__name__) \ - == recv_inst_chg_calls[node.name] + == recv_inst_chg_calls[node.name] # Disconnect master's primary for node in txnPoolNodeSet: diff --git a/plenum/test/waits.py b/plenum/test/waits.py index ac272bcb6c..9e0e89f977 100644 --- a/plenum/test/waits.py +++ b/plenum/test/waits.py @@ -7,7 +7,6 @@ logger = getlogger() - # Peer (node/client) to peer message delivery time __Peer2PeerRequestDeliveryTime = 0.5 __Peer2PeerRequestExchangeTime = 2 * __Peer2PeerRequestDeliveryTime @@ -94,7 +93,7 @@ def expectedPoolConsistencyProof(nodeCount): """ config = getConfig() nodeCPTimeout = __Peer2PeerRequestExchangeTime + \ - config.ConsistencyProofsTimeout + config.ConsistencyProofsTimeout return nodeCount * nodeCPTimeout @@ -113,8 +112,8 @@ def expectedPoolGetReadyTimeout(nodeCount): To: the pool ledger is equal across the Nodes """ return expectedPoolInterconnectionTime(nodeCount) + \ - expectedPoolConsistencyProof(nodeCount) + \ - expectedPoolCatchupTime(nodeCount) + expectedPoolConsistencyProof(nodeCount) + \ + expectedPoolCatchupTime(nodeCount) def expectedPoolLedgerRepliedMsgPersisted(nodeCount): @@ -240,7 +239,7 @@ def expectedClientToPoolConnectionTimeout(nodeCount): # TODO check actual state config = getConfig() return config.ExpectedConnectTime * nodeCount + \ - config.RETRY_TIMEOUT_RESTRICTED + config.RETRY_TIMEOUT_RESTRICTED def expectedClientConsistencyProof(nodeCount): @@ -251,7 +250,7 @@ def expectedClientConsistencyProof(nodeCount): config = getConfig() qN = Quorums(nodeCount).commit.value return qN * __Peer2PeerRequestExchangeTime + \ - config.ConsistencyProofsTimeout + config.ConsistencyProofsTimeout def expectedClientCatchupTime(nodeCount): @@ -262,7 +261,7 @@ def expectedClientCatchupTime(nodeCount): config = getConfig() qN = Quorums(nodeCount).commit.value return qN * 2 * __Peer2PeerRequestExchangeTime + \ - config.CatchupTransactionsTimeout + config.CatchupTransactionsTimeout def expectedClientToPoolRequestDeliveryTime(nodeCount): diff --git a/plenum/test/wallet/test_wallet_storage_helper.py b/plenum/test/wallet/test_wallet_storage_helper.py index a35f6b825d..1aa1ba850c 100644 --- a/plenum/test/wallet/test_wallet_storage_helper.py +++ b/plenum/test/wallet/test_wallet_storage_helper.py @@ -94,7 +94,6 @@ def test_keyring_base_dir_exists_as_dir(tdir_hierarchy): def test_store_wallet_by_empty_path_fail( tdir_for_func, keyrings_base_dir, test_wallet): - wsh = WalletStorageHelper(keyrings_base_dir) for path in (None, ''): @@ -106,7 +105,6 @@ def test_store_wallet_by_empty_path_fail( def test_store_wallet_outside_fail( tdir_for_func, keyrings_base_dir, test_wallet): - wsh = WalletStorageHelper(keyrings_base_dir) inv_paths = [ @@ -202,7 +200,6 @@ def test_stored_wallet_data(tdir_for_func, keyrings_base_dir, test_wallet): def test_load_wallet_by_empty_path_fail(tdir_for_func, keyrings_base_dir): - wsh = WalletStorageHelper(keyrings_base_dir) for path in (None, ''): @@ -213,7 +210,6 @@ def test_load_wallet_by_empty_path_fail(tdir_for_func, keyrings_base_dir): def test_load_wallet_outside_fail(tdir_for_func, keyrings_base_dir): - wsh = WalletStorageHelper(keyrings_base_dir) inv_paths = [ diff --git a/plenum/test/zstack_tests/test_zstack_reconnection.py b/plenum/test/zstack_tests/test_zstack_reconnection.py index 5bafd9c00a..5361952024 100644 --- a/plenum/test/zstack_tests/test_zstack_reconnection.py +++ b/plenum/test/zstack_tests/test_zstack_reconnection.py @@ -12,7 +12,6 @@ logger = getlogger() - TestRunningTimeLimitSec = 300 From 72dc0ad367143bfef6b9441c96147d0e57c86a22 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 13 Mar 2018 19:29:13 +0300 Subject: [PATCH 09/73] Script fix. bls validation fix Signed-off-by: ArtObr --- plenum/common/messages/client_request.py | 4 ++-- plenum/test/bls/helper.py | 5 +++-- plenum/test/conftest.py | 18 +++++++++--------- plenum/test/pool_transactions/helper.py | 8 ++++---- plenum/test/sdk/test_sdk_bindings.py | 14 ++++++++++++++ scripts/generate_txns.py | 8 ++++---- 6 files changed, 36 insertions(+), 21 deletions(-) diff --git a/plenum/common/messages/client_request.py b/plenum/common/messages/client_request.py index 1b8e98047a..8ab96c6339 100644 --- a/plenum/common/messages/client_request.py +++ b/plenum/common/messages/client_request.py @@ -7,7 +7,7 @@ ChooseField, ConstantField, DestNodeField, VerkeyField, DestNymField, \ RoleField, TxnSeqNoField, IdentifierField, \ NonNegativeNumberField, SignatureField, MapField, LimitedLengthStringField, \ - ProtocolVersionField, LedgerIdField + ProtocolVersionField, LedgerIdField, Base58Field from plenum.common.messages.message_base import MessageValidator from plenum.common.types import OPERATION, f from plenum.config import ALIAS_FIELD_LIMIT, DIGEST_FIELD_LIMIT, \ @@ -22,7 +22,7 @@ class ClientNodeOperationData(MessageValidator): (CLIENT_PORT, NetworkPortField(optional=True)), (ALIAS, LimitedLengthStringField(max_length=ALIAS_FIELD_LIMIT)), (SERVICES, IterableField(ChooseField(values=(VALIDATOR,)), optional=True)), - (BLS_KEY, LimitedLengthStringField(max_length=BLS_KEY_LIMIT, optional=True)), + (BLS_KEY, Base58Field(byte_lengths=BLS_KEY_LIMIT, optional=True)), ) def _validate_message(self, dct): diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 84236b4cd8..7527e707f4 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -1,6 +1,8 @@ import base58 import os +from plenum.test.input_validation.utils import b58_by_len + from plenum.test.node_request.helper import sdk_ensure_pool_functional from common.serializers.serialization import state_roots_serializer @@ -228,8 +230,7 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, key_in_txn = new_bls or new_blspk \ if not add_wrong \ - else ''.join(random_from_alphabet(32, base58.alphabet)) - + else b58_by_len(32) node_dest = hexToFriendly(node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, diff --git a/plenum/test/conftest.py b/plenum/test/conftest.py index 72e3d3c26e..6deb24a707 100644 --- a/plenum/test/conftest.py +++ b/plenum/test/conftest.py @@ -1080,6 +1080,15 @@ def sdk_wallet_steward(looper, sdk_wallet_handle, sdk_steward_seed): return sdk_wallet_handle, steward_did +@pytest.fixture(scope='module') +def sdk_wallet_new_steward(looper, sdk_pool_handle, sdk_wallet_steward): + wh, client_did = sdk_add_new_nym(looper, sdk_pool_handle, + sdk_wallet_steward, + alias='new_steward_qwerty', + role='STEWARD') + return wh, client_did + + @pytest.fixture(scope='module') def sdk_wallet_stewards(looper, sdk_wallet_handle, poolTxnStewardNames, poolTxnData): stewards = [] @@ -1116,12 +1125,3 @@ def sdk_wallet_new_client(looper, sdk_pool_handle, sdk_wallet_steward, sdk_wallet_steward, seed=sdk_new_client_seed) return wh, client_did - - -@pytest.fixture(scope='module') -def sdk_wallet_new_steward(looper, sdk_pool_handle, sdk_wallet_steward): - wh, client_did = sdk_add_new_nym(looper, sdk_pool_handle, - sdk_wallet_steward, - alias='new_steward_qwerty', - role='STEWARD') - return wh, client_did diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index dba3ba87c5..bd2646f2a2 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -370,7 +370,7 @@ async def prepare_nym_request(wallet, named_seed, alias, role): async def prepare_node_request(steward_did, new_node_name=None, clientIp=None, - clientPort=None, nodeIp=None, nodePort=None, blsKey=None, + clientPort=None, nodeIp=None, nodePort=None, bls_key=None, sigseed=None, destination=None, services=[VALIDATOR]): use_sigseed = sigseed is not None use_dest = destination is not None @@ -391,8 +391,8 @@ async def prepare_node_request(steward_did, new_node_name=None, clientIp=None, data['node_ip'] = nodeIp if nodePort is not None: data['node_port'] = nodePort - if blsKey is not None: - data['blskey'] = blsKey + if bls_key is not None: + data['blskey'] = bls_key if services is not None: data['services'] = services @@ -437,7 +437,7 @@ def sdk_send_update_node(looper, sdk_submitter_wallet, clientPort=client_port, nodeIp=node_ip, nodePort=node_port, - blsKey=bls_key, + bls_key=bls_key, destination=destination, services=services)) diff --git a/plenum/test/sdk/test_sdk_bindings.py b/plenum/test/sdk/test_sdk_bindings.py index a0af950371..faec4593d7 100644 --- a/plenum/test/sdk/test_sdk_bindings.py +++ b/plenum/test/sdk/test_sdk_bindings.py @@ -63,6 +63,12 @@ def test_sdk_client_send(looper, sdk_pool_handle, sdk_wallet_client): assert j_resp['result'] +def test_sdk_client2_send(looper, sdk_pool_handle, sdk_wallet_client2): + resp_task = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client2) + _, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0] + assert j_resp['result'] + + def test_sdk_new_client_send(looper, sdk_pool_handle, sdk_wallet_new_client): resp_task = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_new_client) _, j_resp = sdk_get_and_check_replies(looper, [resp_task])[0] @@ -82,6 +88,14 @@ def test_sdk_steward_send_many(looper, sdk_pool_handle, sdk_wallet_steward): assert resp['result'] +def test_sdk_many_stewards_send_many(looper, sdk_pool_handle, sdk_wallet_stewards): + for sdk_wallet_steward in sdk_wallet_stewards: + resp_task = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_steward, 30) + repl = sdk_get_and_check_replies(looper, resp_task) + for _, resp in repl: + assert resp['result'] + + def test_sdk_pool_refresh(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): sdk_pool_refresh(looper, sdk_pool_handle) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, diff --git a/scripts/generate_txns.py b/scripts/generate_txns.py index b0900b10c6..2d6cb8cab6 100644 --- a/scripts/generate_txns.py +++ b/scripts/generate_txns.py @@ -13,7 +13,7 @@ from plenum.common.config_util import getConfig from plenum.common.txn_util import sdk_reqToTxn from indy.ledger import sign_request -from indy import signus, wallet +from indy import did, wallet from stp_core.loop.looper import Looper config = getConfig() @@ -31,12 +31,12 @@ async def get_wallet_and_pool(): await wallet.create_wallet(pool_name, their_wallet_name, None, None, None) their_wallet_handle = await wallet.open_wallet(their_wallet_name, None, None) - await signus.create_and_store_my_did(my_wallet_handle, "{}") + await did.create_and_store_my_did(my_wallet_handle, "{}") - (their_did, their_verkey) = await signus.create_and_store_my_did(their_wallet_handle, + (their_did, their_verkey) = await did.create_and_store_my_did(their_wallet_handle, json.dumps({"seed": seed_trustee1})) - await signus.store_their_did(my_wallet_handle, json.dumps({'did': their_did, 'verkey': their_verkey})) + await did.store_their_did(my_wallet_handle, json.dumps({'did': their_did, 'verkey': their_verkey})) return their_wallet_handle, their_did From 6cca8c5f4357e5bc5983305b3cecc342a5c5a4c4 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 14 Mar 2018 08:15:04 +0300 Subject: [PATCH 10/73] more changes Signed-off-by: ArtObr --- ...not_become_active_with_less_than_four_servers.py | 1 - .../test/instances/test_msgs_from_slow_instances.py | 1 - plenum/test/instances/test_multiple_commit.py | 1 - plenum/test/instances/test_multiple_pre_prepare.py | 1 - plenum/test/instances/test_multiple_prepare.py | 1 - plenum/test/instances/test_pre_prepare_digest.py | 1 - plenum/test/logging/test_logging_txn_state.py | 1 - plenum/test/monitoring/test_avg_latency.py | 1 - .../test_instance_change_with_req_Lambda.py | 1 - .../monitoring/test_monitoring_params_with_zfn.py | 2 -- .../monitoring/test_no_check_if_no_new_requests.py | 1 - .../test/monitoring/test_post_monitoring_stats.py | 2 -- plenum/test/monitoring/test_throughput.py | 1 - plenum/test/node_catchup/conftest.py | 4 ---- .../node_catchup/test_no_catchup_if_got_from_3pc.py | 2 -- .../test_node_catchup_causes_no_desync.py | 3 --- ...ode_catchup_when_3_not_primary_node_restarted.py | 3 --- .../test/node_catchup/test_node_ledger_manager.py | 5 ----- plenum/test/node_request/conftest.py | 1 - ...est_node_request_missing_three_phase_messages.py | 1 - .../test_node_requests_missing_preprepare.py | 1 - ...three_phase_messages_after_long_disconnection.py | 1 - .../message_request/test_preprepare_request.py | 1 - .../test_requested_preprepare_handling.py | 1 - .../test_belated_request_not_processed.py | 1 - .../test_commit/test_commits_recvd_first.py | 1 - .../test_commit/test_commits_without_prepares.py | 1 - .../test_ordering_when_pre_prepare_not_received.py | 1 - .../test_order/test_request_ordering_1.py | 1 - ...nore_pre_prepare_pp_seq_no_less_than_expected.py | 2 -- .../test_node_lacks_finalised_requests.py | 1 - ...rum_f_plus_2_nodes_but_not_primary_off_and_on.py | 2 -- ...m_f_plus_2_nodes_including_primary_off_and_on.py | 2 -- plenum/test/node_request/test_request_forwarding.py | 1 - plenum/test/node_request/test_timestamp/conftest.py | 3 --- plenum/test/observer/conftest.py | 2 -- plenum/test/plugin/demo_plugin/conftest.py | 4 ---- .../test_change_ha_persists_post_nodes_restart.py | 1 - plenum/test/primary_selection/conftest.py | 3 --- .../test_catchup_after_view_change.py | 4 ---- .../test_catchup_multiple_rounds.py | 3 --- ...ction_after_primary_demotion_and_pool_restart.py | 1 - ...election_after_primary_demotion_and_promotion.py | 1 - ...ction_after_primary_demotion_and_view_changes.py | 1 - .../test_recover_more_than_f_failure.py | 9 ++------- .../test_recover_primary_no_view_change.py | 13 +++---------- .../test_selection_f_plus_one_quorum.py | 3 --- plenum/test/sdk/test_sdk_bindings.py | 1 - plenum/test/validator_info/test_validator_info.py | 1 - ...est_view_change_2_of_4_nodes_with_new_primary.py | 2 -- ...est_view_change_2_of_4_nodes_with_non_primary.py | 2 -- ..._change_2_of_4_nodes_with_old_and_new_primary.py | 2 -- ...est_view_change_2_of_4_nodes_with_old_primary.py | 2 -- .../slow_nodes/test_view_change_all_nodes.py | 2 -- .../test_view_change_all_nodes_random_delay.py | 2 -- .../slow_nodes/test_view_change_complex.py | 2 -- .../test_view_change_gc_all_nodes_random_delay.py | 2 -- ...ode_join_after_view_change_by_primary_restart.py | 4 ---- ...diconnected_node_reconnects_after_view_change.py | 1 - .../test_last_ordered_reset_for_new_view.py | 1 - .../test_master_primary_different_from_previous.py | 1 - .../test_new_node_joins_after_view_change.py | 10 +++------- .../test_no_instance_change_before_node_is_ready.py | 4 ---- ..._node_detecting_lag_from_view_change_messages.py | 2 -- .../view_change/test_pp_seq_no_starts_from_1.py | 4 +--- plenum/test/view_change/test_reverted_unordered.py | 1 - ...n_ledger_the_same_after_restart_for_all_nodes.py | 1 - ...fter_back_to_quorum_with_disconnected_primary.py | 1 - .../view_change/test_view_change_done_delayed.py | 1 - .../test_view_change_max_catchup_rounds.py | 1 - .../test_view_change_n_minus_f_quorum.py | 3 --- .../test_view_change_without_any_reqs.py | 1 - ...t_view_changes_if_master_primary_disconnected.py | 1 - plenum/test/view_change/test_view_not_changed.py | 2 -- ...en_primary_disconnected_from_less_than_quorum.py | 1 - ...est_view_not_changed_when_short_disconnection.py | 1 - .../test/zstack_tests/test_zstack_reconnection.py | 2 -- 77 files changed, 9 insertions(+), 151 deletions(-) delete mode 100644 plenum/test/node_request/test_timestamp/conftest.py diff --git a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py index 65a57edfee..719e93536f 100644 --- a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py +++ b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py @@ -11,7 +11,6 @@ checkNodeRemotes from plenum.test.test_stack import CONNECTED, JOINED_NOT_ALLOWED from plenum.test import waits -from plenum.test.pool_transactions.conftest import looper logger = getlogger() diff --git a/plenum/test/instances/test_msgs_from_slow_instances.py b/plenum/test/instances/test_msgs_from_slow_instances.py index 533a90bf12..d390c3d2e7 100644 --- a/plenum/test/instances/test_msgs_from_slow_instances.py +++ b/plenum/test/instances/test_msgs_from_slow_instances.py @@ -6,7 +6,6 @@ from plenum.test.delayers import delayerMsgTuple from plenum.test.test_node import TestNode from plenum.test import waits -from plenum.test.pool_transactions.conftest import looper from plenum.test.node_request.conftest import committed1, \ prepared1, preprepared1, propagated1, reqAcked1, \ sent1, noRetryReq, faultyNodes diff --git a/plenum/test/instances/test_multiple_commit.py b/plenum/test/instances/test_multiple_commit.py index d36c476701..feac6ea9e2 100644 --- a/plenum/test/instances/test_multiple_commit.py +++ b/plenum/test/instances/test_multiple_commit.py @@ -12,7 +12,6 @@ sendDuplicate3PhaseMsg from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.pool_transactions.conftest import looper from plenum.test.node_request.conftest import committed1, \ prepared1, preprepared1, propagated1, reqAcked1, \ sent1, noRetryReq, faultyNodes diff --git a/plenum/test/instances/test_multiple_pre_prepare.py b/plenum/test/instances/test_multiple_pre_prepare.py index e3e80b7c43..b326c054ce 100644 --- a/plenum/test/instances/test_multiple_pre_prepare.py +++ b/plenum/test/instances/test_multiple_pre_prepare.py @@ -12,7 +12,6 @@ sendDuplicate3PhaseMsg from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.pool_transactions.conftest import looper from plenum.test.node_request.conftest import committed1, \ prepared1, preprepared1, propagated1, reqAcked1, \ sent1, noRetryReq, faultyNodes diff --git a/plenum/test/instances/test_multiple_prepare.py b/plenum/test/instances/test_multiple_prepare.py index 2ef335035e..ae34618a77 100644 --- a/plenum/test/instances/test_multiple_prepare.py +++ b/plenum/test/instances/test_multiple_prepare.py @@ -11,7 +11,6 @@ sendDuplicate3PhaseMsg from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.pool_transactions.conftest import looper from plenum.test.node_request.conftest import committed1, \ prepared1, preprepared1, propagated1, reqAcked1, \ sent1, noRetryReq, faultyNodes diff --git a/plenum/test/instances/test_pre_prepare_digest.py b/plenum/test/instances/test_pre_prepare_digest.py index 6c698d0bab..cc528504b7 100644 --- a/plenum/test/instances/test_pre_prepare_digest.py +++ b/plenum/test/instances/test_pre_prepare_digest.py @@ -12,7 +12,6 @@ send3PhaseMsgWithIncorrectDigest from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.pool_transactions.conftest import looper from plenum.test.node_request.conftest import committed1, \ prepared1, preprepared1, propagated1, reqAcked1, \ sent1, noRetryReq, faultyNodes diff --git a/plenum/test/logging/test_logging_txn_state.py b/plenum/test/logging/test_logging_txn_state.py index b28a02c2ad..c86cc7a8ea 100644 --- a/plenum/test/logging/test_logging_txn_state.py +++ b/plenum/test/logging/test_logging_txn_state.py @@ -9,7 +9,6 @@ from plenum.common.constants import DOMAIN_LEDGER_ID, STEWARD_STRING -from plenum.test.pool_transactions.conftest import looper from plenum.test.pool_transactions.helper import prepare_nym_request, \ sdk_sign_and_send_prepared_request from plenum.test import waits diff --git a/plenum/test/monitoring/test_avg_latency.py b/plenum/test/monitoring/test_avg_latency.py index b17137822d..2a8177cb51 100644 --- a/plenum/test/monitoring/test_avg_latency.py +++ b/plenum/test/monitoring/test_avg_latency.py @@ -1,6 +1,5 @@ from stp_core.common.log import getlogger from plenum.test.helper import sdk_send_random_and_check -from plenum.test.pool_transactions.conftest import looper nodeCount = 4 logger = getlogger() diff --git a/plenum/test/monitoring/test_instance_change_with_req_Lambda.py b/plenum/test/monitoring/test_instance_change_with_req_Lambda.py index dced47af86..dcbd3edd9b 100644 --- a/plenum/test/monitoring/test_instance_change_with_req_Lambda.py +++ b/plenum/test/monitoring/test_instance_change_with_req_Lambda.py @@ -6,7 +6,6 @@ from plenum.test.test_node import getPrimaryReplica from plenum.test.spy_helpers import getAllReturnVals from plenum.test.helper import sdk_send_random_and_check -from plenum.test.pool_transactions.conftest import looper nodeCount = 7 whitelist = ["discarding message"] diff --git a/plenum/test/monitoring/test_monitoring_params_with_zfn.py b/plenum/test/monitoring/test_monitoring_params_with_zfn.py index 678b8c5b9c..4c3c5fd128 100644 --- a/plenum/test/monitoring/test_monitoring_params_with_zfn.py +++ b/plenum/test/monitoring/test_monitoring_params_with_zfn.py @@ -1,7 +1,5 @@ import pytest -from plenum.test.pool_transactions.conftest import looper - nodeCount = 7 diff --git a/plenum/test/monitoring/test_no_check_if_no_new_requests.py b/plenum/test/monitoring/test_no_check_if_no_new_requests.py index 54755d4f53..9cf7e2f188 100644 --- a/plenum/test/monitoring/test_no_check_if_no_new_requests.py +++ b/plenum/test/monitoring/test_no_check_if_no_new_requests.py @@ -1,4 +1,3 @@ -from plenum.test.pool_transactions.conftest import looper from plenum.test.view_change.conftest import perf_chk_patched from plenum.test.helper import sdk_send_random_and_check diff --git a/plenum/test/monitoring/test_post_monitoring_stats.py b/plenum/test/monitoring/test_post_monitoring_stats.py index 5d33b84b23..d6fc96d167 100644 --- a/plenum/test/monitoring/test_post_monitoring_stats.py +++ b/plenum/test/monitoring/test_post_monitoring_stats.py @@ -1,7 +1,5 @@ -from plenum.common.config_util import getConfig from stp_core.loop.eventually import eventually from plenum.server.monitor import Monitor -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_random_and_check diff --git a/plenum/test/monitoring/test_throughput.py b/plenum/test/monitoring/test_throughput.py index f2f9005601..f89bc5e12f 100644 --- a/plenum/test/monitoring/test_throughput.py +++ b/plenum/test/monitoring/test_throughput.py @@ -1,6 +1,5 @@ from typing import Iterable from stp_core.common.log import getlogger -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_random_and_check nodeCount = 4 diff --git a/plenum/test/node_catchup/conftest.py b/plenum/test/node_catchup/conftest.py index 3ca7b18a07..643e66e24a 100644 --- a/plenum/test/node_catchup/conftest.py +++ b/plenum/test/node_catchup/conftest.py @@ -9,10 +9,6 @@ check_last_3pc_master from plenum.test.pool_transactions.helper import \ addNewStewardAndNode, buildPoolClientAndWallet -# noinspection PyUnresolvedReferences -from plenum.test.pool_transactions.conftest import stewardAndWallet1, \ - steward1, stewardWallet, clientAndWallet1, client1, wallet1, \ - client1Connected from plenum.test.test_client import TestClient from plenum.test.test_node import checkNodesConnected diff --git a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py index 1e319f2bda..8d02589089 100644 --- a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py +++ b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py @@ -1,8 +1,6 @@ from plenum.common.constants import DOMAIN_LEDGER_ID from plenum.common.messages.node_messages import Commit, ConsistencyProof from plenum.test.delayers import cpDelay, cDelay -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper from plenum.test.helper import send_reqs_batches_and_get_suff_replies from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data, \ diff --git a/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py b/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py index 76734adf2c..7d68fe9efb 100644 --- a/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py +++ b/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py @@ -11,9 +11,6 @@ disconnect_node_and_ensure_disconnected, \ reconnect_node_and_ensure_connected -# noinspection PyUnresolvedReferences -from plenum.test.pool_transactions.conftest import \ - clientAndWallet1, client1, wallet1, client1Connected, looper from stp_core.loop.eventually import eventually logger = getlogger() diff --git a/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py b/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py index 5ec0825698..679154091c 100644 --- a/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py +++ b/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py @@ -5,9 +5,6 @@ from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState, POOL_LEDGER_ID from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies -from plenum.test.pool_transactions.conftest import looper, \ - steward1, stewardWallet, stewardAndWallet1 - from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually from plenum.test.node_catchup.helper import check_ledger_state diff --git a/plenum/test/node_catchup/test_node_ledger_manager.py b/plenum/test/node_catchup/test_node_ledger_manager.py index a9e586169f..9e764ba668 100644 --- a/plenum/test/node_catchup/test_node_ledger_manager.py +++ b/plenum/test/node_catchup/test_node_ledger_manager.py @@ -1,8 +1,3 @@ -# noinspection PyUnresolvedReferences -from plenum.test.pool_transactions.conftest import \ - clientAndWallet1, client1, wallet1, client1Connected, looper - - def test_ledger_sync_order(looper, txnPoolNodeSet): for node in txnPoolNodeSet: ledger_ids = node.ledger_ids diff --git a/plenum/test/node_request/conftest.py b/plenum/test/node_request/conftest.py index 16811c539d..333adc5220 100644 --- a/plenum/test/node_request/conftest.py +++ b/plenum/test/node_request/conftest.py @@ -10,7 +10,6 @@ from plenum.test.node_request.node_request_helper import checkPrePrepared, \ checkPropagated, checkPrepared from plenum.test.node_request.node_request_helper import checkCommitted -from plenum.test.pool_transactions.conftest import looper from plenum.common.util import getNoInstances diff --git a/plenum/test/node_request/message_request/test_node_request_missing_three_phase_messages.py b/plenum/test/node_request/message_request/test_node_request_missing_three_phase_messages.py index b6d76ba112..26fcde92f1 100644 --- a/plenum/test/node_request/message_request/test_node_request_missing_three_phase_messages.py +++ b/plenum/test/node_request/message_request/test_node_request_missing_three_phase_messages.py @@ -4,7 +4,6 @@ from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected, reconnect_node_and_ensure_connected -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_random_requests, sdk_send_random_and_check logger = getlogger() diff --git a/plenum/test/node_request/message_request/test_node_requests_missing_preprepare.py b/plenum/test/node_request/message_request/test_node_requests_missing_preprepare.py index 2c9fa11ce1..7a8c9c5917 100644 --- a/plenum/test/node_request/message_request/test_node_requests_missing_preprepare.py +++ b/plenum/test/node_request/message_request/test_node_requests_missing_preprepare.py @@ -12,7 +12,6 @@ from plenum.test.node_request.message_request.helper import split_nodes from plenum.test.spy_helpers import get_count from plenum.test.test_node import getNonPrimaryReplicas, get_master_primary_node -from plenum.test.pool_transactions.conftest import looper whitelist = ['does not have expected state'] diff --git a/plenum/test/node_request/message_request/test_node_requests_missing_three_phase_messages_after_long_disconnection.py b/plenum/test/node_request/message_request/test_node_requests_missing_three_phase_messages_after_long_disconnection.py index 51909bfc4f..6086170028 100644 --- a/plenum/test/node_request/message_request/test_node_requests_missing_three_phase_messages_after_long_disconnection.py +++ b/plenum/test/node_request/message_request/test_node_requests_missing_three_phase_messages_after_long_disconnection.py @@ -7,7 +7,6 @@ from plenum.test.waits import expectedPoolGetReadyTimeout from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_random_requests, sdk_send_random_and_check logger = getlogger() diff --git a/plenum/test/node_request/message_request/test_preprepare_request.py b/plenum/test/node_request/message_request/test_preprepare_request.py index 18cab89768..f4036356fd 100644 --- a/plenum/test/node_request/message_request/test_preprepare_request.py +++ b/plenum/test/node_request/message_request/test_preprepare_request.py @@ -7,7 +7,6 @@ from plenum.test.spy_helpers import getAllReturnVals, get_count from stp_core.loop.eventually import eventually -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_batches_of_random_and_check diff --git a/plenum/test/node_request/message_request/test_requested_preprepare_handling.py b/plenum/test/node_request/message_request/test_requested_preprepare_handling.py index 45b9debdee..aa7f8b60ec 100644 --- a/plenum/test/node_request/message_request/test_requested_preprepare_handling.py +++ b/plenum/test/node_request/message_request/test_requested_preprepare_handling.py @@ -9,7 +9,6 @@ from plenum.test.spy_helpers import get_count from stp_core.loop.eventually import eventually -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_batches_of_random_and_check diff --git a/plenum/test/node_request/test_belated_request_not_processed.py b/plenum/test/node_request/test_belated_request_not_processed.py index 1dbb51ff5d..cd2a9430c0 100644 --- a/plenum/test/node_request/test_belated_request_not_processed.py +++ b/plenum/test/node_request/test_belated_request_not_processed.py @@ -2,7 +2,6 @@ from plenum.test.delayers import cDelay, req_delay, ppgDelay from plenum.test.helper import sdk_signed_random_requests, \ sdk_send_signed_requests, sdk_send_and_check -from plenum.test.pool_transactions.conftest import looper from plenum.test.test_node import ensureElectionsDone from plenum.test.view_change.helper import ensure_view_change diff --git a/plenum/test/node_request/test_commit/test_commits_recvd_first.py b/plenum/test/node_request/test_commit/test_commits_recvd_first.py index 25aa353154..c1d3655efb 100644 --- a/plenum/test/node_request/test_commit/test_commits_recvd_first.py +++ b/plenum/test/node_request/test_commit/test_commits_recvd_first.py @@ -2,7 +2,6 @@ from plenum.test.delayers import ppDelay, pDelay from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.test_node import getNonPrimaryReplicas -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_batches_of_random_and_check diff --git a/plenum/test/node_request/test_commit/test_commits_without_prepares.py b/plenum/test/node_request/test_commit/test_commits_without_prepares.py index e4cbdaa3fb..1472adb4ff 100644 --- a/plenum/test/node_request/test_commit/test_commits_without_prepares.py +++ b/plenum/test/node_request/test_commit/test_commits_without_prepares.py @@ -1,7 +1,6 @@ from plenum.test.delayers import pDelay from plenum.test.test_node import get_master_primary_node -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_random_and_check diff --git a/plenum/test/node_request/test_order/test_ordering_when_pre_prepare_not_received.py b/plenum/test/node_request/test_order/test_ordering_when_pre_prepare_not_received.py index d364a54df7..e5cc96ad49 100644 --- a/plenum/test/node_request/test_order/test_ordering_when_pre_prepare_not_received.py +++ b/plenum/test/node_request/test_order/test_ordering_when_pre_prepare_not_received.py @@ -6,7 +6,6 @@ from plenum.test.delayers import ppDelay, pDelay from plenum.test.helper import sdk_send_random_request from plenum.test.test_node import getNonPrimaryReplicas -from plenum.test.pool_transactions.conftest import looper def testOrderingWhenPrePrepareNotReceived(looper, txnPoolNodeSet, diff --git a/plenum/test/node_request/test_order/test_request_ordering_1.py b/plenum/test/node_request/test_order/test_request_ordering_1.py index 4481eaa064..3654e7e153 100644 --- a/plenum/test/node_request/test_order/test_request_ordering_1.py +++ b/plenum/test/node_request/test_order/test_request_ordering_1.py @@ -4,7 +4,6 @@ from plenum.test.helper import sdk_send_random_request from plenum.test.malicious_behaviors_node import delaysPrePrepareProcessing from plenum.test.test_node import getNonPrimaryReplicas -from plenum.test.pool_transactions.conftest import looper def testOrderingCase1(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): diff --git a/plenum/test/node_request/test_pre_prepare/test_ignore_pre_prepare_pp_seq_no_less_than_expected.py b/plenum/test/node_request/test_pre_prepare/test_ignore_pre_prepare_pp_seq_no_less_than_expected.py index 1b4353ca83..8a7da085fc 100644 --- a/plenum/test/node_request/test_pre_prepare/test_ignore_pre_prepare_pp_seq_no_less_than_expected.py +++ b/plenum/test/node_request/test_pre_prepare/test_ignore_pre_prepare_pp_seq_no_less_than_expected.py @@ -1,9 +1,7 @@ import pytest -from stp_core.common.util import adict from plenum.test.helper import sdk_send_random_and_check from plenum.test.test_node import getNonPrimaryReplicas -from plenum.test.pool_transactions.conftest import looper def test_ignore_pre_prepare_pp_seq_no_less_than_expected(looper, diff --git a/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py b/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py index 7453182566..1e9f4263b0 100644 --- a/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py +++ b/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py @@ -3,7 +3,6 @@ from plenum.test.spy_helpers import get_count, getAllReturnVals from plenum.test.test_node import getNonPrimaryReplicas from plenum.test.helper import sdk_send_random_and_check -from plenum.test.pool_transactions.conftest import looper from plenum.test.node_request.helper import sdk_ensure_pool_functional diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py index 60908d2590..18c0bad7dc 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py @@ -2,8 +2,6 @@ from plenum.test.helper import checkViewNoForNodes, sendRandomRequest, \ waitForSufficientRepliesForRequests, \ verify_request_not_replied_and_not_ordered -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.test_node import ensureElectionsDone, getRequiredInstances diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py index ced0a76f87..3f9fa275a4 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py @@ -2,8 +2,6 @@ from plenum.test.helper import waitForViewChange, checkViewNoForNodes, \ sendRandomRequest, waitForSufficientRepliesForRequests, \ verify_request_not_replied_and_not_ordered -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.test_node import ensureElectionsDone, getRequiredInstances diff --git a/plenum/test/node_request/test_request_forwarding.py b/plenum/test/node_request/test_request_forwarding.py index 5c17bb9166..cc6a00b1d1 100644 --- a/plenum/test/node_request/test_request_forwarding.py +++ b/plenum/test/node_request/test_request_forwarding.py @@ -2,7 +2,6 @@ from plenum.test import waits from plenum.test.delayers import nom_delay, delay_3pc_messages from plenum.test.batching_3pc.conftest import tconf -from plenum.test.pool_transactions.conftest import looper from plenum.test.test_node import ensureElectionsDone from plenum.test.view_change.helper import ensure_view_change from stp_core.loop.eventually import eventually diff --git a/plenum/test/node_request/test_timestamp/conftest.py b/plenum/test/node_request/test_timestamp/conftest.py deleted file mode 100644 index 38dac4e336..0000000000 --- a/plenum/test/node_request/test_timestamp/conftest.py +++ /dev/null @@ -1,3 +0,0 @@ -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper, nodeThetaAdded, \ - stewardAndWallet1, steward1, stewardWallet diff --git a/plenum/test/observer/conftest.py b/plenum/test/observer/conftest.py index bed4b19377..a0bd752df6 100644 --- a/plenum/test/observer/conftest.py +++ b/plenum/test/observer/conftest.py @@ -10,8 +10,6 @@ from plenum.test.test_node import TestNode from plenum.test.testable import spyable -from plenum.test.pool_transactions.conftest import looper - @spyable(methods=[Observable.append_input, Observable.send_to_observers, diff --git a/plenum/test/plugin/demo_plugin/conftest.py b/plenum/test/plugin/demo_plugin/conftest.py index 03c4e62f5d..a50c33f617 100644 --- a/plenum/test/plugin/demo_plugin/conftest.py +++ b/plenum/test/plugin/demo_plugin/conftest.py @@ -7,10 +7,6 @@ from plenum.common.pkg_util import update_module_vars from plenum.test.plugin.demo_plugin.main import integrate_plugin_in_node -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper, stewardAndWallet1, \ - stewardWallet, steward1 - def do_plugin_initialisation_for_tests(): # The next imports and reloading are needed only in tests, since in diff --git a/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py b/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py index 35d461b86a..1edd3cfe81 100644 --- a/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py +++ b/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py @@ -7,7 +7,6 @@ from plenum.test.test_node import TestNode, checkNodesConnected from stp_core.network.port_dispenser import genHa from plenum.common.config_helper import PNodeConfigHelper -from plenum.test.pool_transactions.conftest import looper logger = getlogger() diff --git a/plenum/test/primary_selection/conftest.py b/plenum/test/primary_selection/conftest.py index 158f9ab9ef..4a7e6daea7 100644 --- a/plenum/test/primary_selection/conftest.py +++ b/plenum/test/primary_selection/conftest.py @@ -3,9 +3,6 @@ from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.primary_selection.helper import check_newly_added_nodes, \ getPrimaryNodesIdxs -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper, nodeThetaAdded, \ - stewardAndWallet1, steward1, stewardWallet from plenum.test.pool_transactions.helper import buildPoolClientAndWallet diff --git a/plenum/test/primary_selection/test_catchup_after_view_change.py b/plenum/test/primary_selection/test_catchup_after_view_change.py index 52405a1644..33c762f608 100644 --- a/plenum/test/primary_selection/test_catchup_after_view_change.py +++ b/plenum/test/primary_selection/test_catchup_after_view_change.py @@ -10,10 +10,6 @@ checkProtocolInstanceSetup, TestReplica from plenum.test.view_change.helper import ensure_view_change from stp_core.loop.eventually import eventually - -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper, nodeThetaAdded, \ - stewardAndWallet1, steward1, stewardWallet from plenum.test.batching_3pc.conftest import tconf Max3PCBatchSize = 4 diff --git a/plenum/test/primary_selection/test_catchup_multiple_rounds.py b/plenum/test/primary_selection/test_catchup_multiple_rounds.py index 1bd9caaf0d..14c16a7746 100644 --- a/plenum/test/primary_selection/test_catchup_multiple_rounds.py +++ b/plenum/test/primary_selection/test_catchup_multiple_rounds.py @@ -5,9 +5,6 @@ from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \ sendRandomRequests, waitForSufficientRepliesForRequests, checkViewNoForNodes from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper, nodeThetaAdded, \ - stewardAndWallet1, steward1, stewardWallet from plenum.test.batching_3pc.conftest import tconf from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica, \ diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py index cac930b4c7..3a9db246ef 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py @@ -1,7 +1,6 @@ from stp_core.common.log import getlogger from plenum.common.constants import ALIAS, SERVICES -from plenum.test.pool_transactions.conftest import looper from plenum.test.pool_transactions.helper import updateNodeData from plenum.test.test_node import TestNode, checkNodesConnected, \ diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_promotion.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_promotion.py index 1f6a547e3b..58af8b3b61 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_promotion.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_promotion.py @@ -1,5 +1,4 @@ from plenum.common.constants import ALIAS, SERVICES, VALIDATOR -from plenum.test.pool_transactions.conftest import looper from plenum.test.pool_transactions.helper import updateNodeData from plenum.test.test_node import ensureElectionsDone diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py index 03967bf846..f846de58e1 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py @@ -2,7 +2,6 @@ from plenum.common.constants import ALIAS, SERVICES -from plenum.test.pool_transactions.conftest import looper from plenum.test.pool_transactions.helper import updateNodeData from plenum.test.helper import checkViewNoForNodes, \ diff --git a/plenum/test/primary_selection/test_recover_more_than_f_failure.py b/plenum/test/primary_selection/test_recover_more_than_f_failure.py index 8672065e36..a502723314 100644 --- a/plenum/test/primary_selection/test_recover_more_than_f_failure.py +++ b/plenum/test/primary_selection/test_recover_more_than_f_failure.py @@ -7,15 +7,10 @@ sendReqsToNodesAndVerifySuffReplies from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.pool_transactions.helper import \ - disconnect_node_and_ensure_disconnected, \ - reconnect_node_and_ensure_connected -from plenum.test.test_node import ensureElectionsDone, ensure_node_disconnected -from plenum.test.view_change.helper import ensure_view_change + disconnect_node_and_ensure_disconnected +from plenum.test.test_node import ensureElectionsDone from plenum.test.view_change.helper import start_stopped_node -# Do not remove these imports -from plenum.test.pool_transactions.conftest import client1, wallet1, client1Connected, looper - logger = getlogger() diff --git a/plenum/test/primary_selection/test_recover_primary_no_view_change.py b/plenum/test/primary_selection/test_recover_primary_no_view_change.py index 53a90f0440..e55f08c730 100644 --- a/plenum/test/primary_selection/test_recover_primary_no_view_change.py +++ b/plenum/test/primary_selection/test_recover_primary_no_view_change.py @@ -3,21 +3,14 @@ from stp_core.common.log import getlogger from plenum.test.conftest import getValueFromModule -from plenum.test.helper import stopNodes, waitForViewChange, \ +from plenum.test.helper import waitForViewChange, \ sendReqsToNodesAndVerifySuffReplies from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.pool_transactions.helper import \ - disconnect_node_and_ensure_disconnected, \ - reconnect_node_and_ensure_connected -from plenum.test.test_node import ensureElectionsDone, ensure_node_disconnected -from plenum.test.view_change.helper import ensure_view_change +from plenum.test.test_node import ensureElectionsDone from plenum.test.view_change.helper import start_stopped_node from plenum.test.primary_selection.test_recover_more_than_f_failure import \ - stop_primary, checkpoint_size, primary_replicas_iter, nodes_have_checkpoints, nodes_do_not_have_checkpoints - -# Do not remove these imports -from plenum.test.pool_transactions.conftest import client1, wallet1, client1Connected, looper + stop_primary, nodes_have_checkpoints, nodes_do_not_have_checkpoints logger = getlogger() diff --git a/plenum/test/primary_selection/test_selection_f_plus_one_quorum.py b/plenum/test/primary_selection/test_selection_f_plus_one_quorum.py index 3e98d4c467..abeb361cd9 100644 --- a/plenum/test/primary_selection/test_selection_f_plus_one_quorum.py +++ b/plenum/test/primary_selection/test_selection_f_plus_one_quorum.py @@ -6,9 +6,6 @@ from plenum.test.view_change.helper import ensure_view_change from plenum.test.view_change.helper import start_stopped_node -# Do not remove these imports -from plenum.test.pool_transactions.conftest import client1, wallet1, client1Connected, looper - def test_selection_f_plus_one_quorum(looper, txnPoolNodeSet, allPluginsPath, tdir, tconf, sdk_pool_handle, sdk_wallet_client): diff --git a/plenum/test/sdk/test_sdk_bindings.py b/plenum/test/sdk/test_sdk_bindings.py index a0af950371..872e89ce2a 100644 --- a/plenum/test/sdk/test_sdk_bindings.py +++ b/plenum/test/sdk/test_sdk_bindings.py @@ -1,5 +1,4 @@ import pytest -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import sdk_send_random_request, \ sdk_send_random_requests, sdk_get_and_check_replies, sdk_send_random_and_check from plenum.test.pool_transactions.helper import sdk_pool_refresh diff --git a/plenum/test/validator_info/test_validator_info.py b/plenum/test/validator_info/test_validator_info.py index a22f939b16..23fb486b67 100644 --- a/plenum/test/validator_info/test_validator_info.py +++ b/plenum/test/validator_info/test_validator_info.py @@ -16,7 +16,6 @@ sendRandomRequest, check_sufficient_replies_received # noinspection PyUnresolvedReferences from plenum.test.node_catchup.helper import ensureClientConnectedToNodesAndPoolLedgerSame -from plenum.test.pool_transactions.conftest import steward1, stewardWallet, client1Connected # noqa from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected from plenum.test.test_client import genTestClient from stp_core.common.constants import ZMQ_NETWORK_PROTOCOL diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py index 395648cb98..243e3a8e07 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py @@ -1,7 +1,5 @@ from plenum.test.test_node import get_last_master_non_primary_node, get_first_master_non_primary_node from plenum.test.view_change.helper import view_change_in_between_3pc -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper def slow_nodes(node_set): diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py index 8241dcc43a..d3f057ba2f 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py @@ -1,7 +1,5 @@ from plenum.test.test_node import getNonPrimaryReplicas from plenum.test.view_change.helper import view_change_in_between_3pc -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper def slow_nodes(node_set): diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py index f802a78201..e4d735bd88 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py @@ -1,7 +1,5 @@ from plenum.test.test_node import get_master_primary_node, get_first_master_non_primary_node from plenum.test.view_change.helper import view_change_in_between_3pc -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper def slow_nodes(node_set): diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py index 7f3e5e2ef0..dd267dc28c 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py @@ -1,7 +1,5 @@ from plenum.test.test_node import get_master_primary_node, get_last_master_non_primary_node from plenum.test.view_change.helper import view_change_in_between_3pc -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper def slow_nodes(node_set): diff --git a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py index 91ec9bbb07..9e791e56cc 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py @@ -1,6 +1,4 @@ from plenum.test.view_change.helper import view_change_in_between_3pc -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper def test_view_change_in_between_3pc_all_nodes(txnPoolNodeSet, looper, diff --git a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py index f6edb486c0..1ac13aee44 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py @@ -1,6 +1,4 @@ from plenum.test.view_change.helper import view_change_in_between_3pc_random_delays -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper TestRunningTimeLimitSec = 300 diff --git a/plenum/test/view_change/slow_nodes/test_view_change_complex.py b/plenum/test/view_change/slow_nodes/test_view_change_complex.py index c04a497a47..cb052e90ca 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_complex.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_complex.py @@ -1,7 +1,5 @@ from plenum.test.view_change.helper import \ view_change_in_between_3pc_random_delays -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper TestRunningTimeLimitSec = 300 diff --git a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py index 52f6e92d66..7638856adf 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py @@ -11,8 +11,6 @@ from plenum.test.delayers import delay_3pc_messages, \ reset_delays_and_process_delayeds from plenum.test.view_change.helper import ensure_view_change_complete -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper def check_nodes_last_ordered_3pc(nodes, last_ordered_3pc): diff --git a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py index b0aebe2d30..d81167e21e 100644 --- a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py +++ b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py @@ -6,10 +6,6 @@ from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState, POOL_LEDGER_ID from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies -from plenum.test.pool_transactions.conftest import wallet1, client1, \ - client1Connected, looper, stewardAndWallet1, steward1, \ - stewardWallet - from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually from plenum.test.node_catchup.helper import check_ledger_state, \ diff --git a/plenum/test/view_change/test_diconnected_node_reconnects_after_view_change.py b/plenum/test/view_change/test_diconnected_node_reconnects_after_view_change.py index 7e3c8929aa..bff25f3762 100644 --- a/plenum/test/view_change/test_diconnected_node_reconnects_after_view_change.py +++ b/plenum/test/view_change/test_diconnected_node_reconnects_after_view_change.py @@ -2,7 +2,6 @@ from plenum.test.helper import checkViewNoForNodes, waitForViewChange, \ sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.pool_transactions.conftest import looper from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected, \ reconnect_node_and_ensure_connected diff --git a/plenum/test/view_change/test_last_ordered_reset_for_new_view.py b/plenum/test/view_change/test_last_ordered_reset_for_new_view.py index d553629bac..c22de7d38a 100644 --- a/plenum/test/view_change/test_last_ordered_reset_for_new_view.py +++ b/plenum/test/view_change/test_last_ordered_reset_for_new_view.py @@ -1,6 +1,5 @@ from plenum.test.helper import sdk_send_random_and_check, checkViewNoForNodes from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.pool_transactions.conftest import looper from plenum.test.view_change.helper import ensure_view_change_complete diff --git a/plenum/test/view_change/test_master_primary_different_from_previous.py b/plenum/test/view_change/test_master_primary_different_from_previous.py index 1ed7b96a8c..210308214a 100644 --- a/plenum/test/view_change/test_master_primary_different_from_previous.py +++ b/plenum/test/view_change/test_master_primary_different_from_previous.py @@ -6,7 +6,6 @@ sdk_send_random_and_check, countDiscarded from plenum.test.malicious_behaviors_node import slow_primary from plenum.test.test_node import getPrimaryReplica, ensureElectionsDone -from plenum.test.pool_transactions.conftest import looper from plenum.test.view_change.helper import provoke_and_wait_for_view_change, ensure_view_change from stp_core.common.log import getlogger diff --git a/plenum/test/view_change/test_new_node_joins_after_view_change.py b/plenum/test/view_change/test_new_node_joins_after_view_change.py index e3bcedf125..e42be2a5f1 100644 --- a/plenum/test/view_change/test_new_node_joins_after_view_change.py +++ b/plenum/test/view_change/test_new_node_joins_after_view_change.py @@ -11,11 +11,8 @@ from plenum.test.helper import send_reqs_to_nodes_and_verify_all_replies, \ checkViewNoForNodes, stopNodes, sendReqsToNodesAndVerifySuffReplies -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper, stewardAndWallet1, steward1, \ - stewardWallet -from plenum.test.primary_selection.conftest import nodeThetaAdded, \ - one_node_added +from plenum.test.pool_transactions.conftest import nodeThetaAdded +from plenum.test.primary_selection.conftest import one_node_added from stp_core.common.log import getlogger @@ -29,8 +26,7 @@ def all_nodes_view_change( stewardWallet, steward1, client1, - wallet1, - client1Connected): + wallet1): for _ in range(5): send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 2) ensure_view_change(looper, txnPoolNodeSet) diff --git a/plenum/test/view_change/test_no_instance_change_before_node_is_ready.py b/plenum/test/view_change/test_no_instance_change_before_node_is_ready.py index ac7818b418..b532f6917f 100644 --- a/plenum/test/view_change/test_no_instance_change_before_node_is_ready.py +++ b/plenum/test/view_change/test_no_instance_change_before_node_is_ready.py @@ -2,10 +2,6 @@ from plenum.server.view_change.view_changer import ViewChanger -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper, stewardAndWallet1, steward1, \ - stewardWallet - from stp_core.common.log import getlogger from plenum.test.pool_transactions.helper import start_not_added_node, add_started_node diff --git a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py index b3c62831a9..9e73ef1fe6 100644 --- a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py +++ b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py @@ -11,8 +11,6 @@ from plenum.test.test_node import getNonPrimaryReplicas from plenum.test.view_change.helper import ensure_view_change from stp_core.loop.eventually import eventually -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper TestRunningTimeLimitSec = 150 diff --git a/plenum/test/view_change/test_pp_seq_no_starts_from_1.py b/plenum/test/view_change/test_pp_seq_no_starts_from_1.py index e7810130fe..1ec3de0255 100644 --- a/plenum/test/view_change/test_pp_seq_no_starts_from_1.py +++ b/plenum/test/view_change/test_pp_seq_no_starts_from_1.py @@ -1,7 +1,5 @@ import pytest -from plenum.test.helper import checkViewNoForNodes, \ - sendReqsToNodesAndVerifySuffReplies -from plenum.test.pool_transactions.conftest import looper +from plenum.test.helper import checkViewNoForNodes from plenum.test.view_change.helper import ensure_view_change from plenum.test.helper import sdk_send_random_and_check diff --git a/plenum/test/view_change/test_reverted_unordered.py b/plenum/test/view_change/test_reverted_unordered.py index dc303dffd4..6702afdaee 100644 --- a/plenum/test/view_change/test_reverted_unordered.py +++ b/plenum/test/view_change/test_reverted_unordered.py @@ -7,7 +7,6 @@ from plenum.test.delayers import cDelay, msg_rep_delay, lsDelay from plenum.test.helper import sdk_send_batches_of_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.pool_transactions.conftest import looper from plenum.test.test_node import getNonPrimaryReplicas, ensureElectionsDone from plenum.test.view_change.helper import ensure_view_change diff --git a/plenum/test/view_change/test_that_domain_ledger_the_same_after_restart_for_all_nodes.py b/plenum/test/view_change/test_that_domain_ledger_the_same_after_restart_for_all_nodes.py index 2f5eebe211..517eec119b 100644 --- a/plenum/test/view_change/test_that_domain_ledger_the_same_after_restart_for_all_nodes.py +++ b/plenum/test/view_change/test_that_domain_ledger_the_same_after_restart_for_all_nodes.py @@ -1,7 +1,6 @@ import pytest from plenum.test.view_change.helper import ensure_view_change_by_primary_restart -from plenum.test.pool_transactions.conftest import looper from stp_core.common.log import getlogger from plenum.common.startable import Mode diff --git a/plenum/test/view_change/test_view_change_after_back_to_quorum_with_disconnected_primary.py b/plenum/test/view_change/test_view_change_after_back_to_quorum_with_disconnected_primary.py index bc30e72121..9cef7de8cc 100644 --- a/plenum/test/view_change/test_view_change_after_back_to_quorum_with_disconnected_primary.py +++ b/plenum/test/view_change/test_view_change_after_back_to_quorum_with_disconnected_primary.py @@ -3,7 +3,6 @@ from plenum.test.view_change.helper import start_stopped_node, ensure_view_change_by_primary_restart from plenum.test.test_node import get_master_primary_node -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import checkViewNoForNodes, waitForViewChange, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data diff --git a/plenum/test/view_change/test_view_change_done_delayed.py b/plenum/test/view_change/test_view_change_done_delayed.py index f189ea3399..57927e27e6 100644 --- a/plenum/test/view_change/test_view_change_done_delayed.py +++ b/plenum/test/view_change/test_view_change_done_delayed.py @@ -2,7 +2,6 @@ from plenum.test.helper import sdk_send_batches_of_random_and_check, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ ensure_all_nodes_have_same_data -from plenum.test.pool_transactions.conftest import looper from plenum.test.test_node import getNonPrimaryReplicas from plenum.test.view_change.helper import ensure_view_change from stp_core.loop.eventually import eventually diff --git a/plenum/test/view_change/test_view_change_max_catchup_rounds.py b/plenum/test/view_change/test_view_change_max_catchup_rounds.py index 650e355b70..0c73da4729 100644 --- a/plenum/test/view_change/test_view_change_max_catchup_rounds.py +++ b/plenum/test/view_change/test_view_change_max_catchup_rounds.py @@ -2,7 +2,6 @@ from plenum.test.delayers import pDelay, cDelay from plenum.test.helper import sdk_send_batches_of_random_and_check, sdk_send_random_requests from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.pool_transactions.conftest import looper from plenum.test.test_node import getNonPrimaryReplicas, ensureElectionsDone from plenum.test.view_change.helper import ensure_view_change diff --git a/plenum/test/view_change/test_view_change_n_minus_f_quorum.py b/plenum/test/view_change/test_view_change_n_minus_f_quorum.py index 3a854afee8..5c41b8cf0e 100644 --- a/plenum/test/view_change/test_view_change_n_minus_f_quorum.py +++ b/plenum/test/view_change/test_view_change_n_minus_f_quorum.py @@ -5,9 +5,6 @@ from plenum.test.view_change.helper import ensure_view_change from plenum.test.helper import stopNodes -from plenum.test.pool_transactions.conftest import clientAndWallet1, \ - client1, wallet1, client1Connected, looper - def test_view_change_n_minus_f_quorum(txnPoolNodeSet, looper): """ diff --git a/plenum/test/view_change/test_view_change_without_any_reqs.py b/plenum/test/view_change/test_view_change_without_any_reqs.py index b4ba0f8910..9f8db0c62e 100644 --- a/plenum/test/view_change/test_view_change_without_any_reqs.py +++ b/plenum/test/view_change/test_view_change_without_any_reqs.py @@ -11,7 +11,6 @@ Max3PCBatchSize = 3 from plenum.test.batching_3pc.conftest import tconf -from plenum.test.pool_transactions.conftest import looper TestRunningTimeLimitSec = 200 diff --git a/plenum/test/view_change/test_view_changes_if_master_primary_disconnected.py b/plenum/test/view_change/test_view_changes_if_master_primary_disconnected.py index 27ac2bef3d..400761df26 100644 --- a/plenum/test/view_change/test_view_changes_if_master_primary_disconnected.py +++ b/plenum/test/view_change/test_view_changes_if_master_primary_disconnected.py @@ -6,7 +6,6 @@ from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.test_node import get_master_primary_node, ensure_node_disconnected -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check, waitForViewChange diff --git a/plenum/test/view_change/test_view_not_changed.py b/plenum/test/view_change/test_view_not_changed.py index 1db41ebd2b..6a0564f0ae 100644 --- a/plenum/test/view_change/test_view_not_changed.py +++ b/plenum/test/view_change/test_view_not_changed.py @@ -1,5 +1,3 @@ -from plenum.test.pool_transactions.conftest import looper - from plenum.common.util import getMaxFailures from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check from plenum.test.delayers import ppDelay diff --git a/plenum/test/view_change/test_view_not_changed_when_primary_disconnected_from_less_than_quorum.py b/plenum/test/view_change/test_view_not_changed_when_primary_disconnected_from_less_than_quorum.py index 85b599da9a..dad39c0859 100644 --- a/plenum/test/view_change/test_view_not_changed_when_primary_disconnected_from_less_than_quorum.py +++ b/plenum/test/view_change/test_view_not_changed_when_primary_disconnected_from_less_than_quorum.py @@ -5,7 +5,6 @@ from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.test_node import getNonPrimaryReplicas, get_master_primary_node from stp_core.loop.eventually import eventually -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check diff --git a/plenum/test/view_change/test_view_not_changed_when_short_disconnection.py b/plenum/test/view_change/test_view_not_changed_when_short_disconnection.py index b97112b7ab..48ba262724 100644 --- a/plenum/test/view_change/test_view_not_changed_when_short_disconnection.py +++ b/plenum/test/view_change/test_view_not_changed_when_short_disconnection.py @@ -1,7 +1,6 @@ import pytest from stp_core.loop.eventually import eventually -from plenum.test.pool_transactions.conftest import looper from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check from plenum.test.test_node import get_master_primary_node diff --git a/plenum/test/zstack_tests/test_zstack_reconnection.py b/plenum/test/zstack_tests/test_zstack_reconnection.py index 5361952024..1438789485 100644 --- a/plenum/test/zstack_tests/test_zstack_reconnection.py +++ b/plenum/test/zstack_tests/test_zstack_reconnection.py @@ -3,8 +3,6 @@ from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, stopNodes, \ send_reqs_to_nodes_and_verify_all_replies from plenum.test.test_node import TestNode, ensureElectionsDone From b2f6e482728afad3aad499c3cd3ed330d114cd2e Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 14 Mar 2018 09:20:02 +0300 Subject: [PATCH 11/73] little fix Signed-off-by: ArtObr --- plenum/common/messages/client_request.py | 2 +- plenum/test/bls/helper.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/plenum/common/messages/client_request.py b/plenum/common/messages/client_request.py index 8ab96c6339..75b0ad7bf2 100644 --- a/plenum/common/messages/client_request.py +++ b/plenum/common/messages/client_request.py @@ -22,7 +22,7 @@ class ClientNodeOperationData(MessageValidator): (CLIENT_PORT, NetworkPortField(optional=True)), (ALIAS, LimitedLengthStringField(max_length=ALIAS_FIELD_LIMIT)), (SERVICES, IterableField(ChooseField(values=(VALIDATOR,)), optional=True)), - (BLS_KEY, Base58Field(byte_lengths=BLS_KEY_LIMIT, optional=True)), + (BLS_KEY, Base58Field(byte_lengths=(32,), optional=True)), ) def _validate_message(self, dct): diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 7527e707f4..0209205620 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -227,7 +227,6 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, add_wrong=False, new_bls=None): new_blspk = init_bls_keys(node.keys_dir, node.name) - key_in_txn = new_bls or new_blspk \ if not add_wrong \ else b58_by_len(32) From 1d96defa54335263ced3a93fa317b24b12b7b9ee Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 14 Mar 2018 11:36:55 +0300 Subject: [PATCH 12/73] Fix for bls validation tests. Not done: update_incorrect test and state_proof test Signed-off-by: ArtObr --- plenum/common/messages/client_request.py | 2 +- plenum/test/bls/helper.py | 2 +- plenum/test/bls/test_bls_key_registry_pool_manager.py | 4 +++- plenum/test/input_validation/test_client_node_op.py | 8 +++++--- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/plenum/common/messages/client_request.py b/plenum/common/messages/client_request.py index 75b0ad7bf2..0e08667c36 100644 --- a/plenum/common/messages/client_request.py +++ b/plenum/common/messages/client_request.py @@ -22,7 +22,7 @@ class ClientNodeOperationData(MessageValidator): (CLIENT_PORT, NetworkPortField(optional=True)), (ALIAS, LimitedLengthStringField(max_length=ALIAS_FIELD_LIMIT)), (SERVICES, IterableField(ChooseField(values=(VALIDATOR,)), optional=True)), - (BLS_KEY, Base58Field(byte_lengths=(32,), optional=True)), + (BLS_KEY, Base58Field(byte_lengths=(128,), optional=True)), ) def _validate_message(self, dct): diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 0209205620..9f56d47203 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -229,7 +229,7 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, new_blspk = init_bls_keys(node.keys_dir, node.name) key_in_txn = new_bls or new_blspk \ if not add_wrong \ - else b58_by_len(32) + else base58.b58encode(randomString(128).encode()) node_dest = hexToFriendly(node.nodestack.verhex) sdk_send_update_node(looper, sdk_wallet_steward, sdk_pool_handle, diff --git a/plenum/test/bls/test_bls_key_registry_pool_manager.py b/plenum/test/bls/test_bls_key_registry_pool_manager.py index 004c673c7a..d305ccfc81 100644 --- a/plenum/test/bls/test_bls_key_registry_pool_manager.py +++ b/plenum/test/bls/test_bls_key_registry_pool_manager.py @@ -1,6 +1,8 @@ +import base58 import pytest from plenum.bls.bls_key_register_pool_manager import BlsKeyRegisterPoolManager from plenum.common.constants import NODE, TXN_TYPE, BLS_KEY, DATA, ALIAS +from plenum.common.util import randomString from plenum.test.bls.helper import sdk_change_bls_key from plenum.test.pool_transactions.helper import updateNodeData @@ -56,7 +58,7 @@ def test_get_key_for_old_root_keys_changed(bls_key_register_ledger, sdk_wallet_steward, sdk_pool_handle): old_bls_key = pool_node_txns[0][DATA][BLS_KEY] - new_bls_key = pool_node_txns[0][DATA][BLS_KEY] + "Changed" + new_bls_key = base58.b58encode(randomString(128).encode()) old_pool_root_hash = node.poolManager.state.committedHeadHash # change BLS keys diff --git a/plenum/test/input_validation/test_client_node_op.py b/plenum/test/input_validation/test_client_node_op.py index 1fbed6c7da..1b2a363aa2 100644 --- a/plenum/test/input_validation/test_client_node_op.py +++ b/plenum/test/input_validation/test_client_node_op.py @@ -1,8 +1,10 @@ +import base58 import pytest from plenum.common.constants import BLS_KEY from plenum.common.messages.client_request import ClientNodeOperationData, ALIAS, SERVICES, NODE_IP, CLIENT_IP, \ NODE_PORT, CLIENT_PORT +from plenum.common.util import randomString op = ClientNodeOperationData() @@ -54,7 +56,7 @@ def test_update_ha_passes(): def test_update_bls_sign(): op.validate({ ALIAS: 'aNode', - BLS_KEY: 'some_key', + BLS_KEY: base58.b58encode(randomString(128).encode()), }) @@ -64,5 +66,5 @@ def test_empty_bls_fails(): BLS_KEY: '', ALIAS: 'aNode' }) - ex_info.match( - 'validation error \[ClientNodeOperationData\]: empty string \(blskey=\)') + assert 'validation error [ClientNodeOperationData]: b58 decoded ' \ + 'value length 0 should be one of [128] (blskey=)' in ex_info.value.args[0] From 2761e98332e6749d9eb9fa98945c65da67b01977 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Thu, 15 Mar 2018 19:44:17 +0300 Subject: [PATCH 13/73] Only state_proof test left Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 82 ++++++++++++++++++- plenum/test/bls/test_state_proof.py | 9 +- .../test/bls/test_update_incorrect_bls_key.py | 6 +- 3 files changed, 88 insertions(+), 9 deletions(-) diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 9f56d47203..4ade8aadf7 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -1,12 +1,23 @@ import base58 import os +from crypto.bls.bls_crypto import BlsCryptoVerifier + +from plenum.bls.bls_crypto_factory import create_default_bls_crypto_factory + +from plenum.server.quorums import Quorums + +from crypto.bls.bls_multi_signature import MultiSignatureValue + +from state.pruning_state import PruningState + from plenum.test.input_validation.utils import b58_by_len from plenum.test.node_request.helper import sdk_ensure_pool_functional -from common.serializers.serialization import state_roots_serializer -from plenum.common.constants import DOMAIN_LEDGER_ID, ALIAS, BLS_KEY +from common.serializers.serialization import state_roots_serializer, proof_nodes_serializer +from plenum.common.constants import DOMAIN_LEDGER_ID, ALIAS, BLS_KEY, STATE_PROOF, TXN_TYPE, MULTI_SIGNATURE, \ + MULTI_SIGNATURE_PARTICIPANTS, MULTI_SIGNATURE_SIGNATURE, MULTI_SIGNATURE_VALUE from plenum.common.keygen_utils import init_bls_keys from plenum.common.messages.node_messages import Commit, Prepare, PrePrepare from plenum.common.util import get_utc_epoch, randomString, random_from_alphabet, hexToFriendly @@ -14,6 +25,9 @@ from plenum.test.node_catchup.helper import waitNodeDataEquality, ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.pool_transactions.helper import updateNodeData, new_client, sdk_send_update_node, sdk_add_new_nym, \ sdk_pool_refresh +from stp_core.common.log import getlogger + +logger = getlogger() def generate_state_root(): @@ -285,3 +299,67 @@ def check_update_bls_key(node_num, saved_multi_sigs_count, sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, saved_multi_sigs_count) + + +def validate_proof(result): + """ + Validates state proof + """ + state_root_hash = result[STATE_PROOF]['root_hash'] + state_root_hash = state_roots_serializer.deserialize(state_root_hash) + proof_nodes = result[STATE_PROOF]['proof_nodes'] + if isinstance(proof_nodes, str): + proof_nodes = proof_nodes.encode() + proof_nodes = proof_nodes_serializer.deserialize(proof_nodes) + key, value = prepare_for_state(result) + valid = PruningState.verify_state_proof(state_root_hash, + key, + value, + proof_nodes, + serialized=True) + return valid + + +def prepare_for_state(result): + if result[TXN_TYPE] == "buy": + from plenum.test.test_node import TestDomainRequestHandler + key, value = TestDomainRequestHandler.prepare_buy_for_state(result) + return key, value + + +def validate_multi_signature(state_proof, nodeCount): + """ + Validates multi signature + """ + multi_signature = state_proof[MULTI_SIGNATURE] + if not multi_signature: + logger.debug("There is a state proof, but no multi signature") + return False + + participants = multi_signature[MULTI_SIGNATURE_PARTICIPANTS] + signature = multi_signature[MULTI_SIGNATURE_SIGNATURE] + value = MultiSignatureValue( + **(multi_signature[MULTI_SIGNATURE_VALUE]) + ).as_single_value() + quorums = Quorums(nodeCount) + if not quorums.bls_signatures.is_reached(len(participants)): + logger.debug("There is not enough participants of " + "multi-signature") + return False + public_keys = [] + for node_name in participants: + key = self._bls_register.get_key_by_name(node_name) + if key is None: + logger.debug("There is no bls key for node {}" + .format(node_name)) + return False + public_keys.append(key) + _multi_sig_verifier = _create_multi_sig_verifier() + return _multi_sig_verifier.verify_multi_sig(signature, + value, + public_keys) + +def _create_multi_sig_verifier() -> BlsCryptoVerifier: + verifier = create_default_bls_crypto_factory() \ + .create_bls_crypto_verifier() + return verifier \ No newline at end of file diff --git a/plenum/test/bls/test_state_proof.py b/plenum/test/bls/test_state_proof.py index ef1de21b32..f0caf782f7 100644 --- a/plenum/test/bls/test_state_proof.py +++ b/plenum/test/bls/test_state_proof.py @@ -7,6 +7,7 @@ from plenum.common.request import SafeRequest from plenum.common.types import f from plenum.common.util import get_utc_epoch +from plenum.test.bls.helper import validate_proof, validate_multi_signature from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, wait_for_requests_ordered, \ randomOperation from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ @@ -37,7 +38,7 @@ def check_result(txnPoolNodeSet, req, client, should_have_proof): if should_have_proof: assert result[STATE_PROOF] == proof - assert client.validate_proof(result) + assert validate_proof(result) else: assert STATE_PROOF not in result @@ -76,7 +77,7 @@ def test_make_proof_bls_enabled(looper, txnPoolNodeSet, MULTI_SIGNATURE_VALUE_TIMESTAMP, MULTI_SIGNATURE_VALUE_TXN_ROOT] == value_keys - assert client1.validate_multi_signature(proof) + assert validate_multi_signature(proof, nodeCount) def test_make_result_bls_enabled(looper, txnPoolNodeSet, @@ -158,8 +159,8 @@ def test_proof_in_reply(looper, txnPoolNodeSet, assert MULTI_SIGNATURE_VALUE_POOL_STATE_ROOT in multi_sig_value assert MULTI_SIGNATURE_VALUE_TIMESTAMP in multi_sig_value - assert client1.validate_multi_signature(state_proof) - assert client1.validate_proof(result) + assert validate_multi_signature(state_proof, nodeCount) + assert validate_proof(result) def test_make_proof_committed_head_used(looper, txnPoolNodeSet, diff --git a/plenum/test/bls/test_update_incorrect_bls_key.py b/plenum/test/bls/test_update_incorrect_bls_key.py index 026d3cff22..b2be1f6244 100644 --- a/plenum/test/bls/test_update_incorrect_bls_key.py +++ b/plenum/test/bls/test_update_incorrect_bls_key.py @@ -37,7 +37,7 @@ def test_update_incorrect_bls_two_nodes(looper, txnPoolNodeSet, Updated with wrong BLS key for 1st and 2d Nodes; do not expect that BLS multi-sigs are applied (we have less than n-f correct BLS sigs) ''' - check_update_bls_key(node_num=1, saved_multi_sigs_count=4, + check_update_bls_key(node_num=1, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, sdk_wallet_client=sdk_wallet_client, @@ -53,7 +53,7 @@ def test_update_incorrect_bls_three_nodes(looper, txnPoolNodeSet, Updated with wrong BLS keys 1-3 Nodes; do not expect that BLS multi-sigs are applied (we have less than n-f correct BLS sigs) ''' - check_update_bls_key(node_num=2, saved_multi_sigs_count=4, + check_update_bls_key(node_num=2, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, sdk_wallet_client=sdk_wallet_client, @@ -69,7 +69,7 @@ def test_update_incorrect_bls_all_nodes(looper, txnPoolNodeSet, Updated with wrong BLS keys all Nodes; do not expect that BLS multi-sigs are applied (we have less than n-f correct BLS sigs) ''' - check_update_bls_key(node_num=3, saved_multi_sigs_count=4, + check_update_bls_key(node_num=3, saved_multi_sigs_count=0, looper=looper, txnPoolNodeSet=txnPoolNodeSet, sdk_wallet_stewards=sdk_wallet_stewards, sdk_wallet_client=sdk_wallet_client, From 8cc56c664063a6d670c792d5a3de12dc5da087bc Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 16 Mar 2018 08:43:34 +0300 Subject: [PATCH 14/73] Tests integration done Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 18 ++++---- plenum/test/bls/test_state_proof.py | 69 ++++++++++++++--------------- 2 files changed, 42 insertions(+), 45 deletions(-) diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 4ade8aadf7..aebb762f31 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -11,8 +11,6 @@ from state.pruning_state import PruningState -from plenum.test.input_validation.utils import b58_by_len - from plenum.test.node_request.helper import sdk_ensure_pool_functional from common.serializers.serialization import state_roots_serializer, proof_nodes_serializer @@ -23,7 +21,7 @@ from plenum.common.util import get_utc_epoch, randomString, random_from_alphabet, hexToFriendly from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, ensureClientConnectedToNodesAndPoolLedgerSame -from plenum.test.pool_transactions.helper import updateNodeData, new_client, sdk_send_update_node, sdk_add_new_nym, \ +from plenum.test.pool_transactions.helper import updateNodeData, sdk_send_update_node, \ sdk_pool_refresh from stp_core.common.log import getlogger @@ -327,7 +325,7 @@ def prepare_for_state(result): return key, value -def validate_multi_signature(state_proof, nodeCount): +def validate_multi_signature(state_proof, txnPoolNodeSet): """ Validates multi signature """ @@ -341,14 +339,15 @@ def validate_multi_signature(state_proof, nodeCount): value = MultiSignatureValue( **(multi_signature[MULTI_SIGNATURE_VALUE]) ).as_single_value() - quorums = Quorums(nodeCount) + quorums = Quorums(len(txnPoolNodeSet)) if not quorums.bls_signatures.is_reached(len(participants)): logger.debug("There is not enough participants of " "multi-signature") return False public_keys = [] for node_name in participants: - key = self._bls_register.get_key_by_name(node_name) + key = next(node.bls_bft.bls_crypto_signer.pk for node + in txnPoolNodeSet if node.name == node_name) if key is None: logger.debug("There is no bls key for node {}" .format(node_name)) @@ -356,10 +355,11 @@ def validate_multi_signature(state_proof, nodeCount): public_keys.append(key) _multi_sig_verifier = _create_multi_sig_verifier() return _multi_sig_verifier.verify_multi_sig(signature, - value, - public_keys) + value, + public_keys) + def _create_multi_sig_verifier() -> BlsCryptoVerifier: verifier = create_default_bls_crypto_factory() \ .create_bls_crypto_verifier() - return verifier \ No newline at end of file + return verifier diff --git a/plenum/test/bls/test_state_proof.py b/plenum/test/bls/test_state_proof.py index f0caf782f7..2b780a7dc4 100644 --- a/plenum/test/bls/test_state_proof.py +++ b/plenum/test/bls/test_state_proof.py @@ -1,5 +1,5 @@ from plenum.common.constants import ROOT_HASH, MULTI_SIGNATURE, PROOF_NODES, TXN_TYPE, DATA, TXN_TIME, STATE_PROOF, \ - DOMAIN_LEDGER_ID, MULTI_SIGNATURE_VALUE, MULTI_SIGNATURE_PARTICIPANTS, MULTI_SIGNATURE_SIGNATURE, \ + MULTI_SIGNATURE_VALUE, MULTI_SIGNATURE_PARTICIPANTS, MULTI_SIGNATURE_SIGNATURE, \ MULTI_SIGNATURE_VALUE_LEDGER_ID, \ MULTI_SIGNATURE_VALUE_STATE_ROOT, MULTI_SIGNATURE_VALUE_TXN_ROOT, MULTI_SIGNATURE_VALUE_POOL_STATE_ROOT, \ MULTI_SIGNATURE_VALUE_TIMESTAMP, DOMAIN_LEDGER_ID @@ -8,16 +8,14 @@ from plenum.common.types import f from plenum.common.util import get_utc_epoch from plenum.test.bls.helper import validate_proof, validate_multi_signature -from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, wait_for_requests_ordered, \ - randomOperation -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected +from plenum.test.helper import wait_for_requests_ordered, \ + randomOperation, sdk_send_random_requests, sdk_json_couples_to_request_list, sdk_send_random_and_check nodeCount = 4 nodes_wth_bls = 4 -def check_result(txnPoolNodeSet, req, client, should_have_proof): +def check_result(txnPoolNodeSet, req, should_have_proof): for node in txnPoolNodeSet: req_handler = node.get_req_handler(DOMAIN_LEDGER_ID) key = req_handler.prepare_buy_key(req.identifier, req.reqId) @@ -44,8 +42,10 @@ def check_result(txnPoolNodeSet, req, client, should_have_proof): def test_make_proof_bls_enabled(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): - reqs = sendRandomRequests(wallet1, client1, 1) + sdk_pool_handle, sdk_wallet_client): + reqs = sdk_json_couples_to_request_list( + sdk_send_random_requests( + looper, sdk_pool_handle, sdk_wallet_client, 1)) wait_for_requests_ordered(looper, txnPoolNodeSet, reqs) req = reqs[0] @@ -77,67 +77,62 @@ def test_make_proof_bls_enabled(looper, txnPoolNodeSet, MULTI_SIGNATURE_VALUE_TIMESTAMP, MULTI_SIGNATURE_VALUE_TXN_ROOT] == value_keys - assert validate_multi_signature(proof, nodeCount) + assert validate_multi_signature(proof, txnPoolNodeSet) def test_make_result_bls_enabled(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): - reqs = sendRandomRequests(wallet1, client1, 1) + sdk_pool_handle, sdk_wallet_client): + reqs = sdk_json_couples_to_request_list( + sdk_send_random_requests( + looper, sdk_pool_handle, sdk_wallet_client, 1)) wait_for_requests_ordered(looper, txnPoolNodeSet, reqs) req = reqs[0] assert req.protocolVersion assert req.protocolVersion >= PlenumProtocolVersion.STATE_PROOF_SUPPORT.value - check_result(txnPoolNodeSet, req, client1, True) + check_result(txnPoolNodeSet, req, True) -def test_make_result_no_protocol_version(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): +def test_make_result_no_protocol_version(looper, txnPoolNodeSet): request = SafeRequest(identifier="1" * 16, reqId=1, operation=randomOperation(), signature="signature") request.protocolVersion = False - check_result(txnPoolNodeSet, request, client1, False) + check_result(txnPoolNodeSet, request, False) def test_make_result_protocol_version_less_than_state_proof(looper, - txnPoolNodeSet, - client1, - client1Connected, - wallet1): + txnPoolNodeSet): request = SafeRequest(identifier="1" * 16, reqId=1, operation=randomOperation(), signature="signature") request.protocolVersion = 0 - check_result(txnPoolNodeSet, request, client1, False) + check_result(txnPoolNodeSet, request, False) def test_make_result_no_protocol_version_in_request_by_default(looper, - txnPoolNodeSet, - client1, - client1Connected, - wallet1): + txnPoolNodeSet): request = SafeRequest(identifier="1" * 16, reqId=1, operation=randomOperation(), signature="signature") - check_result(txnPoolNodeSet, request, client1, False) + check_result(txnPoolNodeSet, request, False) def test_proof_in_reply(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): - reqs = sendRandomRequests(wallet1, client1, 1) - waitForSufficientRepliesForRequests(looper, client1, requests=reqs) + sdk_pool_handle, sdk_wallet_client): + resp = sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, 1) - req = reqs[0] - result = client1.getReply(req.identifier, req.reqId)[0] + req = resp[0][0] + result = resp[0][1]['result'] assert result assert result[TXN_TYPE] == "buy" - assert result[f.IDENTIFIER.nm] == req.identifier - assert result[f.REQ_ID.nm] == req.reqId + assert result[f.IDENTIFIER.nm] == req[f.IDENTIFIER.nm] + assert result[f.REQ_ID.nm] == req[f.REQ_ID.nm] assert result[f.SEQ_NO.nm] assert result[TXN_TIME] assert STATE_PROOF in result @@ -159,13 +154,15 @@ def test_proof_in_reply(looper, txnPoolNodeSet, assert MULTI_SIGNATURE_VALUE_POOL_STATE_ROOT in multi_sig_value assert MULTI_SIGNATURE_VALUE_TIMESTAMP in multi_sig_value - assert validate_multi_signature(state_proof, nodeCount) + assert validate_multi_signature(state_proof, txnPoolNodeSet) assert validate_proof(result) def test_make_proof_committed_head_used(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): - reqs = sendRandomRequests(wallet1, client1, 1) + sdk_pool_handle, sdk_wallet_client): + reqs = sdk_json_couples_to_request_list( + sdk_send_random_requests( + looper, sdk_pool_handle, sdk_wallet_client, 1)) wait_for_requests_ordered(looper, txnPoolNodeSet, reqs) req = reqs[0] req_handler = txnPoolNodeSet[0].get_req_handler(DOMAIN_LEDGER_ID) @@ -174,4 +171,4 @@ def test_make_proof_committed_head_used(looper, txnPoolNodeSet, for node in txnPoolNodeSet: node.states[DOMAIN_LEDGER_ID].set(key, b'somevalue') - check_result(txnPoolNodeSet, req, client1, True) + check_result(txnPoolNodeSet, req, True) From 509a919305d92cbc469cfa9ecd2468b1426ffd7e Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 16 Mar 2018 08:55:41 +0300 Subject: [PATCH 15/73] Deletion of Client1Connected. More test fix Signed-off-by: ArtObr --- plenum/test/batching_3pc/conftest.py | 6 ------ plenum/test/bls/test_send_txns_bls_less_than_consensus.py | 4 +--- plenum/test/bls/test_send_txns_full_bls.py | 4 +--- plenum/test/bls/test_update_incorrect_bls_key.py | 2 -- .../checkpoints/test_checkpoint_bounds_after_catchup.py | 2 +- .../test_checkpoint_stabilization_after_catchup.py | 2 +- plenum/test/node_catchup/test_large_catchup.py | 1 - plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py | 3 +-- .../node_catchup/test_node_catchup_after_checkpoints.py | 1 - .../test/node_catchup/test_node_catchup_causes_no_desync.py | 2 +- .../node_catchup/test_remove_request_keys_post_catchup.py | 3 +-- plenum/test/node_catchup/test_revert_during_catchup.py | 3 +-- ...test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py | 2 +- ...st_quorum_f_plus_2_nodes_including_primary_off_and_on.py | 2 +- plenum/test/plugin/demo_plugin/test_plugin_basic.py | 2 +- .../test/plugin/demo_plugin/test_plugin_request_handling.py | 6 +++--- .../test_new_node_accepts_chosen_primary.py | 2 +- .../primary_selection/test_recover_more_than_f_failure.py | 3 +-- .../test_recover_primary_no_view_change.py | 3 +-- plenum/test/test_current_state_propagation.py | 3 --- plenum/test/test_performance.py | 5 +---- plenum/test/test_state_regenerated_from_ledger.py | 4 ---- plenum/test/view_change/slow_nodes/conftest.py | 6 ------ .../test_master_primary_different_from_previous.py | 2 +- .../test_node_detecting_lag_from_view_change_messages.py | 1 - plenum/test/zstack_tests/test_zstack_reconnection.py | 2 +- 26 files changed, 20 insertions(+), 56 deletions(-) delete mode 100644 plenum/test/view_change/slow_nodes/conftest.py diff --git a/plenum/test/batching_3pc/conftest.py b/plenum/test/batching_3pc/conftest.py index 4f382c489c..4f35a7364a 100644 --- a/plenum/test/batching_3pc/conftest.py +++ b/plenum/test/batching_3pc/conftest.py @@ -13,9 +13,3 @@ def reset(): request.addfinalizer(reset) return tconf - - -@pytest.fixture(scope="module") -def client(tconf, looper, txnPoolNodeSet, client1, - client1Connected): - return client1Connected diff --git a/plenum/test/bls/test_send_txns_bls_less_than_consensus.py b/plenum/test/bls/test_send_txns_bls_less_than_consensus.py index 96630ab5a4..cf84e82e3c 100644 --- a/plenum/test/bls/test_send_txns_bls_less_than_consensus.py +++ b/plenum/test/bls/test_send_txns_bls_less_than_consensus.py @@ -1,7 +1,5 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 7 nodes_wth_bls = 4 @@ -14,7 +12,7 @@ def test_each_node_has_bls(txnPoolNodeSet): def test_send_txns_bls_less_than_consensus(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): + client1, wallet1): # make sure that we have commits from all nodes, and have 4 of 7 ( < n-f) BLS sigs there is not enough for node in txnPoolNodeSet: node.quorums.commit = Quorum(nodeCount) diff --git a/plenum/test/bls/test_send_txns_full_bls.py b/plenum/test/bls/test_send_txns_full_bls.py index fbfa936ebb..1e7b8dfecb 100644 --- a/plenum/test/bls/test_send_txns_full_bls.py +++ b/plenum/test/bls/test_send_txns_full_bls.py @@ -1,6 +1,4 @@ from plenum.test.bls.helper import check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 7 nodes_wth_bls = 7 @@ -13,7 +11,7 @@ def test_each_node_has_bls(txnPoolNodeSet): def test_send_txns_full_bls(looper, txnPoolNodeSet, - client1, client1Connected, wallet1): + client1, wallet1): check_bls_multi_sig_after_send(looper, txnPoolNodeSet, client1, wallet1, saved_multi_sigs_count=nodeCount) diff --git a/plenum/test/bls/test_update_incorrect_bls_key.py b/plenum/test/bls/test_update_incorrect_bls_key.py index 29694fe732..21d5bceda6 100644 --- a/plenum/test/bls/test_update_incorrect_bls_key.py +++ b/plenum/test/bls/test_update_incorrect_bls_key.py @@ -1,7 +1,5 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected nodeCount = 4 nodes_wth_bls = 4 diff --git a/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py index 872b7827f5..87c4bde305 100644 --- a/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py @@ -8,7 +8,7 @@ def test_upper_bound_of_checkpoint_after_catchup_is_divisible_by_chk_freq( chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet, - client1, wallet1, client1Connected, tdir, client_tdir, tconf, + client1, wallet1, tdir, client_tdir, tconf, allPluginsPath): sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) diff --git a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py index 8a50eaf2e1..ce154503bd 100644 --- a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py @@ -10,7 +10,7 @@ def test_second_checkpoint_after_catchup_can_be_stabilized( chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet, - client1, wallet1, client1Connected, tdir, client_tdir, tconf, + client1, wallet1, tdir, client_tdir, tconf, allPluginsPath): _, _, epsilon = addNewStewardAndNode(looper, steward1, stewardWallet, 'EpsilonSteward', 'Epsilon', diff --git a/plenum/test/node_catchup/test_large_catchup.py b/plenum/test/node_catchup/test_large_catchup.py index 067394f0bb..fa022ccf87 100644 --- a/plenum/test/node_catchup/test_large_catchup.py +++ b/plenum/test/node_catchup/test_large_catchup.py @@ -46,7 +46,6 @@ def test_large_catchup(tdir, tconf, txnPoolNodeSet, wallet1, client1, - client1Connected, allPluginsPath): """ Checks that node can catchup large ledgers diff --git a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py index 8d02589089..9e14786c74 100644 --- a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py +++ b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py @@ -12,8 +12,7 @@ from plenum.test.view_change.helper import ensure_view_change -def test_no_catchup_if_got_from_3pc(looper, txnPoolNodeSet, wallet1, client1, - client1Connected): +def test_no_catchup_if_got_from_3pc(looper, txnPoolNodeSet, wallet1, client1): """ A node is slow to receive COMMIT messages so after a view change it starts catchup. But before it can start requesting txns, the COMMITs messages diff --git a/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py b/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py index e78dcbbf8d..900ca02475 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py +++ b/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py @@ -21,7 +21,6 @@ def test_node_catchup_after_checkpoints( txnPoolNodeSet, wallet1, client1, - client1Connected, broken_node_and_others): """ For some reason a node misses 3pc messages but eventually the node stashes diff --git a/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py b/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py index 7d68fe9efb..8d60e0052e 100644 --- a/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py +++ b/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py @@ -42,7 +42,7 @@ def replicas_synced(node): def test_node_catchup_causes_no_desync(looper, txnPoolNodeSet, client1, - wallet1, client1Connected, monkeypatch): + wallet1, monkeypatch): """ Checks that transactions received by catchup do not break performance monitoring diff --git a/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py b/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py index 786f6fc1d2..dc543d9195 100644 --- a/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py +++ b/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py @@ -12,8 +12,7 @@ @pytest.fixture(scope='module', params=['some', 'all']) -def setup(request, looper, txnPoolNodeSet, client1, wallet1, - client1Connected): +def setup(request, looper, txnPoolNodeSet, client1, wallet1): slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[1].node fast_nodes = [n for n in txnPoolNodeSet if n != slow_node] # Delay catchup reply so that the test gets time to make the check, diff --git a/plenum/test/node_catchup/test_revert_during_catchup.py b/plenum/test/node_catchup/test_revert_during_catchup.py index c18a4530e9..67e97a317d 100644 --- a/plenum/test/node_catchup/test_revert_during_catchup.py +++ b/plenum/test/node_catchup/test_revert_during_catchup.py @@ -23,8 +23,7 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, txnPoolNodeSet, client1, - wallet1, - client1Connected): + wallet1): """ Delay COMMITs to a node such that when it needs to catchup, it needs to revert some unordered state. Also till this time the node should have diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py index 18c0bad7dc..a411b1f4cf 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py @@ -19,7 +19,7 @@ def stop_node(node_to_stop, looper, pool_nodes): def test_quorum_after_f_plus_2_nodes_but_not_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, - txnPoolNodeSet, wallet1, client1, client1Connected): + txnPoolNodeSet, wallet1, client1): nodes = txnPoolNodeSet request1 = sendRandomRequest(wallet1, client1) diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py index 3f9fa275a4..a0a7f754fe 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py @@ -19,7 +19,7 @@ def stop_node(node_to_stop, looper, pool_nodes): def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, - txnPoolNodeSet, wallet1, client1, client1Connected): + txnPoolNodeSet, wallet1, client1): nodes = txnPoolNodeSet request1 = sendRandomRequest(wallet1, client1) diff --git a/plenum/test/plugin/demo_plugin/test_plugin_basic.py b/plenum/test/plugin/demo_plugin/test_plugin_basic.py index 53b5abd289..ef0eb5dad4 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_basic.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_basic.py @@ -20,7 +20,7 @@ def test_plugin_setup(nodeSet): def test_plugin_client_req_fields(nodeSet, looper, stewardWallet, - steward1, client1Connected, + steward1, sdk_wallet_steward, sdk_pool_handle): """ Test that plugin's addition of request fields and their validation is diff --git a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py index 9103dc6783..f0402bb550 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py @@ -17,7 +17,7 @@ def successful_op(looper, op, sdk_wallet, sdk_pool_handle): def test_plugin_static_validation(nodeSet, looper, stewardWallet, - steward1, client1Connected, + steward1, sdk_wallet_steward, sdk_pool_handle): """ Check plugin static validation fails and passes @@ -60,7 +60,7 @@ def test_plugin_static_validation(nodeSet, looper, stewardWallet, def test_plugin_dynamic_validation(nodeSet, looper, stewardWallet, - steward1, client1Connected, + steward1, sdk_wallet_steward, sdk_pool_handle): """ Check plugin dynamic validation fails and passes @@ -88,7 +88,7 @@ def test_plugin_dynamic_validation(nodeSet, looper, stewardWallet, @pytest.fixture(scope="module") def some_requests(nodeSet, looper, stewardWallet, - steward1, client1Connected, + steward1, sdk_wallet_steward, sdk_pool_handle): op = { TXN_TYPE: AUCTION_START, diff --git a/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py b/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py index ff67f61d03..de4ec93517 100644 --- a/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py +++ b/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py @@ -40,7 +40,7 @@ def testNodeClass(patchPluginManager): @pytest.fixture("module") -def txnPoolNodeSet(txnPoolNodeSet, looper, client1, wallet1, client1Connected, +def txnPoolNodeSet(txnPoolNodeSet, looper, client1, wallet1, tconf, tdirWithPoolTxns, allPluginsPath): logger.debug("Do several view changes to round the list of primaries") diff --git a/plenum/test/primary_selection/test_recover_more_than_f_failure.py b/plenum/test/primary_selection/test_recover_more_than_f_failure.py index a502723314..e373712fda 100644 --- a/plenum/test/primary_selection/test_recover_more_than_f_failure.py +++ b/plenum/test/primary_selection/test_recover_more_than_f_failure.py @@ -15,8 +15,7 @@ def test_recover_stop_primaries(looper, checkpoint_size, txnPoolNodeSet, - allPluginsPath, tdir, tconf, client1, wallet1, - client1Connected): + allPluginsPath, tdir, tconf, client1, wallet1): """ Test that we can recover after having more than f nodes disconnected: - stop current master primary (Alpha) diff --git a/plenum/test/primary_selection/test_recover_primary_no_view_change.py b/plenum/test/primary_selection/test_recover_primary_no_view_change.py index e55f08c730..19ec82af8d 100644 --- a/plenum/test/primary_selection/test_recover_primary_no_view_change.py +++ b/plenum/test/primary_selection/test_recover_primary_no_view_change.py @@ -24,8 +24,7 @@ def tconf(tconf): def test_recover_stop_primaries_no_view_change(looper, checkpoint_size, txnPoolNodeSet, - allPluginsPath, tdir, tconf, client1, wallet1, - client1Connected): + allPluginsPath, tdir, tconf, client1, wallet1): """ Test that we can recover after having more than f nodes disconnected: - send txns diff --git a/plenum/test/test_current_state_propagation.py b/plenum/test/test_current_state_propagation.py index b73cea8059..01aa43246a 100644 --- a/plenum/test/test_current_state_propagation.py +++ b/plenum/test/test_current_state_propagation.py @@ -5,9 +5,6 @@ disconnect_node_and_ensure_disconnected, reconnect_node_and_ensure_connected from plenum.test.node_catchup.helper import waitNodeDataEquality, \ waitNodeDataInequality, checkNodeDataForEquality -from plenum.test.pool_transactions.conftest import stewardAndWallet1, \ - steward1, stewardWallet, clientAndWallet1, client1, wallet1, \ - client1Connected from plenum.test.test_node import checkNodesConnected from plenum.test.view_change.helper import start_stopped_node diff --git a/plenum/test/test_performance.py b/plenum/test/test_performance.py index bc6f325459..086b6f04b1 100644 --- a/plenum/test/test_performance.py +++ b/plenum/test/test_performance.py @@ -25,8 +25,6 @@ # noinspection PyUnresolvedReferences from plenum.test.node_catchup.conftest import whitelist, \ nodeCreatedAfterSomeTxns, nodeSetWithNodeAddedAfterSomeTxns, newNodeCaughtUp -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected @pytest.fixture @@ -173,7 +171,7 @@ def test_node_load_consistent_time(tconf, change_checkpoint_freq, def test_node_load_after_add(newNodeCaughtUp, txnPoolNodeSet, tconf, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, looper, client1, wallet1, - client1Connected, capsys): + capsys): """ A node that restarts after some transactions should eventually get the transactions which happened while it was down @@ -202,7 +200,6 @@ def test_node_load_after_add(newNodeCaughtUp, txnPoolNodeSet, tconf, @skipper def test_node_load_after_add_then_disconnect(newNodeCaughtUp, txnPoolNodeSet, tconf, looper, client1, wallet1, - client1Connected, tdirWithPoolTxns, allPluginsPath, poolTxnStewardData, capsys): """ diff --git a/plenum/test/test_state_regenerated_from_ledger.py b/plenum/test/test_state_regenerated_from_ledger.py index 06f6e145e5..6bb014c49d 100644 --- a/plenum/test/test_state_regenerated_from_ledger.py +++ b/plenum/test/test_state_regenerated_from_ledger.py @@ -5,9 +5,6 @@ from plenum.test.helper import send_reqs_batches_and_get_suff_replies from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data, \ waitNodeDataEquality - -from plenum.test.pool_transactions.conftest import looper, clientAndWallet1, \ - client1, wallet1, client1Connected from plenum.test.test_node import checkNodesConnected, TestNode from plenum.common.config_helper import PNodeConfigHelper from stp_core.types import HA @@ -20,7 +17,6 @@ def test_state_regenerated_from_ledger( txnPoolNodeSet, client1, wallet1, - client1Connected, tdir, tconf, allPluginsPath): diff --git a/plenum/test/view_change/slow_nodes/conftest.py b/plenum/test/view_change/slow_nodes/conftest.py deleted file mode 100644 index c3a45be43c..0000000000 --- a/plenum/test/view_change/slow_nodes/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -import pytest - - -@pytest.fixture(scope="module") -def client(looper, txnPoolNodeSet, client1, client1Connected): - return client1Connected diff --git a/plenum/test/view_change/test_master_primary_different_from_previous.py b/plenum/test/view_change/test_master_primary_different_from_previous.py index 210308214a..af396404de 100644 --- a/plenum/test/view_change/test_master_primary_different_from_previous.py +++ b/plenum/test/view_change/test_master_primary_different_from_previous.py @@ -41,7 +41,7 @@ def test_master_primary_different_from_previous(txnPoolNodeSet, looper, @pytest.mark.skip(reason='Nodes use round robin primary selection') def test_master_primary_different_from_previous_view_for_itself( - txnPoolNodeSet, looper, client1, wallet1, client1Connected): + txnPoolNodeSet, looper, client1, wallet1): """ After a view change, primary must be different from previous primary for master instance, it does not matter for other instance. Break it into diff --git a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py index 9e73ef1fe6..d5d13b3e12 100644 --- a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py +++ b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py @@ -19,7 +19,6 @@ def test_node_detecting_lag_from_view_change_done_messages(txnPoolNodeSet, looper, wallet1, client1, - client1Connected, tconf): """ A node is slow and after view change starts, it marks it's `last_prepared` diff --git a/plenum/test/zstack_tests/test_zstack_reconnection.py b/plenum/test/zstack_tests/test_zstack_reconnection.py index 1438789485..f351ead6fd 100644 --- a/plenum/test/zstack_tests/test_zstack_reconnection.py +++ b/plenum/test/zstack_tests/test_zstack_reconnection.py @@ -20,7 +20,7 @@ def tconf(tconf): def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet, client1, wallet1, - tdir, client1Connected): + tdir): sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) npr = [n for n in txnPoolNodeSet if not n.hasPrimary] From c4dc58301542e0a98294520cfa04f5d8302ebf82 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 16 Mar 2018 09:03:57 +0300 Subject: [PATCH 16/73] Import deletion. flake8 error fix. Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 2 +- plenum/test/bls/test_no_state_proof.py | 2 +- plenum/test/bls/test_send_txns_bls_consensus.py | 1 - plenum/test/bls/test_send_txns_bls_less_than_consensus.py | 3 +-- plenum/test/bls/test_send_txns_full_bls.py | 1 - plenum/test/bls/test_send_txns_no_bls.py | 1 - plenum/test/bls/test_update_bls_key.py | 1 - plenum/test/bls/test_update_incorrect_bls_key.py | 1 - scripts/generate_txns.py | 6 +++--- 9 files changed, 6 insertions(+), 12 deletions(-) diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index aebb762f31..037a92deb6 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -21,7 +21,7 @@ from plenum.common.util import get_utc_epoch, randomString, random_from_alphabet, hexToFriendly from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, ensureClientConnectedToNodesAndPoolLedgerSame -from plenum.test.pool_transactions.helper import updateNodeData, sdk_send_update_node, \ +from plenum.test.pool_transactions.helper import updateNodeData, sdk_send_update_node, \ sdk_pool_refresh from stp_core.common.log import getlogger diff --git a/plenum/test/bls/test_no_state_proof.py b/plenum/test/bls/test_no_state_proof.py index 3c2f09aff3..a8130c79d5 100644 --- a/plenum/test/bls/test_no_state_proof.py +++ b/plenum/test/bls/test_no_state_proof.py @@ -1,6 +1,6 @@ from plenum.common.constants import TXN_TYPE, STATE_PROOF, DOMAIN_LEDGER_ID from plenum.common.util import get_utc_epoch -from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, sdk_send_random_and_check, \ +from plenum.test.helper import sdk_send_random_and_check, \ sdk_json_to_request_object nodeCount = 4 diff --git a/plenum/test/bls/test_send_txns_bls_consensus.py b/plenum/test/bls/test_send_txns_bls_consensus.py index 7409ceba72..0f0a29df61 100644 --- a/plenum/test/bls/test_send_txns_bls_consensus.py +++ b/plenum/test/bls/test_send_txns_bls_consensus.py @@ -1,6 +1,5 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import sdk_check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper nodeCount = 7 nodes_wth_bls = 5 diff --git a/plenum/test/bls/test_send_txns_bls_less_than_consensus.py b/plenum/test/bls/test_send_txns_bls_less_than_consensus.py index 370d6c3993..f4a7654cda 100644 --- a/plenum/test/bls/test_send_txns_bls_less_than_consensus.py +++ b/plenum/test/bls/test_send_txns_bls_less_than_consensus.py @@ -1,6 +1,5 @@ from plenum.server.quorums import Quorum -from plenum.test.bls.helper import check_bls_multi_sig_after_send, sdk_check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper +from plenum.test.bls.helper import sdk_check_bls_multi_sig_after_send nodeCount = 7 nodes_wth_bls = 4 diff --git a/plenum/test/bls/test_send_txns_full_bls.py b/plenum/test/bls/test_send_txns_full_bls.py index e5dbf74f16..82134e6533 100644 --- a/plenum/test/bls/test_send_txns_full_bls.py +++ b/plenum/test/bls/test_send_txns_full_bls.py @@ -1,5 +1,4 @@ from plenum.test.bls.helper import sdk_check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper nodeCount = 7 nodes_wth_bls = 7 diff --git a/plenum/test/bls/test_send_txns_no_bls.py b/plenum/test/bls/test_send_txns_no_bls.py index e0ed9bc53e..8d48d3b2a3 100644 --- a/plenum/test/bls/test_send_txns_no_bls.py +++ b/plenum/test/bls/test_send_txns_no_bls.py @@ -1,5 +1,4 @@ from plenum.test.bls.helper import sdk_check_bls_multi_sig_after_send -from plenum.test.pool_transactions.conftest import looper nodeCount = 4 nodes_wth_bls = 0 diff --git a/plenum/test/bls/test_update_bls_key.py b/plenum/test/bls/test_update_bls_key.py index 2dcb70d28c..910b764b42 100644 --- a/plenum/test/bls/test_update_bls_key.py +++ b/plenum/test/bls/test_update_bls_key.py @@ -1,5 +1,4 @@ from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper nodeCount = 4 nodes_wth_bls = 4 diff --git a/plenum/test/bls/test_update_incorrect_bls_key.py b/plenum/test/bls/test_update_incorrect_bls_key.py index b2be1f6244..9a020f5ea3 100644 --- a/plenum/test/bls/test_update_incorrect_bls_key.py +++ b/plenum/test/bls/test_update_incorrect_bls_key.py @@ -1,6 +1,5 @@ from plenum.server.quorums import Quorum from plenum.test.bls.helper import check_update_bls_key -from plenum.test.pool_transactions.conftest import looper nodeCount = 4 nodes_wth_bls = 4 diff --git a/scripts/generate_txns.py b/scripts/generate_txns.py index 2d6cb8cab6..068186db99 100644 --- a/scripts/generate_txns.py +++ b/scripts/generate_txns.py @@ -34,7 +34,7 @@ async def get_wallet_and_pool(): await did.create_and_store_my_did(my_wallet_handle, "{}") (their_did, their_verkey) = await did.create_and_store_my_did(their_wallet_handle, - json.dumps({"seed": seed_trustee1})) + json.dumps({"seed": seed_trustee1})) await did.store_their_did(my_wallet_handle, json.dumps({'did': their_did, 'verkey': their_verkey})) @@ -84,10 +84,10 @@ def sdk_signed_random_requests(looper, sdk_wallet, count): with ExitStack() as exit_stack: with Looper() as looper: - sdk_wallet, did = looper.loop.run_until_complete(get_wallet_and_pool()) + sdk_wallet, DID = looper.loop.run_until_complete(get_wallet_and_pool()) with open(path_to_save, 'w') as outpath: for _ in range(args.count): - req = sdk_signed_random_requests(looper, (sdk_wallet, did), 1)[0] + req = sdk_signed_random_requests(looper, (sdk_wallet, DID), 1)[0] txn = sdk_reqToTxn(req, int(time.time())) outpath.write(json.dumps(txn)) outpath.write(os.linesep) From 8cfe95c49e5339e4a7854ba17cd817e5389d77fe Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 16 Mar 2018 09:46:05 +0300 Subject: [PATCH 17/73] More changes to txnPoolNodeSet Signed-off-by: ArtObr --- ...st_primary_sends_preprepare_of_high_num.py | 12 +++---- .../test_primary_selection.py | 35 +++++++++---------- .../test/restart/test_restart_node_4_all.py | 4 --- plenum/test/restart/test_restart_nodes.py | 3 -- .../restart/test_restart_nodes_4_all_wp.py | 4 --- .../test/restart/test_restart_nodes_4_np.py | 4 --- plenum/test/restart/test_restart_nodes_6.py | 4 --- .../test/restart/test_restart_nodes_6_all.py | 4 --- .../restart/test_restart_nodes_6_all_wp.py | 4 --- .../test/restart/test_restart_nodes_6_np.py | 4 --- plenum/test/restart/test_restart_nodes_7.py | 4 --- .../test/restart/test_restart_nodes_7_all.py | 4 --- ...st_discard_inst_chng_msg_from_past_view.py | 26 +++++++------- 13 files changed, 36 insertions(+), 76 deletions(-) diff --git a/plenum/test/node_request/test_pre_prepare/test_primary_sends_preprepare_of_high_num.py b/plenum/test/node_request/test_pre_prepare/test_primary_sends_preprepare_of_high_num.py index 1dd2216c75..0322bc16cb 100644 --- a/plenum/test/node_request/test_pre_prepare/test_primary_sends_preprepare_of_high_num.py +++ b/plenum/test/node_request/test_pre_prepare/test_primary_sends_preprepare_of_high_num.py @@ -15,9 +15,9 @@ @pytest.mark.skip(reason="SOV-555. Not implemented in replica. Add a check in " "replica to check value of preprepare seq number.") -def testPrePrepareWithHighSeqNo(looper, nodeSet, propagated1): +def testPrePrepareWithHighSeqNo(looper, txnPoolNodeSet, propagated1): def chk(): - for r in getNonPrimaryReplicas(nodeSet, instId): + for r in getNonPrimaryReplicas(txnPoolNodeSet, instId): nodeSuspicions = len(getNodeSuspicions( r.node, Suspicions.WRONG_PPSEQ_NO.code)) assert nodeSuspicions == 1 @@ -26,11 +26,11 @@ def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares): assert (replica.prePrepares[viewNo, ppSeqNo][0]) == \ (req.identifier, req.reqId, req.digest) - primary = getPrimaryReplica(nodeSet, instId) - nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, instId) + primary = getPrimaryReplica(txnPoolNodeSet, instId) + nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, instId) req = propagated1.reqDigest primary.doPrePrepare(req) - timeout = waits.expectedPrePrepareTime(len(nodeSet)) + timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) for np in nonPrimaryReplicas: looper.run( eventually(checkPreprepare, np, primary.viewNo, @@ -45,5 +45,5 @@ def checkPreprepare(replica, viewNo, ppSeqNo, req, numOfPrePrepares): get_utc_epoch()) primary.send(incorrectPrePrepareReq, TPCStat.PrePrepareSent) - timeout = waits.expectedPrePrepareTime(len(nodeSet)) + timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) diff --git a/plenum/test/primary_selection/test_primary_selection.py b/plenum/test/primary_selection/test_primary_selection.py index f87c05cf1c..433378d276 100644 --- a/plenum/test/primary_selection/test_primary_selection.py +++ b/plenum/test/primary_selection/test_primary_selection.py @@ -22,14 +22,14 @@ @pytest.fixture() -def primaryReplicas(nodeSet, up): +def primaryReplicas(txnPoolNodeSet): instanceCount = getNoInstances(nodeCount) - return [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)] + return [getPrimaryReplica(txnPoolNodeSet, i) for i in range(instanceCount)] # noinspection PyIncorrectDocstring def testPrimarySelectionAfterPoolReady( - looper, nodeSet, ready, wallet1, client1): + looper, txnPoolNodeSet, wallet1, client1): """ Once the pool is ready(node has connected to at least 3 other nodes), appropriate primary replicas should be selected. @@ -37,7 +37,7 @@ def testPrimarySelectionAfterPoolReady( def checkPrimaryPlacement(): # Node names sorted by rank - sortedNodes = sorted(nodeSet.nodes.values(), + sortedNodes = sorted(txnPoolNodeSet, key=operator.attrgetter("rank")) for idx, node in enumerate(sortedNodes): @@ -65,25 +65,25 @@ def checkPrimaryPlacement(): assert not node.replicas[1].isPrimary assert node.replicas[2].isPrimary - check_rank_consistent_across_each_node(nodeSet) + check_rank_consistent_across_each_node(txnPoolNodeSet) # Check if the primary is on the correct node - timeout = waits.expectedPoolElectionTimeout(len(nodeSet)) + timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) looper.run(eventually(checkPrimaryPlacement, retryWait=1, timeout=timeout)) # Check if every protocol instance has one and only one primary and any node # has no more than one primary - checkProtocolInstanceSetup(looper, nodeSet, retryWait=1) + checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) @pytest.fixture(scope='module') -def catchup_complete_count(nodeSet): - return {n.name: n.spylog.count(n.allLedgersCaughtUp) for n in nodeSet} +def catchup_complete_count(txnPoolNodeSet): + return {n.name: n.spylog.count(n.allLedgersCaughtUp) for n in txnPoolNodeSet} @pytest.fixture(scope='module') # noqa -def view_change_done(looper, nodeSet): - ensure_view_change(looper, nodeSet) - ensureElectionsDone(looper=looper, nodes=nodeSet) +def view_change_done(looper, txnPoolNodeSet): + ensure_view_change(looper, txnPoolNodeSet) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) # noinspection PyIncorrectDocstring @@ -91,8 +91,7 @@ def view_change_done(looper, nodeSet): def testPrimarySelectionAfterViewChange( # noqa looper, - nodeSet, - ready, + txnPoolNodeSet, primaryReplicas, catchup_complete_count, view_change_done): @@ -102,7 +101,7 @@ def testPrimarySelectionAfterViewChange( # noqa """ # TODO: This test can fail due to view change. - for n in nodeSet: + for n in txnPoolNodeSet: assert n.spylog.count( n.allLedgersCaughtUp) > catchup_complete_count[n.name] @@ -111,11 +110,11 @@ def testPrimarySelectionAfterViewChange( # noqa # Primary replicas after view change instanceCount = getNoInstances(nodeCount) - prAfterVC = [getPrimaryReplica(nodeSet, i) for i in range(instanceCount)] + prAfterVC = [getPrimaryReplica(txnPoolNodeSet, i) for i in range(instanceCount)] # Primary replicas have moved to the next node for br, ar in zip(prBeforeVC, prAfterVC): assert ar.node.rank - br.node.rank == 1 - check_rank_consistent_across_each_node(nodeSet) - checkProtocolInstanceSetup(looper, nodeSet, retryWait=1) + check_rank_consistent_across_each_node(txnPoolNodeSet) + checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) diff --git a/plenum/test/restart/test_restart_node_4_all.py b/plenum/test/restart/test_restart_node_4_all.py index 46dbd471e1..6f39800dad 100644 --- a/plenum/test/restart/test_restart_node_4_all.py +++ b/plenum/test/restart/test_restart_node_4_all.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_np(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 4, include_primary=False) diff --git a/plenum/test/restart/test_restart_nodes.py b/plenum/test/restart/test_restart_nodes.py index 665b502002..8816bb3255 100644 --- a/plenum/test/restart/test_restart_nodes.py +++ b/plenum/test/restart/test_restart_nodes.py @@ -5,8 +5,6 @@ from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode -from plenum.test.pool_transactions.conftest import looper - def get_group(nodeSet, group_cnt, include_primary=False): if group_cnt >= len(nodeSet): @@ -60,7 +58,6 @@ def restart_nodes(looper, nodeSet, restart_set, tconf, tdir, allPluginsPath, def test_restart_groups(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 4, include_primary=False) diff --git a/plenum/test/restart/test_restart_nodes_4_all_wp.py b/plenum/test/restart/test_restart_nodes_4_all_wp.py index 5c8f743513..8576f90e95 100644 --- a/plenum/test/restart/test_restart_nodes_4_all_wp.py +++ b/plenum/test/restart/test_restart_nodes_4_all_wp.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_wp(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 4, include_primary=True) diff --git a/plenum/test/restart/test_restart_nodes_4_np.py b/plenum/test/restart/test_restart_nodes_4_np.py index fa7e3ee9ef..b0a6ab713f 100644 --- a/plenum/test/restart/test_restart_nodes_4_np.py +++ b/plenum/test/restart/test_restart_nodes_4_np.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_wp(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 4, include_primary=True) diff --git a/plenum/test/restart/test_restart_nodes_6.py b/plenum/test/restart/test_restart_nodes_6.py index 5dbee1bce1..dd21658233 100644 --- a/plenum/test/restart/test_restart_nodes_6.py +++ b/plenum/test/restart/test_restart_nodes_6.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_wp(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 6, include_primary=False) diff --git a/plenum/test/restart/test_restart_nodes_6_all.py b/plenum/test/restart/test_restart_nodes_6_all.py index 1b476a87cb..5496e7c9b4 100644 --- a/plenum/test/restart/test_restart_nodes_6_all.py +++ b/plenum/test/restart/test_restart_nodes_6_all.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_np(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 6, include_primary=False) diff --git a/plenum/test/restart/test_restart_nodes_6_all_wp.py b/plenum/test/restart/test_restart_nodes_6_all_wp.py index 2631ee3576..5aebce9826 100644 --- a/plenum/test/restart/test_restart_nodes_6_all_wp.py +++ b/plenum/test/restart/test_restart_nodes_6_all_wp.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_wp(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 6, include_primary=True) diff --git a/plenum/test/restart/test_restart_nodes_6_np.py b/plenum/test/restart/test_restart_nodes_6_np.py index f41ee608c3..873ed9ef1a 100644 --- a/plenum/test/restart/test_restart_nodes_6_np.py +++ b/plenum/test/restart/test_restart_nodes_6_np.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_wp(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 6, include_primary=True) diff --git a/plenum/test/restart/test_restart_nodes_7.py b/plenum/test/restart/test_restart_nodes_7.py index a5ee975e40..35b9ef35a8 100644 --- a/plenum/test/restart/test_restart_nodes_7.py +++ b/plenum/test/restart/test_restart_nodes_7.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_wp(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 7, include_primary=False) diff --git a/plenum/test/restart/test_restart_nodes_7_all.py b/plenum/test/restart/test_restart_nodes_7_all.py index f1943fa93b..d11968e92a 100644 --- a/plenum/test/restart/test_restart_nodes_7_all.py +++ b/plenum/test/restart/test_restart_nodes_7_all.py @@ -6,15 +6,11 @@ from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes -from plenum.test.pool_transactions.conftest import looper - - nodeCount = 7 def test_restart_groups_wp(looper, txnPoolNodeSet, tconf, tdir, sdk_pool_handle, sdk_wallet_client, allPluginsPath): - tm = tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) restart_group = get_group(txnPoolNodeSet, 7, include_primary=True) diff --git a/plenum/test/view_change/test_discard_inst_chng_msg_from_past_view.py b/plenum/test/view_change/test_discard_inst_chng_msg_from_past_view.py index a8b5965fce..9c18399952 100644 --- a/plenum/test/view_change/test_discard_inst_chng_msg_from_past_view.py +++ b/plenum/test/view_change/test_discard_inst_chng_msg_from_past_view.py @@ -5,7 +5,7 @@ # noinspection PyIncorrectDocstring -def testDiscardInstChngMsgFrmPastView(nodeSet, looper, ensureView): +def testDiscardInstChngMsgFrmPastView(txnPoolNodeSet, looper, ensureView): """ Once a view change is done, any further INSTANCE_CHANGE messages for that view must be discarded by the node. @@ -14,23 +14,23 @@ def testDiscardInstChngMsgFrmPastView(nodeSet, looper, ensureView): curViewNo = ensureView # Send an instance change for an old instance message to all nodes - icMsg = nodeSet.Alpha.view_changer._create_instance_change_msg(curViewNo, 0) - nodeSet.Alpha.send(icMsg) + icMsg = txnPoolNodeSet[0].view_changer._create_instance_change_msg(curViewNo, 0) + txnPoolNodeSet[0].send(icMsg) # ensure every node but Alpha discards the invalid instance change request - timeout = waits.expectedPoolViewChangeStartedTimeout(len(nodeSet)) + timeout = waits.expectedPoolViewChangeStartedTimeout(len(txnPoolNodeSet)) # Check that that message is discarded. - looper.run(eventually(checkDiscardMsg, nodeSet, icMsg, + looper.run(eventually(checkDiscardMsg, txnPoolNodeSet, icMsg, 'which is not more than its view no', - nodeSet.Alpha, timeout=timeout)) + txnPoolNodeSet[0], timeout=timeout)) - waitForViewChange(looper, nodeSet) + waitForViewChange(looper, txnPoolNodeSet) # noinspection PyIncorrectDocstring def testDoNotSendInstChngMsgIfMasterDoesntSeePerformanceProblem( - nodeSet, looper, ensureView): + txnPoolNodeSet, looper, ensureView): """ A node that received an INSTANCE_CHANGE message must not send an INSTANCE_CHANGE message if it doesn't observe too much difference in @@ -42,17 +42,17 @@ def testDoNotSendInstChngMsgIfMasterDoesntSeePerformanceProblem( # Count sent instance changes of all nodes sentInstChanges = {} instChngMethodName = ViewChanger.sendInstanceChange.__name__ - for n in nodeSet: + for n in txnPoolNodeSet: sentInstChanges[n.name] = n.view_changer.spylog.count(instChngMethodName) # Send an instance change message to all nodes - icMsg = nodeSet.Alpha.view_changer._create_instance_change_msg(curViewNo, 0) - nodeSet.Alpha.send(icMsg) + icMsg = txnPoolNodeSet[0].view_changer._create_instance_change_msg(curViewNo, 0) + txnPoolNodeSet[0].send(icMsg) # Check that that message is discarded. - waitForViewChange(looper, nodeSet) + waitForViewChange(looper, txnPoolNodeSet) # No node should have sent a view change and thus must not have called # `sendInstanceChange` - for n in nodeSet: + for n in txnPoolNodeSet: assert n.spylog.count(instChngMethodName) == \ sentInstChanges.get(n.name, 0) From c5285dab45bb9bde902e072a73aaf4806564dba9 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 19 Mar 2018 16:37:06 +0300 Subject: [PATCH 18/73] Changes in tests Signed-off-by: ArtObr --- .../test/batching_3pc/test_basic_batching.py | 2 +- .../test_primary_election_case6.py | 13 ++-- .../test_primary_selection_routes.py | 2 +- .../test_connections_with_converted_key.py | 6 +- plenum/test/test_dirty_read.py | 4 +- plenum/test/test_node_basic.py | 8 +-- plenum/test/test_node_request.py | 5 +- ...xecuted_once_and_without_failing_behind.py | 3 +- .../test_round_trip_with_one_faulty_node.py | 5 +- plenum/test/view_change/conftest.py | 4 +- plenum/test/view_change/test_view_change.py | 36 +++++------ .../test_view_change_start_without_primary.py | 16 ++--- .../view_change/test_view_change_timeout.py | 60 +++++++++---------- 13 files changed, 82 insertions(+), 82 deletions(-) diff --git a/plenum/test/batching_3pc/test_basic_batching.py b/plenum/test/batching_3pc/test_basic_batching.py index 1006cc3648..4471e71e7d 100644 --- a/plenum/test/batching_3pc/test_basic_batching.py +++ b/plenum/test/batching_3pc/test_basic_batching.py @@ -22,7 +22,7 @@ def testRequestStaticValidation(tconf, looper, txnPoolNodeSet, node.doStaticValidation(req) -def test3PCOverBatchWithThresholdReqs(tconf, looper, txnPoolNodeSet, client, +def test3PCOverBatchWithThresholdReqs(tconf, looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle): """ Check that 3 phase commit happens when threshold number of requests are diff --git a/plenum/test/primary_election/test_primary_election_case6.py b/plenum/test/primary_election/test_primary_election_case6.py index 258100142d..f658b9774b 100644 --- a/plenum/test/primary_election/test_primary_election_case6.py +++ b/plenum/test/primary_election/test_primary_election_case6.py @@ -9,8 +9,8 @@ @pytest.fixture(scope='module') -def case_6_setup(startedNodes): - A, B, C, D = startedNodes.nodes.values() +def case_6_setup(txnPoolNodeSet): + A, B, C, D = txnPoolNodeSet # A will get Nomination, Primary, Reelection from after elections get over for m in (Nomination, Primary, Reelection): @@ -22,11 +22,10 @@ def case_6_setup(startedNodes): # noinspection PyIncorrectDocstring @pytest.fixture(scope='module') -def elections_done(case_6_setup, looper, keySharedNodes): +def elections_done(case_6_setup, looper, txnPoolNodeSet): # Make sure elections are done successfully - nodeSet = keySharedNodes - A, B, C, D = nodeSet.nodes.values() - looper.run(checkNodesConnected(nodeSet)) + A, B, C, D = txnPoolNodeSet + looper.run(checkNodesConnected(txnPoolNodeSet)) inst_ids = (0, 1) @@ -48,7 +47,7 @@ def chk(): assert primary_send_times[i][0] > max(primary_recv_times[i]) looper.run(eventually(chk, retryWait=1, timeout=15)) - checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1) + checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1) # Make sure no Nominations or Primary are received by A from B for i in inst_ids: diff --git a/plenum/test/primary_selection/test_primary_selection_routes.py b/plenum/test/primary_selection/test_primary_selection_routes.py index 1c2d348864..2df9723e87 100644 --- a/plenum/test/primary_selection/test_primary_selection_routes.py +++ b/plenum/test/primary_selection/test_primary_selection_routes.py @@ -3,6 +3,6 @@ nodeCount = 7 -def test_routes(nodeSet, up): +def test_routes(txnPoolNodeSet): # TODO: Low priority. pass diff --git a/plenum/test/test_connections_with_converted_key.py b/plenum/test/test_connections_with_converted_key.py index f572ec74b7..1766f05e95 100644 --- a/plenum/test/test_connections_with_converted_key.py +++ b/plenum/test/test_connections_with_converted_key.py @@ -3,8 +3,8 @@ from stp_core.crypto.util import ed25519SkToCurve25519, ed25519PkToCurve25519 -def testNodesConnectedUsingConvertedKeys(nodeSet, up): - for node in nodeSet: +def testNodesConnectedUsingConvertedKeys(txnPoolNodeSet): + for node in txnPoolNodeSet: secretKey = ed25519SkToCurve25519(node.nodestack.keyhex) publicKey = ed25519PkToCurve25519(node.nodestack.verhex) assert unhexlify(node.nodestack.prihex) == secretKey @@ -16,7 +16,7 @@ def testNodesConnectedUsingConvertedKeys(nodeSet, up): assert unhexlify(node.clientstack.pubhex) == publicKey -def testClientConnectedUsingConvertedKeys(nodeSet, up, client1, replied1): +def testClientConnectedUsingConvertedKeys(txnPoolNodeSet, client1, replied1): secretKey = ed25519SkToCurve25519(client1.nodestack.keyhex) publicKey = ed25519PkToCurve25519(client1.nodestack.verhex) assert unhexlify(client1.nodestack.prihex) == secretKey diff --git a/plenum/test/test_dirty_read.py b/plenum/test/test_dirty_read.py index 9948924bc7..e386ae0de1 100644 --- a/plenum/test/test_dirty_read.py +++ b/plenum/test/test_dirty_read.py @@ -22,13 +22,13 @@ async def serviceReplicas(limit): node.serviceReplicas = serviceReplicas -def test_dirty_read(looper, nodeSet, client1, wallet1): +def test_dirty_read(looper, txnPoolNodeSet, client1, wallet1): """ Tests the case when read request comes before write request is not executed on some nodes """ - slow_nodes = list(nodeSet)[2:4] + slow_nodes = list(txnPoolNodeSet)[2:4] for node in slow_nodes: logger.debug("Making node {} slow".format(node)) make_node_slow(node) diff --git a/plenum/test/test_node_basic.py b/plenum/test/test_node_basic.py index 4b812e4a03..6b9f177979 100644 --- a/plenum/test/test_node_basic.py +++ b/plenum/test/test_node_basic.py @@ -13,12 +13,12 @@ @pytest.fixture(scope="module") -def pool(looper, nodeSet): +def pool(looper, txnPoolNodeSet): # for n in nodeSet: # type: TestNode # n.startKeySharing() - looper.run(checkNodesConnected(nodeSet)) - checkProtocolInstanceSetup(looper, nodeSet) - return adict(looper=looper, nodeset=nodeSet) + looper.run(checkNodesConnected(txnPoolNodeSet)) + checkProtocolInstanceSetup(looper, txnPoolNodeSet) + return adict(looper=looper, nodeset=txnPoolNodeSet) def testConnectNodes(pool): diff --git a/plenum/test/test_node_request.py b/plenum/test/test_node_request.py index cbd85c676d..75938e78cb 100644 --- a/plenum/test/test_node_request.py +++ b/plenum/test/test_node_request.py @@ -134,8 +134,9 @@ async def checkIfPropagateRecvdFromNode(recvrNode: TestNode, assert senderNode.name in recvrNode.requests[key].propagates -def testClientSendingSameRequestAgainBeforeFirstIsProcessed(looper, nodeSet, - up, wallet1, +def testClientSendingSameRequestAgainBeforeFirstIsProcessed(looper, + txnPoolNodeSet, + wallet1, client1): size = len(client1.inBox) req = sendRandomRequest(wallet1, client1) diff --git a/plenum/test/test_request_executed_once_and_without_failing_behind.py b/plenum/test/test_request_executed_once_and_without_failing_behind.py index 6cb7304567..083b7191de 100644 --- a/plenum/test/test_request_executed_once_and_without_failing_behind.py +++ b/plenum/test/test_request_executed_once_and_without_failing_behind.py @@ -19,7 +19,8 @@ def send(msg, stat=None): def test_request_executed_once_and_without_failing_behind(tconf, looper, - nodeSet, client1, + txnPoolNodeSet, + client1, wallet1): """ Checks that all requests executed only once and without failing behind in diff --git a/plenum/test/test_round_trip_with_one_faulty_node.py b/plenum/test/test_round_trip_with_one_faulty_node.py index 0d8dcf18d4..2022896961 100644 --- a/plenum/test/test_round_trip_with_one_faulty_node.py +++ b/plenum/test/test_round_trip_with_one_faulty_node.py @@ -13,13 +13,12 @@ # noinspection PyIncorrectDocstring @pytest.fixture("module") -def node_doesnt_propagate(startedNodes): +def node_doesnt_propagate(txnPoolNodeSet): """ Makes the node named Alpha in the given set of nodes faulty. After applying this behavior, the node Alpha no longer sends propagate requests. """ - nodes = startedNodes def evilProcessPropagate(self, msg, frm): logger.info("TEST: Evil {} is not processing PROPAGATE".format(self)) @@ -29,7 +28,7 @@ def evilPropagateRequest(self, request, clientName): format(self)) # Choosing a node which will not be primary - node = nodes.Delta + node = txnPoolNodeSet[3] epp = types.MethodType(evilProcessPropagate, node) node.nodeMsgRouter.routes[Propagate] = epp node.processPropagate = epp diff --git a/plenum/test/view_change/conftest.py b/plenum/test/view_change/conftest.py index cc77cdbae5..1680317212 100644 --- a/plenum/test/view_change/conftest.py +++ b/plenum/test/view_change/conftest.py @@ -4,9 +4,9 @@ @pytest.fixture() -def viewNo(nodeSet): +def viewNo(txnPoolNodeSet): viewNos = set() - for n in nodeSet: + for n in txnPoolNodeSet: viewNos.add(n.viewNo) assert len(viewNos) == 1 return viewNos.pop() diff --git a/plenum/test/view_change/test_view_change.py b/plenum/test/view_change/test_view_change.py index 92cbc6c793..d38a543ef4 100644 --- a/plenum/test/view_change/test_view_change.py +++ b/plenum/test/view_change/test_view_change.py @@ -8,49 +8,49 @@ # noinspection PyIncorrectDocstring -def test_view_change_on_empty_ledger(nodeSet, up, looper): +def test_view_change_on_empty_ledger(txnPoolNodeSet, looper): """ Check that view change is done when no txns in the ldegr """ - ensure_view_change(looper, nodeSet) - ensureElectionsDone(looper=looper, nodes=nodeSet) - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) + ensure_view_change(looper, txnPoolNodeSet) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # noinspection PyIncorrectDocstring -def test_view_change_after_some_txns(looper, nodeSet, up, viewNo, +def test_view_change_after_some_txns(looper, txnPoolNodeSet, viewNo, wallet1, client1): """ Check that view change is done after processing some of txns """ sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) - ensure_view_change(looper, nodeSet) - ensureElectionsDone(looper=looper, nodes=nodeSet) - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) + ensure_view_change(looper, txnPoolNodeSet) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # noinspection PyIncorrectDocstring -def test_send_more_after_view_change(looper, nodeSet, up, +def test_send_more_after_view_change(looper, txnPoolNodeSet, wallet1, client1): """ Check that we can send more requests after view change """ sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) - ensure_view_change(looper, nodeSet) - ensureElectionsDone(looper=looper, nodes=nodeSet) - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) + ensure_view_change(looper, txnPoolNodeSet) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 10) -def test_node_notified_about_primary_election_result(nodeSet, looper, up): +def test_node_notified_about_primary_election_result(txnPoolNodeSet, looper): old_counts = {node.name: get_count( - node, node.primary_selected) for node in nodeSet} - ensure_view_change(looper, nodeSet) - ensureElectionsDone(looper=looper, nodes=nodeSet) - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) + node, node.primary_selected) for node in txnPoolNodeSet} + ensure_view_change(looper, txnPoolNodeSet) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) - for node in nodeSet: + for node in txnPoolNodeSet: assert get_count(node, node.primary_selected) > old_counts[node.name] diff --git a/plenum/test/view_change/test_view_change_start_without_primary.py b/plenum/test/view_change/test_view_change_start_without_primary.py index 49d410a02e..a95c58a6f9 100644 --- a/plenum/test/view_change/test_view_change_start_without_primary.py +++ b/plenum/test/view_change/test_view_change_start_without_primary.py @@ -8,17 +8,17 @@ view_change_timeout = 10 -def test_view_change_without_primary(nodeSet, looper, +def test_view_change_without_primary(txnPoolNodeSet, looper, patched_view_change_timeout): - first, others = stop_nodes_and_remove_first(looper, nodeSet) + first, others = stop_nodes_and_remove_first(looper, txnPoolNodeSet) start_and_connect_nodes(looper, others) - timeout = waits.expectedPoolElectionTimeout(len(nodeSet)) + patched_view_change_timeout + timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + patched_view_change_timeout checkProtocolInstanceSetup(looper=looper, nodes=others, retryWait=1, customTimeout=timeout, - numInstances=getRequiredInstances(len(nodeSet))) + numInstances=getRequiredInstances(len(txnPoolNodeSet))) def stop_nodes_and_remove_first(looper, nodes): @@ -37,10 +37,10 @@ def start_and_connect_nodes(looper, nodes): @pytest.fixture(scope='function') -def patched_view_change_timeout(nodeSet): - old_view_change_timeout = nodeSet[0]._view_change_timeout - for node in nodeSet: +def patched_view_change_timeout(txnPoolNodeSet): + old_view_change_timeout = txnPoolNodeSet[0]._view_change_timeout + for node in txnPoolNodeSet: node._view_change_timeout = view_change_timeout yield view_change_timeout - for node in nodeSet: + for node in txnPoolNodeSet: node._view_change_timeout = old_view_change_timeout diff --git a/plenum/test/view_change/test_view_change_timeout.py b/plenum/test/view_change/test_view_change_timeout.py index 4837ad12f3..cd69d952ad 100644 --- a/plenum/test/view_change/test_view_change_timeout.py +++ b/plenum/test/view_change/test_view_change_timeout.py @@ -14,15 +14,15 @@ @pytest.fixture() -def setup(nodeSet, looper): - m_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) - initial_view_no = waitForViewChange(looper, nodeSet) +def setup(txnPoolNodeSet, looper): + m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) + initial_view_no = waitForViewChange(looper, txnPoolNodeSet) # Setting view change timeout to low value to make test pass quicker - for node in nodeSet: + for node in txnPoolNodeSet: node._view_change_timeout = view_change_timeout timeout_callback_stats = {} - for node in nodeSet: + for node in txnPoolNodeSet: timeout_callback_stats[node.name] = { 'called': get_count(node, node._check_view_change_completed), 'returned_true': len(getAllReturnVals( @@ -32,33 +32,33 @@ def setup(nodeSet, looper): def test_view_change_retry_by_timeout( - nodeSet, looper, up, setup, wallet1, client1): + txnPoolNodeSet, looper, setup, wallet1, client1): """ Verifies that a view change is restarted if it is not completed in time """ m_primary_node, initial_view_no, timeout_callback_stats = setup - delay_view_change_done_msg(nodeSet) + delay_view_change_done_msg(txnPoolNodeSet) - start_view_change(nodeSet, initial_view_no + 1) + start_view_change(txnPoolNodeSet, initial_view_no + 1) # First view change should fail, because of delayed ViewChangeDone # messages. This then leads to new view change that we need. with pytest.raises(AssertionError): ensureElectionsDone(looper=looper, - nodes=nodeSet, + nodes=txnPoolNodeSet, customTimeout=view_change_timeout + 2) # Resetting delays to let second view change go well - reset_delays_and_process_delayeds(nodeSet) + reset_delays_and_process_delayeds(txnPoolNodeSet) # This view change should be completed with no problems - ensureElectionsDone(looper=looper, nodes=nodeSet) - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) - new_m_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) + new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) assert m_primary_node.name != new_m_primary_node.name # The timeout method was called one time - for node in nodeSet: + for node in txnPoolNodeSet: assert get_count(node, node._check_view_change_completed) - \ timeout_callback_stats[node.name]['called'] == 1 @@ -68,26 +68,26 @@ def test_view_change_retry_by_timeout( timeout_callback_stats[node.name]['returned_true'] == 1 # 2 view changes have been initiated - for node in nodeSet: + for node in txnPoolNodeSet: assert node.viewNo - initial_view_no == 2 - ensure_pool_functional(looper, nodeSet, wallet1, client1) + ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1) def test_multiple_view_change_retries_by_timeouts( - nodeSet, looper, up, setup, wallet1, client1): + txnPoolNodeSet, looper, setup, wallet1, client1): """ Verifies that a view change is restarted each time when the previous one is timed out """ _, initial_view_no, timeout_callback_stats = setup - delay_view_change_done_msg(nodeSet) + delay_view_change_done_msg(txnPoolNodeSet) - start_view_change(nodeSet, initial_view_no + 1) + start_view_change(txnPoolNodeSet, initial_view_no + 1) def check_timeout_callback_called(times): - for node in nodeSet: + for node in txnPoolNodeSet: assert get_count(node, node._check_view_change_completed) - \ timeout_callback_stats[node.name]['called'] == times @@ -104,33 +104,33 @@ def check_timeout_callback_called(times): # Check that the last view change has failed with pytest.raises(AssertionError): - ensureElectionsDone(looper=looper, nodes=nodeSet, customTimeout=1) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=1) # Reset delays to let the next view change go well - reset_delays_and_process_delayeds(nodeSet) + reset_delays_and_process_delayeds(txnPoolNodeSet) # This view change must be completed with no problems - ensureElectionsDone(looper=looper, nodes=nodeSet) - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # 4 view changes must have been initiated (initial one + 3 retries) - for node in nodeSet: + for node in txnPoolNodeSet: assert node.viewNo - initial_view_no == 4 - ensure_pool_functional(looper, nodeSet, wallet1, client1) + ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1) def test_view_change_restarted_by_timeout_if_next_primary_disconnected( - nodeSet, looper, up, setup, wallet1, client1): + txnPoolNodeSet, looper, setup, wallet1, client1): """ Verifies that a view change is restarted by timeout if the next primary has been disconnected """ _, initial_view_no, timeout_callback_stats = setup - start_view_change(nodeSet, initial_view_no + 1) + start_view_change(txnPoolNodeSet, initial_view_no + 1) - alive_nodes = stop_next_primary(nodeSet) + alive_nodes = stop_next_primary(txnPoolNodeSet) ensureElectionsDone(looper=looper, nodes=alive_nodes, numInstances=3) @@ -151,7 +151,7 @@ def test_view_change_restarted_by_timeout_if_next_primary_disconnected( def stop_next_primary(nodes): m_next_primary_name = nodes[0]._elector._next_primary_node_name_for_master() - nodes[m_next_primary_name].stop() + next(node for node in nodes if node.name == m_next_primary_name).stop() alive_nodes = list(filter(lambda x: x.name != m_next_primary_name, nodes)) return alive_nodes From 5ec36c51099ff64e7d92965b35e458fea35bc75d Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 19 Mar 2018 18:25:46 +0300 Subject: [PATCH 19/73] Remove pf TestNodeSet. Change of helper methods. Part 1. Signed-off-by: ArtObr --- plenum/test/helper.py | 64 ++++++------ ...come_active_with_less_than_four_servers.py | 99 +++++++++---------- plenum/test/test_node_basic.py | 9 +- ...xecuted_once_and_without_failing_behind.py | 4 +- 4 files changed, 84 insertions(+), 92 deletions(-) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 6b07d98ab5..77dfee9593 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -32,7 +32,8 @@ from plenum.server.node import Node from plenum.test import waits from plenum.test.msgs import randomMsg -from plenum.test.spy_helpers import getLastClientReqReceivedForNode, getAllArgs, getAllReturnVals +from plenum.test.spy_helpers import getLastClientReqReceivedForNode, getAllArgs, getAllReturnVals, \ + getAllMsgReceivedForNode from plenum.test.test_client import TestClient, genTestClient from plenum.test.test_node import TestNode, TestReplica, TestNodeSet, \ checkNodesConnected, ensureElectionsDone, NodeRef, getPrimaryReplica @@ -331,83 +332,78 @@ def buildCompletedTxnFromReply(request, reply: Reply) -> Dict: return txn -async def msgAll(nodes: TestNodeSet): +async def msgAll(nodes): # test sending messages from every node to every other node # TODO split send and check so that the messages can be sent concurrently - for p in permutations(nodes.nodeNames, 2): - await sendMessageAndCheckDelivery(nodes, p[0], p[1]) + for p in permutations(nodes, 2): + await sendMessageAndCheckDelivery(p[0], p[1]) -def sendMessage(nodes: TestNodeSet, - frm: NodeRef, - to: NodeRef, +def sendMessage(sender: Node, + reciever: Node, msg: Optional[Tuple] = None): """ Sends message from one node to another :param nodes: - :param frm: sender - :param to: recepient + :param sender: sender + :param reciever: recepient :param msg: optional message - by default random one generated :return: """ - logger.debug("Sending msg from {} to {}".format(frm, to)) + logger.debug("Sending msg from {} to {}".format(sender.name, reciever.name)) msg = msg if msg else randomMsg() - sender = nodes.getNode(frm) - rid = sender.nodestack.getRemote(nodes.getNodeName(to)).uid + rid = sender.nodestack.getRemote(reciever.name).uid sender.nodestack.send(msg, rid) -async def sendMessageAndCheckDelivery(nodes: TestNodeSet, - frm: NodeRef, - to: NodeRef, +async def sendMessageAndCheckDelivery(sender: Node, + reciever: Node, msg: Optional[Tuple] = None, method=None, customTimeout=None): """ Sends message from one node to another and checks that it was delivered - :param nodes: - :param frm: sender - :param to: recepient + :param sender: sender + :param reciever: recepient :param msg: optional message - by default random one generated :param customTimeout: :return: """ - logger.debug("Sending msg from {} to {}".format(frm, to)) + logger.debug("Sending msg from {} to {}".format(sender.name, reciever.name)) msg = msg if msg else randomMsg() - sender = nodes.getNode(frm) - rid = sender.nodestack.getRemote(nodes.getNodeName(to)).uid + rid = sender.nodestack.getRemote(reciever.name).uid sender.nodestack.send(msg, rid) timeout = customTimeout or waits.expectedNodeToNodeMessageDeliveryTime() - await eventually(checkMessageReceived, msg, nodes, to, method, + await eventually(checkMessageReceived, msg, reciever, method, retryWait=.1, timeout=timeout, ratchetSteps=10) -def sendMessageToAll(nodes: TestNodeSet, - frm: NodeRef, +def sendMessageToAll(nodes, + sender: Node, msg: Optional[Tuple] = None): """ Sends message from one node to all others :param nodes: - :param frm: sender + :param sender: sender :param msg: optional message - by default random one generated :return: """ for node in nodes: - if node != frm: - sendMessage(nodes, frm, node, msg) + if node != sender: + sendMessage(sender, node, msg) -async def sendMessageAndCheckDeliveryToAll(nodes: TestNodeSet, - frm: NodeRef, +async def sendMessageAndCheckDeliveryToAll(nodes, + sender: Node, msg: Optional[Tuple] = None, method=None, customTimeout=None): @@ -415,7 +411,7 @@ async def sendMessageAndCheckDeliveryToAll(nodes: TestNodeSet, Sends message from one node to all other and checks that it was delivered :param nodes: - :param frm: sender + :param sender: sender :param msg: optional message - by default random one generated :param customTimeout: :return: @@ -423,13 +419,13 @@ async def sendMessageAndCheckDeliveryToAll(nodes: TestNodeSet, customTimeout = customTimeout or waits.expectedNodeToAllNodesMessageDeliveryTime( len(nodes)) for node in nodes: - if node != frm: - await sendMessageAndCheckDelivery(nodes, frm, node, msg, method, customTimeout) + if node != sender: + await sendMessageAndCheckDelivery(sender, node, msg, method, customTimeout) break -def checkMessageReceived(msg, nodes, to, method: str = None): - allMsgs = nodes.getAllMsgReceived(to, method) +def checkMessageReceived(msg, receiver, method: str = None): + allMsgs = getAllMsgReceivedForNode(receiver, method) assert msg in allMsgs diff --git a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py index 719e93536f..38e4eb4411 100644 --- a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py +++ b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py @@ -22,7 +22,7 @@ def limitTestRunningTime(): # noinspection PyIncorrectDocstring def testProtocolInstanceCannotBecomeActiveWithLessThanFourServers( - tconf_for_func, tdir_for_func): + looper, txnPoolNodeSet): """ A protocol instance must have at least 4 nodes to come up. The status of the nodes will change from starting to started only after the @@ -33,54 +33,51 @@ def testProtocolInstanceCannotBecomeActiveWithLessThanFourServers( minimumNodesToBeUp = nodeCount - f nodeNames = genNodeNames(nodeCount) - with TestNodeSet(tconf_for_func, names=nodeNames, tmpdir=tdir_for_func) as nodeSet: - with Looper(nodeSet) as looper: - # helpers - - def genExpectedStates(connecteds: Iterable[str]): - return { - nn: CONNECTED if nn in connecteds else JOINED_NOT_ALLOWED - for nn in nodeNames} - - def checkNodeStatusRemotesAndF(expectedStatus: Status, - nodeIdx: int): - for node in nodeSet.nodes.values(): - checkNodeRemotes(node, - genExpectedStates(nodeNames[:nodeIdx + 1])) - assert node.status == expectedStatus - - def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status): - logger.info("Add back the {} node and see status of {}". - format(ordinal(nodeIdx + 1), expectedStatus)) - addNodeBack(nodeSet, looper, nodeNames[nodeIdx]) - - timeout = waits.expectedNodeStartUpTimeout() + \ - waits.expectedPoolInterconnectionTime(len(nodeSet)) - # TODO: Probably it's better to modify waits.* functions - timeout *= 1.5 - looper.run(eventually(checkNodeStatusRemotesAndF, - expectedStatus, - nodeIdx, - retryWait=1, timeout=timeout)) - - logger.debug("Sharing keys") - looper.run(checkNodesConnected(nodeSet)) - - logger.debug("Remove all the nodes") - for n in nodeNames: - looper.removeProdable(nodeSet.nodes[n]) - nodeSet.removeNode(n) - - looper.runFor(10) - - logger.debug("Add nodes back one at a time") - for i in range(nodeCount): - nodes = i + 1 - if nodes < minimumNodesToBeUp: - expectedStatus = Status.starting - elif nodes < nodeCount: - expectedStatus = Status.started_hungry - else: - expectedStatus = Status.started - addNodeBackAndCheck(i, expectedStatus) + # helpers + def genExpectedStates(connecteds: Iterable[str]): + return { + nn: CONNECTED if nn in connecteds else JOINED_NOT_ALLOWED + for nn in nodeNames} + + def checkNodeStatusRemotesAndF(expectedStatus: Status, + nodeIdx: int): + for node in txnPoolNodeSet: + checkNodeRemotes(node, + genExpectedStates(nodeNames[:nodeIdx + 1])) + assert node.status == expectedStatus + + def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status): + logger.info("Add back the {} node and see status of {}". + format(ordinal(nodeIdx + 1), expectedStatus)) + addNodeBack(txnPoolNodeSet, looper, nodeNames[nodeIdx]) + + timeout = waits.expectedNodeStartUpTimeout() + \ + waits.expectedPoolInterconnectionTime(len(txnPoolNodeSet)) + # TODO: Probably it's better to modify waits.* functions + timeout *= 1.5 + looper.run(eventually(checkNodeStatusRemotesAndF, + expectedStatus, + nodeIdx, + retryWait=1, timeout=timeout)) + + logger.debug("Sharing keys") + looper.run(checkNodesConnected(txnPoolNodeSet)) + + logger.debug("Remove all the nodes") + for n in nodeNames: + looper.removeProdable(txnPoolNodeSet[n]) + txnPoolNodeSet.remove(n) + + looper.runFor(10) + + logger.debug("Add nodes back one at a time") + for i in range(nodeCount): + nodes = i + 1 + if nodes < minimumNodesToBeUp: + expectedStatus = Status.starting + elif nodes < nodeCount: + expectedStatus = Status.started_hungry + else: + expectedStatus = Status.started + addNodeBackAndCheck(i, expectedStatus) diff --git a/plenum/test/test_node_basic.py b/plenum/test/test_node_basic.py index 6b9f177979..c35317f520 100644 --- a/plenum/test/test_node_basic.py +++ b/plenum/test/test_node_basic.py @@ -29,14 +29,13 @@ def testAllBroadcast(pool): pool.looper.run(msgAll(pool.nodeset)) -def testMsgSendingTime(pool, nodeReg): - nodeNames = list(nodeReg.keys()) +def testMsgSendingTime(pool): + nodes = pool.nodeset msg = randomMsg() timeout = waits.expectedNodeStartUpTimeout() pool.looper.run( - sendMessageAndCheckDelivery(pool.nodeset, - nodeNames[0], - nodeNames[1], + sendMessageAndCheckDelivery(nodes[0], + nodes[1], msg, customTimeout=timeout)) diff --git a/plenum/test/test_request_executed_once_and_without_failing_behind.py b/plenum/test/test_request_executed_once_and_without_failing_behind.py index 083b7191de..3851362bb3 100644 --- a/plenum/test/test_request_executed_once_and_without_failing_behind.py +++ b/plenum/test/test_request_executed_once_and_without_failing_behind.py @@ -37,7 +37,7 @@ def test_request_executed_once_and_without_failing_behind(tconf, looper, number_of_requests = 5 tconf.CHK_FREQ = 1 - for node in nodeSet: + for node in txnPoolNodeSet: for replica in node.replicas: set_checkpoint_faking(replica) @@ -51,7 +51,7 @@ def test_request_executed_once_and_without_failing_behind(tconf, looper, requests=[request]) expected = [request.reqId for request in requests] - for node in nodeSet: + for node in txnPoolNodeSet: real_ledger_state = [txn[1]["reqId"] for txn in node.getLedger(DOMAIN_LEDGER_ID).getAllTxn()] assert expected == real_ledger_state From e6cbc108e9a578a3d9463377ab8cff5b41e4a281 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 20 Mar 2018 12:32:57 +0300 Subject: [PATCH 20/73] Remove TestNodeSet from helper. part 2 Signed-off-by: ArtObr --- plenum/test/helper.py | 26 ++++++++++---- ...come_active_with_less_than_four_servers.py | 35 +++++++++++-------- .../test_primary_election_case1.py | 30 +++++++--------- .../test_primary_election_case2.py | 17 +++++---- .../test_primary_election_case4.py | 6 ++-- .../test_primary_election_case5.py | 19 +++++----- .../test_primary_election_contested.py | 15 ++++---- ...test_primary_election_with_clear_winner.py | 20 +++++------ .../test_primary_election_with_tie.py | 6 ++-- 9 files changed, 92 insertions(+), 82 deletions(-) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 77dfee9593..20731c3e02 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -8,14 +8,14 @@ from shutil import copyfile from sys import executable from time import sleep -from typing import Tuple, Iterable, Dict, Optional, NamedTuple, List, Any, Sequence, Union +from typing import Tuple, Iterable, Dict, Optional, List, Any, Sequence, Union import pytest from psutil import Popen import json import asyncio -from indy.ledger import sign_and_submit_request, sign_request, submit_request, build_nym_request +from indy.ledger import sign_and_submit_request, sign_request, submit_request from indy.error import ErrorCode, IndyError from ledger.genesis_txn.genesis_txn_file_util import genesis_txn_file @@ -28,6 +28,7 @@ from plenum.common.messages.node_messages import Reply, PrePrepare, Prepare, Commit from plenum.common.types import f from plenum.common.util import getNoInstances, get_utc_epoch +from plenum.common.config_helper import PNodeConfigHelper from plenum.common.request import Request from plenum.server.node import Node from plenum.test import waits @@ -429,12 +430,23 @@ def checkMessageReceived(msg, receiver, method: str = None): assert msg in allMsgs -def addNodeBack(nodeSet: TestNodeSet, +def addNodeBack(node_set, looper: Looper, - nodeName: str) -> TestNode: - node = nodeSet.addNode(nodeName) - looper.add(node) - return node + node: Node, + tconf, + tdir) -> TestNode: + config_helper = PNodeConfigHelper(node.name, tconf, chroot=tdir) + restartedNode = TestNode(node.name, + config_helper=config_helper, + config=tconf, + ha=node.nodestack.ha, + cliha=node.clientstack.ha) + for node in node_set: + if node.name != restartedNode.name: + node.nodestack.reconnectRemoteWithName(restartedNode.name) + node_set.append(restartedNode) + looper.add(restartedNode) + return restartedNode def checkPropagateReqCountOfNode(node: TestNode, identifier: str, reqId: int): diff --git a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py index 38e4eb4411..c521c924c0 100644 --- a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py +++ b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py @@ -1,19 +1,24 @@ from typing import Iterable import pytest + +from plenum.test.node_request.helper import get_node_by_name from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger -from stp_core.loop.looper import Looper from plenum.common.startable import Status from plenum.test.greek import genNodeNames from plenum.test.helper import addNodeBack, ordinal -from plenum.test.test_node import TestNodeSet, checkNodesConnected, \ +from plenum.test.test_node import checkNodesConnected, \ checkNodeRemotes from plenum.test.test_stack import CONNECTED, JOINED_NOT_ALLOWED from plenum.test import waits logger = getlogger() +nodeCount = 13 +f = 4 +minimumNodesToBeUp = nodeCount - f + @pytest.fixture(scope="function", autouse=True) def limitTestRunningTime(): @@ -22,19 +27,16 @@ def limitTestRunningTime(): # noinspection PyIncorrectDocstring def testProtocolInstanceCannotBecomeActiveWithLessThanFourServers( - looper, txnPoolNodeSet): + txnPoolNodeSet, looper, tconf, tdir): """ A protocol instance must have at least 4 nodes to come up. The status of the nodes will change from starting to started only after the addition of the fourth node to the system. """ - nodeCount = 13 - f = 4 - minimumNodesToBeUp = nodeCount - f nodeNames = genNodeNames(nodeCount) + current_node_set = list(txnPoolNodeSet) - # helpers def genExpectedStates(connecteds: Iterable[str]): return { nn: CONNECTED if nn in connecteds else JOINED_NOT_ALLOWED @@ -42,7 +44,7 @@ def genExpectedStates(connecteds: Iterable[str]): def checkNodeStatusRemotesAndF(expectedStatus: Status, nodeIdx: int): - for node in txnPoolNodeSet: + for node in current_node_set: checkNodeRemotes(node, genExpectedStates(nodeNames[:nodeIdx + 1])) assert node.status == expectedStatus @@ -50,10 +52,13 @@ def checkNodeStatusRemotesAndF(expectedStatus: Status, def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status): logger.info("Add back the {} node and see status of {}". format(ordinal(nodeIdx + 1), expectedStatus)) - addNodeBack(txnPoolNodeSet, looper, nodeNames[nodeIdx]) - + addNodeBack( + current_node_set, looper, + get_node_by_name(txnPoolNodeSet, nodeNames[nodeIdx]), + tconf, tdir) + looper.run(checkNodesConnected(current_node_set)) timeout = waits.expectedNodeStartUpTimeout() + \ - waits.expectedPoolInterconnectionTime(len(txnPoolNodeSet)) + waits.expectedPoolInterconnectionTime(len(current_node_set)) # TODO: Probably it's better to modify waits.* functions timeout *= 1.5 looper.run(eventually(checkNodeStatusRemotesAndF, @@ -62,12 +67,14 @@ def addNodeBackAndCheck(nodeIdx: int, expectedStatus: Status): retryWait=1, timeout=timeout)) logger.debug("Sharing keys") - looper.run(checkNodesConnected(txnPoolNodeSet)) + looper.run(checkNodesConnected(current_node_set)) logger.debug("Remove all the nodes") for n in nodeNames: - looper.removeProdable(txnPoolNodeSet[n]) - txnPoolNodeSet.remove(n) + node_n = get_node_by_name(current_node_set, n) + looper.removeProdable(node_n) + node_n.stop() + current_node_set.remove(node_n) looper.runFor(10) diff --git a/plenum/test/primary_election/test_primary_election_case1.py b/plenum/test/primary_election/test_primary_election_case1.py index bb410b57a4..ca9671bc97 100644 --- a/plenum/test/primary_election/test_primary_election_case1.py +++ b/plenum/test/primary_election/test_primary_election_case1.py @@ -9,7 +9,7 @@ from plenum.test.helper import whitelistNode from plenum.test.primary_election.helpers import checkNomination, \ getSelfNominationByNode, nominationByNode -from plenum.test.test_node import TestNodeSet, checkNodesConnected, \ +from plenum.test.test_node import checkNodesConnected, \ ensureElectionsDone from plenum.test import waits @@ -23,11 +23,8 @@ @pytest.fixture() -def case1Setup(startedNodes: TestNodeSet): - nodes = startedNodes - nodeNames = nodes.nodeNames - - nodeB = nodes.getNode(nodeNames[1]) +def case1Setup(txnPoolNodeSet): + nodeB = txnPoolNodeSet[1] # Node B delays self nomination so A's nomination reaches everyone nodeB.delaySelfNomination(10) @@ -35,8 +32,8 @@ def case1Setup(startedNodes: TestNodeSet): nodeB.nodeIbStasher.delay(delayerMsgTuple(delayOfNomination, Nomination)) # Add node C and node D - nodeC = nodes.getNode(nodeNames[2]) - nodeD = nodes.getNode(nodeNames[3]) + nodeC = txnPoolNodeSet[2] + nodeD = txnPoolNodeSet[3] # Node C should not try to nominate itself until it gets NOMINATE from B nodeC.delaySelfNomination(10) @@ -50,30 +47,29 @@ def case1Setup(startedNodes: TestNodeSet): # multiple nominations from the same node have any impact on # the election whitelistNode(nodeB.name, - [node for node in nodes if node != nodeB], + [node for node in txnPoolNodeSet if node != nodeB], Suspicions.DUPLICATE_NOM_SENT.code) - return nodes + return txnPoolNodeSet # noinspection PyIncorrectDocstring @pytest.mark.skip('Nodes use round robin primary selection') -def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes): +def testPrimaryElectionCase1(case1Setup, looper, txnPoolNodeSet): """ Case 1 - A node making multiple nominations for a particular node. Consider 4 nodes A, B, C and D. Lets say node B is malicious and is repeatedly nominating Node D """ - nodes = keySharedNodes - nodeA, nodeB, nodeC, nodeD = [nodes.getNode(nm) for nm in nodes.nodeNames] + nodeA, nodeB, nodeC, nodeD = txnPoolNodeSet # Doesn't matter if nodes reach the ready state or not. Just start them - looper.run(checkNodesConnected(nodes)) + looper.run(checkNodesConnected(txnPoolNodeSet)) # Node B sends multiple NOMINATE messages for Node D but only after A has # nominated itself timeout = waits.expectedPoolNominationTimeout( - nodeCount=len(keySharedNodes)) + nodeCount=len(txnPoolNodeSet)) looper.run(eventually(checkNomination, nodeA, nodeA.name, retryWait=.25, timeout=timeout)) @@ -95,9 +91,9 @@ def testPrimaryElectionCase1(case1Setup, looper, keySharedNodes): timeout = waits.expectedPoolElectionTimeout(nodeCount) + delayOfNomination primaryReplicas = ensureElectionsDone(looper=looper, - nodes=nodes, customTimeout=timeout) + nodes=txnPoolNodeSet, customTimeout=timeout) - for node in nodes: + for node in txnPoolNodeSet: logger.debug( "{}'s nominations {}".format(node, node.elector.nominations)) # Node D should not have any primary diff --git a/plenum/test/primary_election/test_primary_election_case2.py b/plenum/test/primary_election/test_primary_election_case2.py index 804ecb41b8..afe08d2df0 100644 --- a/plenum/test/primary_election/test_primary_election_case2.py +++ b/plenum/test/primary_election/test_primary_election_case2.py @@ -7,7 +7,7 @@ from plenum.test.delayers import delayerMsgTuple from plenum.test.primary_election.helpers import checkNomination, \ getSelfNominationByNode, nominationByNode -from plenum.test.test_node import TestNodeSet, checkNodesConnected, \ +from plenum.test.test_node import checkNodesConnected, \ ensureElectionsDone from plenum.test import waits @@ -17,8 +17,8 @@ @pytest.fixture() -def case2Setup(startedNodes: TestNodeSet): - A, B, C, D = startedNodes.nodes.values() +def case2Setup(txnPoolNodeSet): + A, B, C, D = txnPoolNodeSet # Node B delays self nomination so A's nomination reaches everyone B.delaySelfNomination(5) @@ -41,19 +41,18 @@ def case2Setup(startedNodes: TestNodeSet): @pytest.mark.skip('Nodes use round robin primary selection') -def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes): +def testPrimaryElectionCase2(case2Setup, looper, txnPoolNodeSet): """ Case 2 - A node making nominations for a multiple other nodes. Consider 4 nodes A, B, C, and D. Lets say node B is malicious and nominates node C to all nodes. Again node B nominates node D to all nodes. """ - nodeSet = keySharedNodes - A, B, C, D = nodeSet.nodes.values() + A, B, C, D = txnPoolNodeSet - looper.run(checkNodesConnected(nodeSet)) + looper.run(checkNodesConnected(txnPoolNodeSet)) # Node B sends multiple NOMINATE msgs but only after A has nominated itself - timeout = waits.expectedPoolNominationTimeout(len(nodeSet)) + timeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet)) looper.run(eventually(checkNomination, A, A.name, retryWait=.25, timeout=timeout)) @@ -71,7 +70,7 @@ def testPrimaryElectionCase2(case2Setup, looper, keySharedNodes): B.send(nominationByNode(DRep, B, instId)) # Ensure elections are done - ensureElectionsDone(looper=looper, nodes=nodeSet) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) # All nodes from node A, node C, node D(node B is malicious anyway so # not considering it) should have nomination for node C from node B since diff --git a/plenum/test/primary_election/test_primary_election_case4.py b/plenum/test/primary_election/test_primary_election_case4.py index 179e037d84..07b879aa84 100644 --- a/plenum/test/primary_election/test_primary_election_case4.py +++ b/plenum/test/primary_election/test_primary_election_case4.py @@ -5,7 +5,7 @@ from plenum.server.suspicion_codes import Suspicions from plenum.test import waits from plenum.test.primary_election.helpers import primaryByNode -from plenum.test.test_node import TestNodeSet, checkNodesConnected, \ +from plenum.test.test_node import checkNodesConnected, \ ensureElectionsDone nodeCount = 4 @@ -18,8 +18,8 @@ @pytest.fixture() -def case4Setup(keySharedNodes: TestNodeSet): - allNodes = keySharedNodes.nodes.values() +def case4Setup(txnPoolNodeSet): + allNodes = txnPoolNodeSet A, B, C, D = allNodes # Delay each of the nodes A, B and C's self nomination so Node B gets to diff --git a/plenum/test/primary_election/test_primary_election_case5.py b/plenum/test/primary_election/test_primary_election_case5.py index c9827a27cc..c65e31cbb4 100644 --- a/plenum/test/primary_election/test_primary_election_case5.py +++ b/plenum/test/primary_election/test_primary_election_case5.py @@ -6,7 +6,7 @@ from plenum.server.replica import Replica from plenum.server.suspicion_codes import Suspicions from plenum.test.primary_election.helpers import primaryByNode -from plenum.test.test_node import TestNodeSet, checkNodesConnected, \ +from plenum.test.test_node import checkNodesConnected, \ ensureElectionsDone from plenum.test.delayers import delayerMsgTuple @@ -23,8 +23,8 @@ @pytest.fixture() -def case5Setup(startedNodes: TestNodeSet): - A, B, C, D = startedNodes.nodes.values() +def case5Setup(txnPoolNodeSet): + A, B, C, D = txnPoolNodeSet # Node B delays self nomination so A's nomination reaches everyone B.delaySelfNomination(30) @@ -56,17 +56,16 @@ def case5Setup(startedNodes: TestNodeSet): # noinspection PyIncorrectDocstring @pytest.mark.skip('Nodes use round robin primary selection') -def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes): +def testPrimaryElectionCase5(case5Setup, looper, txnPoolNodeSet): """ Case 5 - A node making primary declarations for a multiple other nodes. Consider 4 nodes A, B, C, and D. Lets say node B is malicious and declares node C as primary to all nodes. Again node B declares node D as primary to all nodes. """ - nodeSet = keySharedNodes - A, B, C, D = nodeSet.nodes.values() + A, B, C, D = txnPoolNodeSet - looper.run(checkNodesConnected(nodeSet)) + looper.run(checkNodesConnected(txnPoolNodeSet)) BRep = Replica.generateName(B.name, 0) CRep = Replica.generateName(C.name, 0) @@ -81,10 +80,10 @@ def testPrimaryElectionCase5(case5Setup, looper, keySharedNodes): # Ensure elections are done # also have to take into account the catchup procedure - timeout = waits.expectedPoolElectionTimeout(len(nodeSet)) + \ - waits.expectedPoolCatchupTime(len(nodeSet)) + \ + timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + \ + waits.expectedPoolCatchupTime(len(txnPoolNodeSet)) + \ delayOfElectionDone - ensureElectionsDone(looper=looper, nodes=nodeSet, customTimeout=timeout) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=timeout) # All nodes from node A, node C, node D(node B is malicious anyway so not # considering it) should have primary declarations for node C from node B diff --git a/plenum/test/primary_election/test_primary_election_contested.py b/plenum/test/primary_election/test_primary_election_contested.py index 2152fce73f..9effdc827a 100644 --- a/plenum/test/primary_election/test_primary_election_contested.py +++ b/plenum/test/primary_election/test_primary_election_contested.py @@ -5,7 +5,7 @@ from plenum.common.messages.node_messages import Nomination from plenum.test.delayers import delayerMsgTuple from plenum.test.primary_election.helpers import checkNomination -from plenum.test.test_node import TestNodeSet, checkPoolReady, \ +from plenum.test.test_node import checkPoolReady, \ checkProtocolInstanceSetup from plenum.test import waits @@ -15,8 +15,8 @@ @pytest.fixture() -def electContFixture(startedNodes: TestNodeSet): - A, B, C, D = startedNodes.nodes.values() +def electContFixture(txnPoolNodeSet): + A, B, C, D = txnPoolNodeSet # Delaying nodeB' self nomination so nodeA's nomination can reach NodeC # and NodeD @@ -33,7 +33,7 @@ def electContFixture(startedNodes: TestNodeSet): # noinspection PyIncorrectDocstring @pytest.mark.skip('Nodes use round robin primary selection') -def testPrimaryElectionContested(electContFixture, looper, keySharedNodes): +def testPrimaryElectionContested(electContFixture, looper, txnPoolNodeSet): """ Primary selection (Rainy Day) A, B, C, D, E @@ -50,10 +50,9 @@ def testPrimaryElectionContested(electContFixture, looper, keySharedNodes): All see the others have sent Primary A, and then the nodes record who is the Primary. """ - nodeSet = keySharedNodes - A, B, C, D = nodeSet.nodes.values() + A, B, C, D = txnPoolNodeSet - checkPoolReady(looper, nodeSet) + checkPoolReady(looper, txnPoolNodeSet) logger.debug("Check nomination") timeout = waits.expectedPoolNominationTimeout(nodeCount) @@ -71,7 +70,7 @@ def testPrimaryElectionContested(electContFixture, looper, keySharedNodes): looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=timeout)) - checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1) + checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1) # Node D should not be primary assert not D.hasPrimary diff --git a/plenum/test/primary_election/test_primary_election_with_clear_winner.py b/plenum/test/primary_election/test_primary_election_with_clear_winner.py index f3def006cb..1a7152ef1a 100644 --- a/plenum/test/primary_election/test_primary_election_with_clear_winner.py +++ b/plenum/test/primary_election/test_primary_election_with_clear_winner.py @@ -2,7 +2,7 @@ from stp_core.loop.eventually import eventually from plenum.test.primary_election.helpers import checkNomination -from plenum.test.test_node import TestNodeSet, checkPoolReady, \ +from plenum.test.test_node import checkPoolReady, \ checkProtocolInstanceSetup from plenum.test import waits @@ -10,8 +10,8 @@ @pytest.fixture() -def electContFixture(startedNodes: TestNodeSet): - A, B, C, D = startedNodes.nodes.values() +def electContFixture(txnPoolNodeSet): + A, B, C, D = txnPoolNodeSet for node in [B, C, D]: node.delaySelfNomination(4) @@ -20,7 +20,7 @@ def electContFixture(startedNodes: TestNodeSet): # noinspection PyIncorrectDocstring @pytest.mark.skip('Nodes use round robin primary selection') def testPrimaryElectionWithAClearWinner( - electContFixture, looper, keySharedNodes): + electContFixture, looper, txnPoolNodeSet): """ Primary selection (Sunny Day) A, B, C, D, E @@ -52,23 +52,21 @@ def testPrimaryElectionWithAClearWinner( D sees at least two other PRIMARY(A) votes (3 including it's own) selects A as primary """ - - nodeSet = keySharedNodes - A, B, C, D = nodeSet.nodes.values() + A, B, C, D = txnPoolNodeSet nodesBCD = [B, C, D] - checkPoolReady(looper, nodeSet) + checkPoolReady(looper, txnPoolNodeSet) # Checking whether one of the replicas of Node A nominated itself - timeout = waits.expectedPoolNominationTimeout(len(nodeSet)) + timeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet)) looper.run(eventually(checkNomination, A, A.name, retryWait=1, timeout=timeout)) - timeout = waits.expectedPoolNominationTimeout(len(nodeSet)) + timeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet)) for n in nodesBCD: # Checking whether Node B, C and D nominated Node A looper.run(eventually(checkNomination, n, A.name, retryWait=1, timeout=timeout)) - checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1) + checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1) assert A.hasPrimary diff --git a/plenum/test/primary_election/test_primary_election_with_tie.py b/plenum/test/primary_election/test_primary_election_with_tie.py index 7124baece3..58d5f7ac64 100644 --- a/plenum/test/primary_election/test_primary_election_with_tie.py +++ b/plenum/test/primary_election/test_primary_election_with_tie.py @@ -5,7 +5,7 @@ from plenum.common.messages.node_messages import Nomination from plenum.test.delayers import delay from plenum.test.primary_election.helpers import checkNomination -from plenum.test.test_node import TestNodeSet, checkPoolReady, \ +from plenum.test.test_node import checkPoolReady, \ checkProtocolInstanceSetup from plenum.test import waits @@ -15,8 +15,8 @@ @pytest.fixture() -def electTieFixture(startedNodes: TestNodeSet): - A, B, C, D = startedNodes.nodes.values() +def electTieFixture(txnPoolNodeSet): + A, B, C, D = txnPoolNodeSet for node in [C, D]: node.delaySelfNomination(10) From 5708f72d2d465c33b99ad5304f770b5d9858efaf Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 20 Mar 2018 13:15:17 +0300 Subject: [PATCH 21/73] More tests integrated Signed-off-by: ArtObr --- .../monitoring/test_monitor_attributes.py | 4 +- .../monitoring/test_warn_unordered_log_msg.py | 24 ++++---- .../storage/test_reply_persistence_ledger.py | 4 +- plenum/test/test_delay.py | 59 +++++++++---------- plenum/test/test_verif_merkle_proof.py | 2 +- 5 files changed, 45 insertions(+), 48 deletions(-) diff --git a/plenum/test/monitoring/test_monitor_attributes.py b/plenum/test/monitoring/test_monitor_attributes.py index cd1afd65d4..73dc047b9e 100644 --- a/plenum/test/monitoring/test_monitor_attributes.py +++ b/plenum/test/monitoring/test_monitor_attributes.py @@ -1,5 +1,5 @@ -def testHasMasterPrimary(nodeSet, up): +def testHasMasterPrimary(txnPoolNodeSet): masterPrimaryCount = 0 - for node in nodeSet: + for node in txnPoolNodeSet: masterPrimaryCount += int(node.monitor.hasMasterPrimary) assert masterPrimaryCount == 1 diff --git a/plenum/test/monitoring/test_warn_unordered_log_msg.py b/plenum/test/monitoring/test_warn_unordered_log_msg.py index e4ccb93584..a1c3b3dcb1 100644 --- a/plenum/test/monitoring/test_warn_unordered_log_msg.py +++ b/plenum/test/monitoring/test_warn_unordered_log_msg.py @@ -11,34 +11,34 @@ # noinspection PyIncorrectDocstring -def test_working_has_no_warn_log_msg(looper, nodeSet, +def test_working_has_no_warn_log_msg(looper, txnPoolNodeSet, wallet1, client1, patch_monitors): - monitor = nodeSet[0].monitor - assert no_any_warn(*nodeSet) + monitor = txnPoolNodeSet[0].monitor + assert no_any_warn(*txnPoolNodeSet) for i in range(monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM): req = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[req]) looper.runFor(monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC) - assert no_any_warn(*nodeSet) + assert no_any_warn(*txnPoolNodeSet) # noinspection PyIncorrectDocstring def test_slow_node_has_warn_unordered_log_msg(looper, - nodeSet, + txnPoolNodeSet, wallet1, client1, patch_monitors): - npr = getNonPrimaryReplicas(nodeSet, 0)[0] + npr = getNonPrimaryReplicas(txnPoolNodeSet, 0)[0] slow_node = npr.node - monitor = nodeSet[0].monitor + monitor = txnPoolNodeSet[0].monitor delay = monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC * \ monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM + 10 delaysCommitProcessing(slow_node, delay=delay) - assert no_any_warn(*nodeSet), \ + assert no_any_warn(*txnPoolNodeSet), \ 'all nodes do not have warnings before test' for i in range(monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM): @@ -46,7 +46,7 @@ def test_slow_node_has_warn_unordered_log_msg(looper, waitForSufficientRepliesForRequests(looper, client1, requests=[req]) looper.runFor(monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC) - others = [node for node in nodeSet if node.name != slow_node.name] + others = [node for node in txnPoolNodeSet if node.name != slow_node.name] assert no_any_warn(*others), \ 'others do not have warning after test' assert has_some_warn(slow_node), \ @@ -90,12 +90,12 @@ def no_last_warn(*nodes): @pytest.fixture(scope="function") -def patch_monitors(nodeSet): +def patch_monitors(txnPoolNodeSet): backup = {} req_num = 3 diff_sec = 1 window_mins = 0.25 - for node in nodeSet: + for node in txnPoolNodeSet: backup[node.name] = ( node.monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM, node.monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC, @@ -105,7 +105,7 @@ def patch_monitors(nodeSet): node.monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC = diff_sec node.monitor.WARN_NOT_PARTICIPATING_WINDOW_MINS = window_mins yield req_num, diff_sec, window_mins - for node in nodeSet: + for node in txnPoolNodeSet: node.monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM = backup[node.name][0] node.monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC = backup[node.name][1] node.monitor.WARN_NOT_PARTICIPATING_WINDOW_MINS = backup[node.name][2] diff --git a/plenum/test/storage/test_reply_persistence_ledger.py b/plenum/test/storage/test_reply_persistence_ledger.py index a8f3d44e29..9025581b33 100644 --- a/plenum/test/storage/test_reply_persistence_ledger.py +++ b/plenum/test/storage/test_reply_persistence_ledger.py @@ -1,5 +1,5 @@ from plenum.test.storage.helper import checkReplyIsPersisted -def testReplyPersistedInLedger(nodeSet, looper, replied1): - checkReplyIsPersisted(nodeSet, looper, replied1) +def testReplyPersistedInLedger(txnPoolNodeSet, looper, replied1): + checkReplyIsPersisted(txnPoolNodeSet, looper, replied1) diff --git a/plenum/test/test_delay.py b/plenum/test/test_delay.py index f78237dc98..cbd219ed30 100644 --- a/plenum/test/test_delay.py +++ b/plenum/test/test_delay.py @@ -7,44 +7,41 @@ from plenum.test import waits from plenum.test.delayers import delayerMsgTuple from plenum.test.helper import sendMessageAndCheckDelivery, addNodeBack, assertExp -from plenum.test.msgs import randomMsg, TestMsg +from plenum.test.msgs import TestMsg from plenum.test.test_node import TestNodeSet, checkNodesConnected, \ ensureElectionsDone, prepareNodeSet logger = getlogger() +nodeCount = 2 + @pytest.mark.skipif('sys.platform == "win32"', reason='SOV-457') -def testTestNodeDelay(tdir_for_func, tconf_for_func): - nodeNames = {"testA", "testB"} - with TestNodeSet(tconf_for_func, names=nodeNames, tmpdir=tdir_for_func) as nodes: - nodeA = nodes.getNode("testA") - nodeB = nodes.getNode("testB") - - with Looper(nodes) as looper: - looper.run(checkNodesConnected(nodes)) - - # send one message, without delay - looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB)) - - # set delay, then send another message - # and find that it doesn't arrive - delay = 5 * waits.expectedNodeToNodeMessageDeliveryTime() - nodeB.nodeIbStasher.delay( - delayerMsgTuple(delay, TestMsg, nodeA.name) - ) - with pytest.raises(AssertionError): - looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB)) - - # but then find that it arrives after the delay - # duration has passed - timeout = waits.expectedNodeToNodeMessageDeliveryTime() + delay - looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB, - customTimeout=timeout)) - - # reset the delay, and find another message comes quickly - nodeB.nodeIbStasher.reset_delays_and_process_delayeds() - looper.run(sendMessageAndCheckDelivery(nodes, nodeA, nodeB)) +def testTestNodeDelay(looper, txnPoolNodeSet): + looper.run(checkNodesConnected(txnPoolNodeSet)) + nodeA = txnPoolNodeSet[0] + nodeB = txnPoolNodeSet[1] + # send one message, without delay + looper.run(sendMessageAndCheckDelivery(nodeA, nodeB)) + + # set delay, then send another message + # and find that it doesn't arrive + delay = 5 * waits.expectedNodeToNodeMessageDeliveryTime() + nodeB.nodeIbStasher.delay( + delayerMsgTuple(delay, TestMsg, nodeA.name) + ) + with pytest.raises(AssertionError): + looper.run(sendMessageAndCheckDelivery(nodeA, nodeB)) + + # but then find that it arrives after the delay + # duration has passed + timeout = waits.expectedNodeToNodeMessageDeliveryTime() + delay + looper.run(sendMessageAndCheckDelivery(nodeA, nodeB, + customTimeout=timeout)) + + # reset the delay, and find another message comes quickly + nodeB.nodeIbStasher.reset_delays_and_process_delayeds() + looper.run(sendMessageAndCheckDelivery(nodeA, nodeB)) @pytest.mark.skip('Nodes use round robin primary selection') diff --git a/plenum/test/test_verif_merkle_proof.py b/plenum/test/test_verif_merkle_proof.py index 37fc85d764..60f5a968a8 100644 --- a/plenum/test/test_verif_merkle_proof.py +++ b/plenum/test/test_verif_merkle_proof.py @@ -12,7 +12,7 @@ def testMerkleProofForFirstLeaf(client1: TestClient, replied1): def testMerkleProofForNonFirstLeaf( - looper, nodeSet, wallet1, client1, replied1): + looper, txnPoolNodeSet, wallet1, client1, replied1): req2 = sendRandomRequest(wallet1, client1) waitForSufficientRepliesForRequests(looper, client1, requests=[req2]) replies = client1.getRepliesFromAllNodes(*req2.key).values() From 5d76f77aed9757d3f7b07e6ac6e1063601827325 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 20 Mar 2018 13:34:59 +0300 Subject: [PATCH 22/73] Functions fixes. Signed-off-by: ArtObr --- plenum/test/delayers.py | 4 ++-- .../test_primary_election_with_tie.py | 17 ++++++++--------- .../primary_election/test_primary_forfeit.py | 4 ++-- plenum/test/view_change/helper.py | 6 +++--- .../view_change/test_disable_view_change.py | 7 +++---- .../test_inst_chng_msg_throttling.py | 6 +++--- 6 files changed, 21 insertions(+), 23 deletions(-) diff --git a/plenum/test/delayers.py b/plenum/test/delayers.py index 6e9292e6b3..911b909471 100644 --- a/plenum/test/delayers.py +++ b/plenum/test/delayers.py @@ -200,9 +200,9 @@ def delay(what, frm, to, howlong): "to type {} for {} not supported".format(type(t), t)) -def delayNonPrimaries(nodeSet, instId, delay): +def delayNonPrimaries(txnPoolNodeSet, instId, delay): from plenum.test.test_node import getNonPrimaryReplicas - nonPrimReps = getNonPrimaryReplicas(nodeSet, instId) + nonPrimReps = getNonPrimaryReplicas(txnPoolNodeSet, instId) for r in nonPrimReps: r.node.nodeIbStasher.delay(ppDelay(delay, instId)) return nonPrimReps diff --git a/plenum/test/primary_election/test_primary_election_with_tie.py b/plenum/test/primary_election/test_primary_election_with_tie.py index 58d5f7ac64..6085eccec6 100644 --- a/plenum/test/primary_election/test_primary_election_with_tie.py +++ b/plenum/test/primary_election/test_primary_election_with_tie.py @@ -29,7 +29,7 @@ def electTieFixture(txnPoolNodeSet): # noinspection PyIncorrectDocstring @pytest.mark.skip('Nodes use round robin primary selection') -def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes): +def testPrimaryElectionWithTie(electTieFixture, looper, txnPoolNodeSet): """ Primary selection (Rainy Day) A, B, C, D, E @@ -56,18 +56,17 @@ def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes): # millis have passed, we send the several queued messages in one # batch. - nodeSet = keySharedNodes - A, B, C, D = nodeSet.nodes.values() + A, B, C, D = txnPoolNodeSet - checkPoolReady(looper, nodeSet.nodes.values()) + checkPoolReady(looper, txnPoolNodeSet) - for node in nodeSet.nodes.values(): + for node in txnPoolNodeSet: for instId, replica in enumerate(node.elector.replicas): logger.debug("replica {} {} with votes {}". format(replica.name, replica.instId, node.elector.nominations.get(instId, {}))) - nominationTimeout = waits.expectedPoolNominationTimeout(len(nodeSet)) + nominationTimeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet)) logger.debug("Check nomination") # Checking whether Node A nominated itself looper.run(eventually(checkNomination, A, A.name, @@ -86,10 +85,10 @@ def testPrimaryElectionWithTie(electTieFixture, looper, keySharedNodes): retryWait=1, timeout=nominationTimeout)) # No node should be primary - for node in nodeSet.nodes.values(): + for node in txnPoolNodeSet: assert node.hasPrimary is False - for node in nodeSet.nodes.values(): + for node in txnPoolNodeSet: node.resetDelays() - checkProtocolInstanceSetup(looper=looper, nodes=nodeSet, retryWait=1) + checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1) diff --git a/plenum/test/primary_election/test_primary_forfeit.py b/plenum/test/primary_election/test_primary_forfeit.py index adfb2acef1..f1245527fa 100644 --- a/plenum/test/primary_election/test_primary_forfeit.py +++ b/plenum/test/primary_election/test_primary_forfeit.py @@ -5,12 +5,12 @@ @pytest.mark.skip(reason="SOV-556. Test implementation pending, " "although bug fixed") -def testPrimaryForfeit(looper, nodeSet, up, client1, wallet1): +def testPrimaryForfeit(looper, txnPoolNodeSet, client1, wallet1): """ The primary of master protocol instance of the pool forfeits the primary status by triggering an election and not nominating itself """ - pr = getPrimaryReplica(nodeSet, instId=0) + pr = getPrimaryReplica(txnPoolNodeSet, instId=0) prNode = pr.node # TODO: Incomplete raise NotImplementedError diff --git a/plenum/test/view_change/helper.py b/plenum/test/view_change/helper.py index 451d38763e..d86db4a4a5 100644 --- a/plenum/test/view_change/helper.py +++ b/plenum/test/view_change/helper.py @@ -68,12 +68,12 @@ def provoke_and_wait_for_view_change(looper, timeout=timeout)) -def simulate_slow_master(looper, nodeSet, wallet, +def simulate_slow_master(looper, txnPoolNodeSet, wallet, client, delay=10, num_reqs=4): - m_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) + m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes - delayNonPrimaries(nodeSet, 0, delay) + delayNonPrimaries(txnPoolNodeSet, 0, delay) sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, num_reqs) return m_primary_node diff --git a/plenum/test/view_change/test_disable_view_change.py b/plenum/test/view_change/test_disable_view_change.py index ace32874a1..1fc79173c9 100644 --- a/plenum/test/view_change/test_disable_view_change.py +++ b/plenum/test/view_change/test_disable_view_change.py @@ -13,8 +13,7 @@ def disable_view_change_config(tconf): def test_disable_view_change( disable_view_change_config, looper, - nodeSet, - up, + txnPoolNodeSet, viewNo, wallet1, client1): @@ -22,7 +21,7 @@ def test_disable_view_change( assert isinstance(disable_view_change_config.unsafe, set) assert 'disable_view_change' in disable_view_change_config.unsafe - simulate_slow_master(looper, nodeSet, wallet1, client1) + simulate_slow_master(looper, txnPoolNodeSet, wallet1, client1) with pytest.raises(AssertionError): - waitForViewChange(looper, nodeSet, expectedViewNo=viewNo + 1) + waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) diff --git a/plenum/test/view_change/test_inst_chng_msg_throttling.py b/plenum/test/view_change/test_inst_chng_msg_throttling.py index 17aa5fe8ee..4bf6e35e48 100644 --- a/plenum/test/view_change/test_inst_chng_msg_throttling.py +++ b/plenum/test/view_change/test_inst_chng_msg_throttling.py @@ -1,7 +1,7 @@ import types -def testInstChngMsgThrottling(nodeSet, looper, up, viewNo): +def testInstChngMsgThrottling(txnPoolNodeSet, looper, viewNo): """ 2 nodes out of 4 keep on sending INSTANCE_CHANGE messages as they find the master to be slow but since we need 3 out of 4 (n-f) to say that @@ -12,8 +12,8 @@ def testInstChngMsgThrottling(nodeSet, looper, up, viewNo): THE TEST BELOW SHOULD TERMINATE. IF IT DOES NOT TERMINATE THEN THE BUG IS STILL PRESENT """ - nodeA = nodeSet.Alpha - nodeB = nodeSet.Beta + nodeA = txnPoolNodeSet[0] + nodeB = txnPoolNodeSet[0] # Nodes that always find master as degraded for node in (nodeA, nodeB): node.monitor.isMasterDegraded = types.MethodType( From 9c6a157939ac3a4dfe5073cdc36d4b2c2be9eabe Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 20 Mar 2018 15:27:15 +0300 Subject: [PATCH 23/73] Fix for genesis txns Signed-off-by: ArtObr --- .../test_request_executed_once_and_without_failing_behind.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plenum/test/test_request_executed_once_and_without_failing_behind.py b/plenum/test/test_request_executed_once_and_without_failing_behind.py index 3851362bb3..36c2533f13 100644 --- a/plenum/test/test_request_executed_once_and_without_failing_behind.py +++ b/plenum/test/test_request_executed_once_and_without_failing_behind.py @@ -52,6 +52,6 @@ def test_request_executed_once_and_without_failing_behind(tconf, looper, expected = [request.reqId for request in requests] for node in txnPoolNodeSet: - real_ledger_state = [txn[1]["reqId"] - for txn in node.getLedger(DOMAIN_LEDGER_ID).getAllTxn()] + real_ledger_state = [txn[1]['reqId'] + for txn in node.getLedger(DOMAIN_LEDGER_ID).getAllTxn() if 'reqId' in txn[1]] assert expected == real_ledger_state From 8b7ee344cb8417766bd7602d66482174f0e1240e Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 20 Mar 2018 16:52:16 +0300 Subject: [PATCH 24/73] More tests integrated Signed-off-by: ArtObr --- ..._blacklist_node_on_multiple_nominations.py | 7 ++-- ...t_node_on_multiple_primary_declarations.py | 8 ++--- .../test/client/test_client_request_nack.py | 8 ++--- plenum/test/helper.py | 6 ++-- .../test_multiple_instance_change_msgs.py | 10 +++--- .../test_instance_change_with_Delta.py | 4 +-- plenum/test/plugin/demo_plugin/conftest.py | 2 +- .../test/plugin/demo_plugin/test_catchup.py | 4 +-- .../plugin/demo_plugin/test_plugin_basic.py | 8 ++--- .../test_plugin_request_handling.py | 18 +++++----- .../test_propagate_recvd_after_request.py | 8 ++--- .../test_propagate_recvd_before_request.py | 10 +++--- ...y_marked_suspicious_for_sending_prepare.py | 16 ++++----- .../test_replica_reject_same_pre_prepare.py | 16 ++++----- plenum/test/signing/test_signing.py | 19 ++++++----- plenum/test/test_node.py | 10 +++--- .../test_3pc_msgs_during_view_change.py | 32 ++++++++--------- .../test_instance_change_msg_checking.py | 8 ++--- .../test_queueing_req_from_future_view.py | 6 ++-- .../test_view_change_happens_post_timeout.py | 4 +-- .../test_view_change_min_cathup_timeout.py | 20 +++++------ .../test_view_change_on_master_degraded.py | 34 +++++++++---------- 22 files changed, 128 insertions(+), 130 deletions(-) diff --git a/plenum/test/blacklist/test_blacklist_node_on_multiple_nominations.py b/plenum/test/blacklist/test_blacklist_node_on_multiple_nominations.py index e442282cf5..575885285e 100644 --- a/plenum/test/blacklist/test_blacklist_node_on_multiple_nominations.py +++ b/plenum/test/blacklist/test_blacklist_node_on_multiple_nominations.py @@ -10,12 +10,11 @@ # noinspection PyIncorrectDocstring,PyUnusedLocal,PyShadowingNames @pytest.mark.skip(reason="SOV-540. Implementation changed.") -def testBlacklistNodeOnMultipleNominations(looper, keySharedNodes, ready): +def testBlacklistNodeOnMultipleNominations(looper, txnPoolNodeSet, ready): """ A node that sends multiple nominations must be blacklisted by other nodes """ - nodeSet = keySharedNodes - A, B, C, D = nodeSet.nodes.values() + A, B, C, D = txnPoolNodeSet # B sends more than 2 nominations for i in range(3): @@ -26,5 +25,5 @@ def chk(): for node in A, C, D: assert node.isNodeBlacklisted(B.name) - timeout = waits.expectedPoolNominationTimeout(len(nodeSet.nodes)) + timeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) diff --git a/plenum/test/blacklist/test_blacklist_node_on_multiple_primary_declarations.py b/plenum/test/blacklist/test_blacklist_node_on_multiple_primary_declarations.py index db2ae44244..1aabcb1887 100644 --- a/plenum/test/blacklist/test_blacklist_node_on_multiple_primary_declarations.py +++ b/plenum/test/blacklist/test_blacklist_node_on_multiple_primary_declarations.py @@ -12,14 +12,12 @@ # noinspection PyIncorrectDocstring @pytest.mark.skip(reason="SOV-541. Implementation changed.") def testBlacklistNodeOnMultiplePrimaryDeclarations(looper, - keySharedNodes, - ready): + txnPoolNodeSet): """ A node that sends multiple primary declarations must be blacklisted by other nodes """ - nodeSet = keySharedNodes - A, B, C, D = nodeSet.nodes.values() + A, B, C, D = txnPoolNodeSet # B sends more than 2 primary declarations for i in range(3): @@ -30,5 +28,5 @@ def chk(): for node in A, C, D: assert node.isNodeBlacklisted(B.name) - timeout = waits.expectedPoolNominationTimeout(len(nodeSet.nodes)) + timeout = waits.expectedPoolNominationTimeout(len(txnPoolNodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) diff --git a/plenum/test/client/test_client_request_nack.py b/plenum/test/client/test_client_request_nack.py index 0d43f2d05b..5e1b97edad 100644 --- a/plenum/test/client/test_client_request_nack.py +++ b/plenum/test/client/test_client_request_nack.py @@ -17,8 +17,8 @@ def verify(operation): @pytest.fixture(scope="module") -def restrictiveVerifier(nodeSet): - for n in nodeSet: +def restrictiveVerifier(txnPoolNodeSet): + for n in txnPoolNodeSet: n.opVerifiers = [TestVerifier()] @@ -35,12 +35,12 @@ def testRequestFullRoundTrip(restrictiveVerifier, client1, sent1, looper, - nodeSet): + txnPoolNodeSet): update = {'reason': 'client request invalid: InvalidClientRequest() ' '[caused by amount too high\nassert 999 <= 100]'} coros2 = [partial(checkReqNack, client1, node, sent1.identifier, sent1.reqId, update) - for node in nodeSet] + for node in txnPoolNodeSet] timeout = waits.expectedReqAckQuorumTime() looper.run(eventuallyAll(*coros2, totalTimeout=timeout)) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 20731c3e02..0b58068065 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -476,9 +476,9 @@ def checkRequestNotReturnedToNode(node: TestNode, identifier: str, reqId: int, assert not requestReturnedToNode(node, identifier, reqId, instId) -def check_request_is_not_returned_to_nodes(nodeSet, request): - instances = range(getNoInstances(len(nodeSet))) - for node, inst_id in itertools.product(nodeSet, instances): +def check_request_is_not_returned_to_nodes(txnPoolNodeSet, request): + instances = range(getNoInstances(len(txnPoolNodeSet))) + for node, inst_id in itertools.product(txnPoolNodeSet, instances): checkRequestNotReturnedToNode(node, request.identifier, request.reqId, diff --git a/plenum/test/instances/test_multiple_instance_change_msgs.py b/plenum/test/instances/test_multiple_instance_change_msgs.py index 1e825dee7d..1a2a9ee966 100644 --- a/plenum/test/instances/test_multiple_instance_change_msgs.py +++ b/plenum/test/instances/test_multiple_instance_change_msgs.py @@ -13,20 +13,20 @@ @pytest.mark.skip(reason="INDY-80. Not yet implemented") -def testMultipleInstanceChangeMsgsMarkNodeAsSuspicious(looper, nodeSet, up): - maliciousNode = nodeSet.Alpha +def testMultipleInstanceChangeMsgsMarkNodeAsSuspicious(looper, txnPoolNodeSet): + maliciousNode = txnPoolNodeSet[0] for i in range(0, 5): maliciousNode.send(maliciousNode.view_changer._create_instance_change_msg(i, 0)) def chk(instId): - for node in nodeSet: + for node in txnPoolNodeSet: if node.name != maliciousNode.name: args = getAllArgs(node, ViewChanger.process_instance_change_msg) assert len(args) == 5 for arg in args: assert arg['frm'] == maliciousNode.name - numOfNodes = len(nodeSet) + numOfNodes = len(txnPoolNodeSet) instanceChangeTimeout = waits.expectedPoolViewChangeStartedTimeout( numOfNodes) @@ -35,7 +35,7 @@ def chk(instId): timeout=instanceChangeTimeout)) def g(): - for node in nodeSet: + for node in txnPoolNodeSet: if node.name != maliciousNode.name: frm, reason, code = getAllArgs(node, Node.reportSuspiciousNode) assert frm == maliciousNode.name diff --git a/plenum/test/monitoring/test_instance_change_with_Delta.py b/plenum/test/monitoring/test_instance_change_with_Delta.py index f7895cee38..46b9523f40 100644 --- a/plenum/test/monitoring/test_instance_change_with_Delta.py +++ b/plenum/test/monitoring/test_instance_change_with_Delta.py @@ -66,8 +66,8 @@ def ensureAnotherPerfCheck(): @pytest.fixture(scope="module") -def step1(looper, nodeSet, up, wallet1, client1): - startedNodes = nodeSet +def step1(looper, txnPoolNodeSet, wallet1, client1): + startedNodes = txnPoolNodeSet """ stand up a pool of nodes and send 5 requests to client """ diff --git a/plenum/test/plugin/demo_plugin/conftest.py b/plenum/test/plugin/demo_plugin/conftest.py index a50c33f617..b42568616a 100644 --- a/plenum/test/plugin/demo_plugin/conftest.py +++ b/plenum/test/plugin/demo_plugin/conftest.py @@ -67,5 +67,5 @@ def _post_node_creation(node): @pytest.fixture(scope="module") -def nodeSet(tconf, do_post_node_creation, txnPoolNodeSet): +def txn_pool_node_set_post_creation(tconf, do_post_node_creation, txnPoolNodeSet): return txnPoolNodeSet diff --git a/plenum/test/plugin/demo_plugin/test_catchup.py b/plenum/test/plugin/demo_plugin/test_catchup.py index 48a93bf38b..887d0e1ab0 100644 --- a/plenum/test/plugin/demo_plugin/test_catchup.py +++ b/plenum/test/plugin/demo_plugin/test_catchup.py @@ -15,13 +15,13 @@ disconnect_node_and_ensure_disconnected, reconnect_node_and_ensure_connected -def test_new_node_catchup_plugin_ledger(nodeSet, looper, some_requests, +def test_new_node_catchup_plugin_ledger(txn_pool_node_set_post_creation, looper, some_requests, newNodeCaughtUp): """ A new node catches up the demo plugin's ledger too """ assert len(newNodeCaughtUp.getLedger(AUCTION_LEDGER_ID)) > 0 - for node in nodeSet[:-1]: + for node in txn_pool_node_set_post_creation[:-1]: assert len(newNodeCaughtUp.getLedger(AUCTION_LEDGER_ID)) == \ len(node.getLedger(AUCTION_LEDGER_ID)) diff --git a/plenum/test/plugin/demo_plugin/test_plugin_basic.py b/plenum/test/plugin/demo_plugin/test_plugin_basic.py index ef0eb5dad4..aa36471a65 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_basic.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_basic.py @@ -8,18 +8,18 @@ from stp_core.loop.eventually import eventually -def test_plugin_setup(nodeSet): +def test_plugin_setup(txn_pool_node_set_post_creation): """ Test that plugin's ledger and state are setup """ - for node in nodeSet: + for node in txn_pool_node_set_post_creation: assert AUCTION_LEDGER_ID in node.ledger_ids assert AUCTION_LEDGER_ID in node.ledgerManager.ledgerRegistry assert node.ledger_ids == node.ledgerManager.ledger_sync_order assert AUCTION_LEDGER_ID in node.states -def test_plugin_client_req_fields(nodeSet, looper, stewardWallet, +def test_plugin_client_req_fields(txn_pool_node_set_post_creation, looper, stewardWallet, steward1, sdk_wallet_steward, sdk_pool_handle): """ @@ -44,7 +44,7 @@ def test_plugin_client_req_fields(nodeSet, looper, stewardWallet, identifier=stewardWallet.defaultId, fix_length_dummy=randomString(dummy_field_length + 1)) steward1.submitReqs(req) - for node in nodeSet: + for node in txn_pool_node_set_post_creation: looper.run(eventually(checkReqNackWithReason, steward1, 'should have length', node.clientstack.name, retryWait=1)) diff --git a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py index f0402bb550..77cbe3fd2a 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py @@ -16,7 +16,7 @@ def successful_op(looper, op, sdk_wallet, sdk_pool_handle): sdk_get_reply(looper, req) -def test_plugin_static_validation(nodeSet, looper, stewardWallet, +def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, stewardWallet, steward1, sdk_wallet_steward, sdk_pool_handle): """ @@ -26,7 +26,7 @@ def test_plugin_static_validation(nodeSet, looper, stewardWallet, TXN_TYPE: AUCTION_START } send_signed_requests(steward1, sign_requests(stewardWallet, [op, ])) - waitReqNackFromPoolWithReason(looper, nodeSet, steward1, + waitReqNackFromPoolWithReason(looper, txn_pool_node_set_post_creation, steward1, 'attribute is missing or not in proper format') op = { @@ -34,7 +34,7 @@ def test_plugin_static_validation(nodeSet, looper, stewardWallet, DATA: 'should be a dict but giving a string' } send_signed_requests(steward1, sign_requests(stewardWallet, [op, ])) - waitReqNackFromPoolWithReason(looper, nodeSet, steward1, + waitReqNackFromPoolWithReason(looper, txn_pool_node_set_post_creation, steward1, 'attribute is missing or not in proper format') op = { @@ -49,7 +49,7 @@ def test_plugin_static_validation(nodeSet, looper, stewardWallet, DATA: {'id': 'abc', AMOUNT: -3} } send_signed_requests(steward1, sign_requests(stewardWallet, [op, ])) - waitReqNackFromPoolWithReason(looper, nodeSet, steward1, + waitReqNackFromPoolWithReason(looper, txn_pool_node_set_post_creation, steward1, 'must be present and should be a number') op = { @@ -59,7 +59,7 @@ def test_plugin_static_validation(nodeSet, looper, stewardWallet, successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle) -def test_plugin_dynamic_validation(nodeSet, looper, stewardWallet, +def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, stewardWallet, steward1, sdk_wallet_steward, sdk_pool_handle): """ @@ -70,7 +70,7 @@ def test_plugin_dynamic_validation(nodeSet, looper, stewardWallet, DATA: {'id': 'abcdef'} } send_signed_requests(steward1, sign_requests(stewardWallet, [op, ])) - waitRejectFromPoolWithReason(looper, nodeSet, steward1, + waitRejectFromPoolWithReason(looper, txn_pool_node_set_post_creation, steward1, 'unknown auction') op = { @@ -87,7 +87,7 @@ def test_plugin_dynamic_validation(nodeSet, looper, stewardWallet, @pytest.fixture(scope="module") -def some_requests(nodeSet, looper, stewardWallet, +def some_requests(txn_pool_node_set_post_creation, looper, stewardWallet, steward1, sdk_wallet_steward, sdk_pool_handle): op = { @@ -102,7 +102,7 @@ def some_requests(nodeSet, looper, stewardWallet, } successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle) - for node in nodeSet: + for node in txn_pool_node_set_post_creation: auctions = node.get_req_handler(AUCTION_LEDGER_ID).auctions assert 'pqr' in auctions assert auctions['pqr'][stewardWallet.defaultId] == 20 @@ -113,7 +113,7 @@ def some_requests(nodeSet, looper, stewardWallet, } successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle) - for node in nodeSet: + for node in txn_pool_node_set_post_creation: auctions = node.get_req_handler(AUCTION_LEDGER_ID).auctions assert 'pqr' in auctions assert auctions['pqr'][stewardWallet.defaultId] == 40 diff --git a/plenum/test/propagate/test_propagate_recvd_after_request.py b/plenum/test/propagate/test_propagate_recvd_after_request.py index be8b109f34..88559c8c2b 100644 --- a/plenum/test/propagate/test_propagate_recvd_after_request.py +++ b/plenum/test/propagate/test_propagate_recvd_after_request.py @@ -13,16 +13,16 @@ @pytest.fixture() -def setup(nodeSet): - A, B, C, D = nodeSet.nodes.values() # type: TestNode +def setup(txnPoolNodeSet): + A, B, C, D = txnPoolNodeSet # type: TestNode delay(Propagate, frm=[B, C, D], to=A, howlong=howlong) # Delay MessageRep by long simulating loss as if Propagate is missing, it # is requested A.nodeIbStasher.delay(msg_rep_delay(10 * howlong, [PROPAGATE, ])) -def testPropagateRecvdAfterRequest(setup, looper, nodeSet, up, sent1): - A, B, C, D = nodeSet.nodes.values() # type: TestNode +def testPropagateRecvdAfterRequest(setup, looper, txnPoolNodeSet, sent1): + A, B, C, D = txnPoolNodeSet # type: TestNode def x(): # A should have received a request from the client diff --git a/plenum/test/propagate/test_propagate_recvd_before_request.py b/plenum/test/propagate/test_propagate_recvd_before_request.py index afca15b112..cbdce9037d 100644 --- a/plenum/test/propagate/test_propagate_recvd_before_request.py +++ b/plenum/test/propagate/test_propagate_recvd_before_request.py @@ -14,14 +14,14 @@ @pytest.fixture() -def setup(nodeSet): - A, B, C, D = nodeSet.nodes.values() +def setup(txnPoolNodeSet): + A, B, C, D = txnPoolNodeSet A.clientIbStasher.delay(lambda x: delaySec) delay(Propagate, frm=[C, D], to=A, howlong=howlong) -def testPropagateRecvdBeforeRequest(setup, looper, nodeSet, up, sent1): - A, B, C, D = nodeSet.nodes.values() +def testPropagateRecvdBeforeRequest(setup, looper, txnPoolNodeSet, sent1): + A, B, C, D = txnPoolNodeSet def x(): # A should not have received a request from the client @@ -48,5 +48,5 @@ def chk(): assertLength(forwardedRequest(A), 1) timeout = waits.expectedClientRequestPropagationTime( - len(nodeSet)) + delaySec + len(txnPoolNodeSet)) + delaySec looper.run(eventually(chk, retryWait=1, timeout=timeout)) diff --git a/plenum/test/replica/test_primary_marked_suspicious_for_sending_prepare.py b/plenum/test/replica/test_primary_marked_suspicious_for_sending_prepare.py index 4f79c1b516..28f8382bcb 100644 --- a/plenum/test/replica/test_primary_marked_suspicious_for_sending_prepare.py +++ b/plenum/test/replica/test_primary_marked_suspicious_for_sending_prepare.py @@ -13,22 +13,22 @@ @pytest.fixture(scope="module") -def delay_commits(nodeSet): +def delay_commits(txnPoolNodeSet): # Delay COMMITs so that ordering is delayed and checks can be made - for n in nodeSet: + for n in txnPoolNodeSet: n.nodeIbStasher.delay(cDelay(5)) -def testPrimarySendsAPrepareAndMarkedSuspicious(looper, nodeSet, delay_commits, +def testPrimarySendsAPrepareAndMarkedSuspicious(looper, txnPoolNodeSet, delay_commits, preprepared1): def sendPrepareFromPrimary(instId): - primary = getPrimaryReplica(nodeSet, instId) + primary = getPrimaryReplica(txnPoolNodeSet, instId) viewNo, ppSeqNo = next(iter(primary.sentPrePrepares.keys())) ppReq = primary.sentPrePrepares[viewNo, ppSeqNo] primary.doPrepare(ppReq) def chk(): - for r in getNonPrimaryReplicas(nodeSet, instId): + for r in getNonPrimaryReplicas(txnPoolNodeSet, instId): l = len([param for param in getAllArgs(r, r.processPrepare) if param['sender'] == primary.name]) assert l == 1 @@ -37,10 +37,10 @@ def chk(): sendPrepareFromPrimary(0) - for node in nodeSet: - if node in getNonPrimaryReplicas(nodeSet, 0): + for node in txnPoolNodeSet: + if node in getNonPrimaryReplicas(txnPoolNodeSet, 0): frm, reason, code = getAllArgs(node, TestNode.reportSuspiciousNode) - assert frm == getPrimaryReplica(nodeSet, 0).node.name + assert frm == getPrimaryReplica(txnPoolNodeSet, 0).node.name assert isinstance(reason, SuspiciousNode) assert len(getNodeSuspicions(node, Suspicions.PR_FRM_PRIMARY.code)) == 10 diff --git a/plenum/test/replica/test_replica_reject_same_pre_prepare.py b/plenum/test/replica/test_replica_reject_same_pre_prepare.py index d02324f6ec..0fb79c6872 100644 --- a/plenum/test/replica/test_replica_reject_same_pre_prepare.py +++ b/plenum/test/replica/test_replica_reject_same_pre_prepare.py @@ -20,7 +20,7 @@ # noinspection PyIncorrectDocstring -def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): +def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and @@ -29,14 +29,14 @@ def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): """ numOfNodes = 4 fValue = getMaxFailures(numOfNodes) - primaryRepl = getPrimaryReplica(nodeSet, 1) + primaryRepl = getPrimaryReplica(txnPoolNodeSet, 1) logger.debug("Primary Replica: {}".format(primaryRepl)) - nonPrimaryReplicas = getNonPrimaryReplicas(nodeSet, 1) + nonPrimaryReplicas = getNonPrimaryReplicas(txnPoolNodeSet, 1) logger.debug("Non Primary Replicas: " + str(nonPrimaryReplicas)) # Delay COMMITs so request is not ordered and checks can be made c_delay = 10 - for node in nodeSet: + for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(delay=c_delay, instId=1)) request1 = sendRandomRequest(wallet1, client1) @@ -69,7 +69,7 @@ def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): primaryRepl._lastPrePrepareSeqNo -= 1 view_no = primaryRepl.viewNo request2 = sendRandomRequest(wallet1, client1) - timeout = waits.expectedPrePrepareTime(len(nodeSet)) + timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=timeout)) @@ -94,7 +94,7 @@ def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): logger.debug("""Checking whether all the non primary replicas have received the pre-prepare request with same sequence number""") - timeout = waits.expectedPrePrepareTime(len(nodeSet)) + timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(checkPrePrepareReqRecvd, nonPrimaryReplicas, prePrepareReq, @@ -103,7 +103,7 @@ def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): logger.debug("""Check that none of the non primary replicas didn't send any prepare message " in response to the pre-prepare message""") - timeout = waits.expectedPrepareTime(len(nodeSet)) + timeout = waits.expectedPrepareTime(len(txnPoolNodeSet)) looper.runFor(timeout) # expect prepare processing timeout # check if prepares have not been sent @@ -117,7 +117,7 @@ def testReplicasRejectSamePrePrepareMsg(looper, nodeSet, client1, wallet1): retryWait=1, timeout=timeout)) - timeout = waits.expectedTransactionExecutionTime(len(nodeSet)) + c_delay + timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + c_delay result1 = \ looper.run(eventually(check_sufficient_replies_received, client1, request1.identifier, request1.reqId, diff --git a/plenum/test/signing/test_signing.py b/plenum/test/signing/test_signing.py index bb8277ab8f..b06814c321 100644 --- a/plenum/test/signing/test_signing.py +++ b/plenum/test/signing/test_signing.py @@ -15,29 +15,30 @@ @pytest.fixture(scope="module") -def setup(nodeSet): - gn = [v for k, v in nodeSet.nodes.items() if k != 'Alpha'] +def setup(txnPoolNodeSet): + pool_without_alpha = list(txnPoolNodeSet) + pool_without_alpha.remove(txnPoolNodeSet[0]) # delay incoming client messages for good nodes by 250 milliseconds # this gives Alpha a chance to send a propagate message - for n in gn: # type: TestNode + for n in pool_without_alpha: # type: TestNode n.clientIbStasher.delay(lambda _: 1) - return adict(goodNodes=gn) + return adict(goodNodes=pool_without_alpha) @pytest.fixture(scope="module") -def evilAlpha(nodeSet): - makeNodeFaulty(nodeSet.Alpha, changesRequest) +def evilAlpha(txnPoolNodeSet): + makeNodeFaulty(txnPoolNodeSet[0], changesRequest) faultyNodes = 1 def testOneNodeAltersAClientRequest(looper, - nodeSet, + txnPoolNodeSet, setup, evilAlpha, sent1): - checkPropagated(looper, nodeSet, sent1, faultyNodes) + checkPropagated(looper, txnPoolNodeSet, sent1, faultyNodes) goodNodes = setup.goodNodes @@ -58,5 +59,5 @@ def check(): for good in goodNodes: assert good.name in props - timeout = waits.expectedClientRequestPropagationTime(len(nodeSet)) + timeout = waits.expectedClientRequestPropagationTime(len(txnPoolNodeSet)) looper.run(eventually(check, retryWait=1, timeout=timeout)) diff --git a/plenum/test/test_node.py b/plenum/test/test_node.py index dc546c79eb..d91c5bd4e1 100644 --- a/plenum/test/test_node.py +++ b/plenum/test/test_node.py @@ -890,16 +890,16 @@ def extractCliNodeReg(self): return nodeReg -def prepareNodeSet(looper: Looper, nodeSet: TestNodeSet): +def prepareNodeSet(looper: Looper, txnPoolNodeSet): # TODO: Come up with a more specific name for this # Key sharing party - looper.run(checkNodesConnected(nodeSet)) + looper.run(checkNodesConnected(txnPoolNodeSet)) # Remove all the nodes - for n in list(nodeSet.nodes.keys()): - looper.removeProdable(nodeSet.nodes[n]) - nodeSet.removeNode(n) + for n in list(txnPoolNodeSet): + looper.removeProdable(txnPoolNodeSet) + txnPoolNodeSet.remove(n) def checkViewChangeInitiatedForNode(node: TestNode, proposedViewNo: int): diff --git a/plenum/test/view_change/test_3pc_msgs_during_view_change.py b/plenum/test/view_change/test_3pc_msgs_during_view_change.py index 7c9b2520a4..933b81105a 100644 --- a/plenum/test/view_change/test_3pc_msgs_during_view_change.py +++ b/plenum/test/view_change/test_3pc_msgs_during_view_change.py @@ -9,47 +9,47 @@ @pytest.mark.skip('Currently we stash client requests during view change') -def test_no_requests_processed_during_view_change(looper, nodeSet, +def test_no_requests_processed_during_view_change(looper, txnPoolNodeSet, client1, wallet1): - for node in nodeSet: + for node in txnPoolNodeSet: node.view_change_in_progress = True sendRandomRequests(wallet1, client1, 10) waitRejectFromPoolWithReason( looper, - nodeSet, + txnPoolNodeSet, client1, 'Can not process requests when view change is in progress') - for node in nodeSet: + for node in txnPoolNodeSet: check_replica_queue_empty(node) @pytest.mark.skip('The filter is not enabled now') def test_no_new_view_3pc_messages_processed_during_view_change( - looper, nodeSet, client1, wallet1): - for node in nodeSet: + looper, txnPoolNodeSet, client1, wallet1): + for node in txnPoolNodeSet: node.view_change_in_progress = True - new_view_no = getPrimaryReplica(nodeSet).node.viewNo + 1 + new_view_no = getPrimaryReplica(txnPoolNodeSet).node.viewNo + 1 pp_seq_no = 1 - send_pre_prepare(new_view_no, pp_seq_no, wallet1, nodeSet) + send_pre_prepare(new_view_no, pp_seq_no, wallet1, txnPoolNodeSet) looper.runFor(1) - check_all_replica_queue_empty(nodeSet) + check_all_replica_queue_empty(txnPoolNodeSet) - send_prepare(new_view_no, pp_seq_no, nodeSet) + send_prepare(new_view_no, pp_seq_no, txnPoolNodeSet) looper.runFor(1) - check_all_replica_queue_empty(nodeSet) + check_all_replica_queue_empty(txnPoolNodeSet) - send_commit(new_view_no, pp_seq_no, nodeSet) + send_commit(new_view_no, pp_seq_no, txnPoolNodeSet) looper.runFor(1) - check_all_replica_queue_empty(nodeSet) + check_all_replica_queue_empty(txnPoolNodeSet) @pytest.mark.skip('The filter is not enabled now') -def test_old_view_requests_processed_during_view_change(looper, nodeSet, +def test_old_view_requests_processed_during_view_change(looper, txnPoolNodeSet, client1, wallet1): """ Make sure that requests sent before view change started are processed and replies are returned: @@ -57,14 +57,14 @@ def test_old_view_requests_processed_during_view_change(looper, nodeSet, - send requests - check that requests are ordered despite of view change being in progress """ - for node in nodeSet: + for node in txnPoolNodeSet: node.view_change_in_progress = False node.nodeIbStasher.delay(ppgDelay(3, 0)) reqs = sendRandomRequests(wallet1, client1, 2) looper.runFor(1) - for node in nodeSet: + for node in txnPoolNodeSet: node.view_change_in_progress = True waitForSufficientRepliesForRequests(looper, client1, requests=reqs) diff --git a/plenum/test/view_change/test_instance_change_msg_checking.py b/plenum/test/view_change/test_instance_change_msg_checking.py index 4f3c430db5..2c70f5100e 100644 --- a/plenum/test/view_change/test_instance_change_msg_checking.py +++ b/plenum/test/view_change/test_instance_change_msg_checking.py @@ -10,9 +10,9 @@ whitelist = [DISCARD_REASON, ] -def testInstanceChangeMsgTypeChecking(nodeSet, looper, up): - nodeA = nodeSet.Alpha - nodeB = nodeSet.Beta +def testInstanceChangeMsgTypeChecking(txnPoolNodeSet, looper): + nodeA = txnPoolNodeSet[0] + nodeB = txnPoolNodeSet[1] ridBeta = nodeA.nodestack.getRemote(nodeB.name).uid @@ -22,7 +22,7 @@ def createInstanceChangeMessage(): goodViewNo = 1 badViewNo = "BAD" - icMsg = nodeSet.Alpha.view_changer._create_instance_change_msg(goodViewNo, 0) + icMsg = txnPoolNodeSet[0].view_changer._create_instance_change_msg(goodViewNo, 0) icMsg._fields["viewNo"] = badViewNo return icMsg diff --git a/plenum/test/view_change/test_queueing_req_from_future_view.py b/plenum/test/view_change/test_queueing_req_from_future_view.py index 3537b39ca0..6973452729 100644 --- a/plenum/test/view_change/test_queueing_req_from_future_view.py +++ b/plenum/test/view_change/test_queueing_req_from_future_view.py @@ -19,7 +19,7 @@ # noinspection PyIncorrectDocstring -def testQueueingReqFromFutureView(delayed_perf_chk, looper, nodeSet, up, +def testQueueingReqFromFutureView(delayed_perf_chk, looper, txnPoolNodeSet, wallet1, client1): """ Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT) @@ -28,7 +28,7 @@ def testQueueingReqFromFutureView(delayed_perf_chk, looper, nodeSet, up, => it starts receiving 3 phase commit messages for next view """ - lagging_node = get_last_master_non_primary_node(nodeSet) + lagging_node = get_last_master_non_primary_node(txnPoolNodeSet) old_view_no = lagging_node.viewNo # Delay processing of InstanceChange and ViewChangeDone so node stashes @@ -53,7 +53,7 @@ def chk_fut_view(view_no, is_empty): # Every node except Node A should do a view change ensure_view_change(looper, - [n for n in nodeSet if n != lagging_node], + [n for n in txnPoolNodeSet if n != lagging_node], [lagging_node]) # send more requests that will be queued for the lagged node diff --git a/plenum/test/view_change/test_view_change_happens_post_timeout.py b/plenum/test/view_change/test_view_change_happens_post_timeout.py index ea3095f518..9f92381786 100644 --- a/plenum/test/view_change/test_view_change_happens_post_timeout.py +++ b/plenum/test/view_change/test_view_change_happens_post_timeout.py @@ -18,8 +18,8 @@ def is_instance_change_sent_for_view_no(node, view_no): return node.view_changer.instanceChanges.hasView(view_no) -def test_instance_change_happens_post_timeout(tconf, looper, nodeSet, up): - non_prim_node = getNonPrimaryReplicas(nodeSet)[0].node +def test_instance_change_happens_post_timeout(tconf, looper, txnPoolNodeSet): + non_prim_node = getNonPrimaryReplicas(txnPoolNodeSet)[0].node old_view_no = non_prim_node.viewNo # first sending on InstanceChange: OK diff --git a/plenum/test/view_change/test_view_change_min_cathup_timeout.py b/plenum/test/view_change/test_view_change_min_cathup_timeout.py index 38f8092f3c..3a6b928b1d 100644 --- a/plenum/test/view_change/test_view_change_min_cathup_timeout.py +++ b/plenum/test/view_change/test_view_change_min_cathup_timeout.py @@ -11,17 +11,17 @@ nodeCount = 7 -def patch_has_ordered_till_last_prepared_certificate(nodeSet): +def patch_has_ordered_till_last_prepared_certificate(txnPoolNodeSet): def patched_has_ordered_till_last_prepared_certificate(self) -> bool: return False - for node in nodeSet: + for node in txnPoolNodeSet: node.has_ordered_till_last_prepared_certificate = \ types.MethodType( patched_has_ordered_till_last_prepared_certificate, node) -def test_view_change_min_catchup_timeout(nodeSet, up, looper, wallet1, client1, +def test_view_change_min_catchup_timeout(txnPoolNodeSet, looper, wallet1, client1, tconf, viewNo): """ @@ -43,26 +43,26 @@ def test_view_change_min_catchup_timeout(nodeSet, up, looper, wallet1, client1, # 2. make the only condition to finish catch-up is # MIN_TIMEOUT_CATCHUPS_DONE_DURING_VIEW_CHANGE - patch_has_ordered_till_last_prepared_certificate(nodeSet) + patch_has_ordered_till_last_prepared_certificate(txnPoolNodeSet) # 3. start view change expected_view_no = viewNo + 1 - for node in nodeSet: + for node in txnPoolNodeSet: node.view_changer.startViewChange(expected_view_no) # 4. check that it's not finished till # MIN_TIMEOUT_CATCHUPS_DONE_DURING_VIEW_CHANGE no_view_chanage_timeout = tconf.MIN_TIMEOUT_CATCHUPS_DONE_DURING_VIEW_CHANGE - 1 with pytest.raises(EventuallyTimeoutException): - ensureElectionsDone(looper=looper, nodes=nodeSet, + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=no_view_chanage_timeout) # 5. make sure that view change is finished eventually # (it should be finished quite soon after we waited for MIN_TIMEOUT_CATCHUPS_DONE_DURING_VIEW_CHANGE) - ensureElectionsDone(looper=looper, nodes=nodeSet, customTimeout=2) - waitForViewChange(looper=looper, nodeSet=nodeSet, + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet, customTimeout=2) + waitForViewChange(looper=looper, txnPoolNodeSet=txnPoolNodeSet, expectedViewNo=expected_view_no) - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # 6. ensure that the pool is still functional. - ensure_pool_functional(looper, nodeSet, wallet1, client1) + ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1) diff --git a/plenum/test/view_change/test_view_change_on_master_degraded.py b/plenum/test/view_change/test_view_change_on_master_degraded.py index a2d18a406b..a7ddbcee17 100644 --- a/plenum/test/view_change/test_view_change_on_master_degraded.py +++ b/plenum/test/view_change/test_view_change_on_master_degraded.py @@ -17,7 +17,7 @@ # noinspection PyIncorrectDocstring -def test_view_change_on_performance_degraded(looper, nodeSet, up, viewNo, +def test_view_change_on_performance_degraded(looper, txnPoolNodeSet, viewNo, wallet1, client1): """ Test that a view change is done when the performance of master goes down @@ -25,37 +25,37 @@ def test_view_change_on_performance_degraded(looper, nodeSet, up, viewNo, instance so that there is a view change. All nodes will agree that master performance degraded """ - old_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) + old_primary_node = get_master_primary_node(list(txnPoolNodeSet)) - simulate_slow_master(looper, nodeSet, wallet1, client1) - waitForViewChange(looper, nodeSet, expectedViewNo=viewNo + 1) + simulate_slow_master(looper, txnPoolNodeSet, wallet1, client1) + waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) - ensureElectionsDone(looper=looper, nodes=nodeSet) - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) - new_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) + new_primary_node = get_master_primary_node(list(txnPoolNodeSet)) assert old_primary_node.name != new_primary_node.name -def test_view_change_on_quorum_of_master_degraded(nodeSet, looper, up, +def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper, wallet1, client1, viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded """ - m_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) + m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes - delayNonPrimaries(nodeSet, 0, 10) + delayNonPrimaries(txnPoolNodeSet, 0, 10) - pr = getPrimaryReplica(nodeSet, 0) + pr = getPrimaryReplica(txnPoolNodeSet, 0) relucatantNode = pr.node # Count sent instance changes of all nodes sentInstChanges = {} instChngMethodName = ViewChanger.sendInstanceChange.__name__ - for n in nodeSet: + for n in txnPoolNodeSet: sentInstChanges[n.name] = n.view_changer.spylog.count(instChngMethodName) # Node reluctant to change view, never says master is degraded @@ -65,11 +65,11 @@ def test_view_change_on_quorum_of_master_degraded(nodeSet, looper, up, sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) # Check that view change happened for all nodes - waitForViewChange(looper, nodeSet, expectedViewNo=viewNo + 1) + waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) # All nodes except the reluctant node should have sent a view change and # thus must have called `sendInstanceChange` - for n in nodeSet: + for n in txnPoolNodeSet: if n.name != relucatantNode.name: assert n.view_changer.spylog.count(instChngMethodName) > \ sentInstChanges.get(n.name, 0) @@ -77,7 +77,7 @@ def test_view_change_on_quorum_of_master_degraded(nodeSet, looper, up, assert n.view_changer.spylog.count(instChngMethodName) == \ sentInstChanges.get(n.name, 0) - ensureElectionsDone(looper=looper, nodes=nodeSet) - new_m_primary_node = get_master_primary_node(list(nodeSet.nodes.values())) + ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) + new_m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) assert m_primary_node.name != new_m_primary_node.name - ensure_all_nodes_have_same_data(looper, nodes=nodeSet) + ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) From 1b2cca1675a6ef98690fbb7cac308a86627f1425 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 20 Mar 2018 17:34:10 +0300 Subject: [PATCH 25/73] primary_selection dependency fix Signed-off-by: ArtObr --- plenum/test/primary_selection/conftest.py | 22 +++++++++++++++++++ .../test_recover_more_than_f_failure.py | 21 ------------------ 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/plenum/test/primary_selection/conftest.py b/plenum/test/primary_selection/conftest.py index 4a7e6daea7..957dbd6975 100644 --- a/plenum/test/primary_selection/conftest.py +++ b/plenum/test/primary_selection/conftest.py @@ -1,9 +1,11 @@ import pytest +from plenum.test.conftest import getValueFromModule from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.primary_selection.helper import check_newly_added_nodes, \ getPrimaryNodesIdxs from plenum.test.pool_transactions.helper import buildPoolClientAndWallet +from plenum.test.pool_transactions.conftest import nodeThetaAdded @pytest.fixture(scope="module") @@ -46,3 +48,23 @@ def stewardAndWalletForMasterNode(looper, poolTxnData, poolTxnStewardNames, looper.run(stewardClient.ensureConnectedToNodes()) return stewardClient, stewardWallet + + +@pytest.fixture(scope="module") +def checkpoint_size(tconf, request): + oldChkFreq = tconf.CHK_FREQ + oldLogSize = tconf.LOG_SIZE + oldMax3PCBatchSize = tconf.Max3PCBatchSize + + tconf.Max3PCBatchSize = 3 + tconf.CHK_FREQ = getValueFromModule(request, "CHK_FREQ", 2) + tconf.LOG_SIZE = 2 * tconf.CHK_FREQ + + def reset(): + tconf.CHK_FREQ = oldChkFreq + tconf.LOG_SIZE = oldLogSize + tconf.Max3PCBatchSize = oldMax3PCBatchSize + + request.addfinalizer(reset) + + return tconf.CHK_FREQ * tconf.Max3PCBatchSize diff --git a/plenum/test/primary_selection/test_recover_more_than_f_failure.py b/plenum/test/primary_selection/test_recover_more_than_f_failure.py index e373712fda..fe1bd3e56a 100644 --- a/plenum/test/primary_selection/test_recover_more_than_f_failure.py +++ b/plenum/test/primary_selection/test_recover_more_than_f_failure.py @@ -2,7 +2,6 @@ from stp_core.common.log import getlogger -from plenum.test.conftest import getValueFromModule from plenum.test.helper import stopNodes, waitForViewChange, \ sendReqsToNodesAndVerifySuffReplies from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data @@ -75,26 +74,6 @@ def stop_primary(looper, active_nodes): return stopped_node, active_nodes -@pytest.fixture(scope="module") -def checkpoint_size(tconf, request): - oldChkFreq = tconf.CHK_FREQ - oldLogSize = tconf.LOG_SIZE - oldMax3PCBatchSize = tconf.Max3PCBatchSize - - tconf.Max3PCBatchSize = 3 - tconf.CHK_FREQ = getValueFromModule(request, "CHK_FREQ", 2) - tconf.LOG_SIZE = 2 * tconf.CHK_FREQ - - def reset(): - tconf.CHK_FREQ = oldChkFreq - tconf.LOG_SIZE = oldLogSize - tconf.Max3PCBatchSize = oldMax3PCBatchSize - - request.addfinalizer(reset) - - return tconf.CHK_FREQ * tconf.Max3PCBatchSize - - def primary_replicas_iter(*nodes): for node in nodes: for replica in node.replicas: From cc152755d2090225bbc32b384882c52720491307 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 20 Mar 2018 17:51:52 +0300 Subject: [PATCH 26/73] Fixture fix for view_change Signed-off-by: ArtObr --- plenum/test/view_change/helper.py | 20 +++++++++---------- ...ew_change_2_of_4_nodes_with_new_primary.py | 8 ++++---- ...ew_change_2_of_4_nodes_with_non_primary.py | 8 ++++---- ...e_2_of_4_nodes_with_old_and_new_primary.py | 8 ++++---- ...ew_change_2_of_4_nodes_with_old_primary.py | 8 ++++---- .../slow_nodes/test_view_change_all_nodes.py | 8 ++++---- ...test_view_change_all_nodes_random_delay.py | 8 ++++---- .../slow_nodes/test_view_change_complex.py | 8 ++++---- ...t_view_change_gc_all_nodes_random_delay.py | 12 +++++------ 9 files changed, 44 insertions(+), 44 deletions(-) diff --git a/plenum/test/view_change/helper.py b/plenum/test/view_change/helper.py index d86db4a4a5..3ea2359158 100644 --- a/plenum/test/view_change/helper.py +++ b/plenum/test/view_change/helper.py @@ -278,12 +278,12 @@ def ensure_view_change_complete_by_primary_restart( return nodes -def view_change_in_between_3pc(looper, nodes, slow_nodes, wallet, client, +def view_change_in_between_3pc(looper, nodes, slow_nodes, wallet, client1, slow_delay=1, wait=None): - send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client, 4) + send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client1, 4) delay_3pc_messages(slow_nodes, 0, delay=slow_delay) - sendRandomRequests(wallet, client, 10) + sendRandomRequests(wallet, client1, 10) if wait: looper.runFor(wait) @@ -292,27 +292,27 @@ def view_change_in_between_3pc(looper, nodes, slow_nodes, wallet, client, reset_delays_and_process_delayeds(slow_nodes) sendReqsToNodesAndVerifySuffReplies( - looper, wallet, client, 5, total_timeout=30) + looper, wallet, client1, 5, total_timeout=30) send_reqs_to_nodes_and_verify_all_replies( - looper, wallet, client, 5, total_timeout=30) + looper, wallet, client1, 5, total_timeout=30) def view_change_in_between_3pc_random_delays( looper, nodes, slow_nodes, - wallet, - client, + wallet1, + client1, tconf, min_delay=0, max_delay=0): - send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client, 4) + send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 4) # max delay should not be more than catchup timeout. max_delay = max_delay or tconf.MIN_TIMEOUT_CATCHUPS_DONE_DURING_VIEW_CHANGE - 1 delay_3pc_messages(slow_nodes, 0, min_delay=min_delay, max_delay=max_delay) - sendRandomRequests(wallet, client, 10) + sendRandomRequests(wallet1, client1, 10) ensure_view_change_complete(looper, nodes, @@ -321,4 +321,4 @@ def view_change_in_between_3pc_random_delays( reset_delays_and_process_delayeds(slow_nodes) - send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client, 10) + send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 10) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py index 243e3a8e07..394f076e4a 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py @@ -8,7 +8,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the the first and the last non-primary node @@ -18,11 +18,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client) + wallet1, client1) def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary_long_delay( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the the first and the last non-primary node @@ -32,5 +32,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary_long_delay( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client, + wallet1, client1, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py index d3f057ba2f..bf765107d8 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py @@ -7,7 +7,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_non_primary( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for 2 of 4 node (2>f). - Both nodes are non-primary for master neither in this nor the next view @@ -15,11 +15,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_non_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client) + wallet1, client1) def test_view_change_in_between_3pc_2_of_4_nodes_with_non_primary_long_delay( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for 2 of 4 node (2>f). - Both nodes are non-primary for master neither in this nor the next view @@ -27,5 +27,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_non_primary_long_delay( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client, + wallet1, client1, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py index e4d735bd88..e6832d10e2 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py @@ -8,7 +8,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_old_and_new_primary( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow both current and next primaries @@ -16,11 +16,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_old_and_new_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client) + wallet1, client1) def test_view_change_in_between_3pc_2_of_4_nodes_with_old_and_new_primary_long_delay( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow both current and next primaries @@ -28,5 +28,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_old_and_new_primary_long_d """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client, + wallet1, client1, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py index dd267dc28c..4fdb869599 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py @@ -8,7 +8,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_old_primary( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the current Primary node and the last non-primary node (it will not @@ -17,11 +17,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_old_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client) + wallet1, client1) def test_view_change_in_between_3pc_2_of_4_nodes_with_old_primary_long_delay( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the current Primary node and the last non-primary node (it will not @@ -30,5 +30,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_old_primary_long_delay( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client, + wallet1, client1, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py index 9e791e56cc..0cbc010156 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py @@ -2,22 +2,22 @@ def test_view_change_in_between_3pc_all_nodes(txnPoolNodeSet, looper, - wallet1, client): + wallet1, client1): """ - Slow processing 3PC messages for all nodes - do view change """ view_change_in_between_3pc(looper, txnPoolNodeSet, txnPoolNodeSet, wallet1, - client) + client1) def test_view_change_in_between_3pc_all_nodes_long_delay( - txnPoolNodeSet, looper, wallet1, client): + txnPoolNodeSet, looper, wallet1, client1): """ - Slow processing 3PC messages for all nodes - do view change """ view_change_in_between_3pc(looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, client, + wallet1, client1, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py index 1ac13aee44..153fe5fea7 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py @@ -4,23 +4,23 @@ def test_view_change_in_between_3pc_all_nodes_random_delays( - txnPoolNodeSet, tconf, looper, wallet1, client): + txnPoolNodeSet, tconf, looper, wallet1, client1): """ - Slow processing 3PC messages for all nodes randomly - do view change """ view_change_in_between_3pc_random_delays(looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, client, tconf) + wallet1, client1, tconf) def test_view_change_in_between_3pc_all_nodes_random_delays_long_delay( - txnPoolNodeSet, looper, wallet1, client, tconf): + txnPoolNodeSet, looper, wallet1, client1, tconf): """ - Slow processing 3PC messages for all nodes randomly - do view change """ view_change_in_between_3pc_random_delays(looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, client, tconf, + wallet1, client1, tconf, min_delay=5) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_complex.py b/plenum/test/view_change/slow_nodes/test_view_change_complex.py index cb052e90ca..ddfc8bb274 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_complex.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_complex.py @@ -5,7 +5,7 @@ def test_view_change_complex( - txnPoolNodeSet, looper, wallet1, client, tconf): + txnPoolNodeSet, looper, wallet1, client1, tconf): """ - Complex scenario with multiple view changes """ @@ -36,7 +36,7 @@ def test_view_change_complex( txnPoolNodeSet, txnPoolNodeSet, wallet1, - client, + client1, tconf, min_delay=0, max_delay=10) @@ -45,7 +45,7 @@ def test_view_change_complex( txnPoolNodeSet, txnPoolNodeSet, wallet1, - client, + client1, tconf, min_delay=1, max_delay=5) @@ -54,6 +54,6 @@ def test_view_change_complex( txnPoolNodeSet, txnPoolNodeSet, wallet1, - client, + client1, tconf, min_delay=5) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py index 7638856adf..2366130d83 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py @@ -26,14 +26,14 @@ def check_nodes_requests_size(nodes, size): def test_view_change_gc_in_between_3pc_all_nodes_delays( - looper, txnPoolNodeSet, wallet1, client): + looper, txnPoolNodeSet, wallet1, client1): """ Test that garbage collector compares the whole 3PC key (viewNo, ppSeqNo) and does not remove messages from node's queues that have higher viewNo than last ordered one even if their ppSeqNo are less or equal """ - numNodes = len(client.nodeReg) + numNodes = len(client1.nodeReg) viewNo = checkViewNoForNodes(txnPoolNodeSet) # 1 send two messages one by one separately to make @@ -43,8 +43,8 @@ def test_view_change_gc_in_between_3pc_all_nodes_delays( # for master instances only cause non-master ones have # specific logic of its management which we don't care in # the test, see Replica::_setup_for_non_master) - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client, 1) - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client, 1) + send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 1) + send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 1) last_ordered_3pc = (viewNo, 2) check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) @@ -72,7 +72,7 @@ def test_view_change_gc_in_between_3pc_all_nodes_delays( delay_3pc_messages(txnPoolNodeSet, 1, delay=propagationTimeout * 2) - requests = sendRandomRequests(wallet1, client, 1) + requests = sendRandomRequests(wallet1, client1, 1) def checkPrePrepareSentAtLeastByPrimary(): for node in txnPoolNodeSet: @@ -102,7 +102,7 @@ def checkPrePrepareSentAtLeastByPrimary(): # -> they should be ordered # -> last_ordered_3pc = (+2, 1) reset_delays_and_process_delayeds(txnPoolNodeSet) - waitForSufficientRepliesForRequests(looper, client, + waitForSufficientRepliesForRequests(looper, client1, requests=requests) checkViewNoForNodes(txnPoolNodeSet, viewNo) From e23b2842b8c2916b6e75bb52fd7eab3d9a3c18d9 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 20 Mar 2018 20:09:31 +0300 Subject: [PATCH 27/73] View change test fix Signed-off-by: ArtObr --- .../view_change/test_view_change_start_without_primary.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/plenum/test/view_change/test_view_change_start_without_primary.py b/plenum/test/view_change/test_view_change_start_without_primary.py index a95c58a6f9..41c7d73cfd 100644 --- a/plenum/test/view_change/test_view_change_start_without_primary.py +++ b/plenum/test/view_change/test_view_change_start_without_primary.py @@ -16,7 +16,9 @@ def test_view_change_without_primary(txnPoolNodeSet, looper, timeout = waits.expectedPoolElectionTimeout(len(txnPoolNodeSet)) + patched_view_change_timeout - checkProtocolInstanceSetup(looper=looper, nodes=others, retryWait=1, + #looper.runFor(40) + + checkProtocolInstanceSetup(looper=looper, nodes=txnPoolNodeSet, retryWait=1, customTimeout=timeout, numInstances=getRequiredInstances(len(txnPoolNodeSet))) From 8b233bd305cbd2964eefdbb00ddd3527cb2f6c7f Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 21 Mar 2018 07:47:50 +0300 Subject: [PATCH 28/73] Skips for merkle proof tests Signed-off-by: ArtObr --- plenum/test/test_verif_merkle_proof.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plenum/test/test_verif_merkle_proof.py b/plenum/test/test_verif_merkle_proof.py index 60f5a968a8..250830a454 100644 --- a/plenum/test/test_verif_merkle_proof.py +++ b/plenum/test/test_verif_merkle_proof.py @@ -6,11 +6,13 @@ from plenum.test.test_client import TestClient +@pytest.skip('Wait for decision') def testMerkleProofForFirstLeaf(client1: TestClient, replied1): replies = client1.getRepliesFromAllNodes(*replied1.key).values() assert Client.verifyMerkleProof(*replies) +@pytest.skip('Wait for decision') def testMerkleProofForNonFirstLeaf( looper, txnPoolNodeSet, wallet1, client1, replied1): req2 = sendRandomRequest(wallet1, client1) From db40c43960b5a78ac8192d50caf64e79a48f11bd Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 21 Mar 2018 07:47:50 +0300 Subject: [PATCH 29/73] Skips for merkle proof tests Signed-off-by: ArtObr --- plenum/test/test_verif_merkle_proof.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/plenum/test/test_verif_merkle_proof.py b/plenum/test/test_verif_merkle_proof.py index 60f5a968a8..ff67c5c37a 100644 --- a/plenum/test/test_verif_merkle_proof.py +++ b/plenum/test/test_verif_merkle_proof.py @@ -6,11 +6,13 @@ from plenum.test.test_client import TestClient +@pytest.mark.skip('Wait for decision') def testMerkleProofForFirstLeaf(client1: TestClient, replied1): replies = client1.getRepliesFromAllNodes(*replied1.key).values() assert Client.verifyMerkleProof(*replies) +@pytest.mark.skip('Wait for decision') def testMerkleProofForNonFirstLeaf( looper, txnPoolNodeSet, wallet1, client1, replied1): req2 = sendRandomRequest(wallet1, client1) From b2219ee77c38b964ed6896379c251b04bb7c4e49 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 21 Mar 2018 10:29:01 +0300 Subject: [PATCH 30/73] Remove sendRandomRequest of old client. Part 1. Signed-off-by: ArtObr --- .../monitoring/test_warn_unordered_log_msg.py | 26 +++++--- ...plus_2_nodes_but_not_primary_off_and_on.py | 66 +++++++++++++------ ...us_2_nodes_including_primary_off_and_on.py | 62 ++++++++++++----- .../pool_transactions/test_suspend_node.py | 2 +- .../test_replica_reject_same_pre_prepare.py | 22 ++++--- plenum/test/test_node_request.py | 66 ++++++++----------- .../validator_info/test_validator_info.py | 13 ++-- 7 files changed, 155 insertions(+), 102 deletions(-) diff --git a/plenum/test/monitoring/test_warn_unordered_log_msg.py b/plenum/test/monitoring/test_warn_unordered_log_msg.py index a1c3b3dcb1..c68dde7895 100644 --- a/plenum/test/monitoring/test_warn_unordered_log_msg.py +++ b/plenum/test/monitoring/test_warn_unordered_log_msg.py @@ -4,7 +4,7 @@ from plenum.test.test_node import getNonPrimaryReplicas from stp_core.common.log import getlogger from plenum.test.helper import sendRandomRequest, \ - waitForSufficientRepliesForRequests + waitForSufficientRepliesForRequests, sdk_send_random_and_check nodeCount = 4 logger = getlogger() @@ -12,13 +12,15 @@ # noinspection PyIncorrectDocstring def test_working_has_no_warn_log_msg(looper, txnPoolNodeSet, - wallet1, client1, patch_monitors): + sdk_pool_handle, sdk_wallet_client, patch_monitors): monitor = txnPoolNodeSet[0].monitor assert no_any_warn(*txnPoolNodeSet) for i in range(monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM): - req = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) looper.runFor(monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC) assert no_any_warn(*txnPoolNodeSet) @@ -27,8 +29,8 @@ def test_working_has_no_warn_log_msg(looper, txnPoolNodeSet, # noinspection PyIncorrectDocstring def test_slow_node_has_warn_unordered_log_msg(looper, txnPoolNodeSet, - wallet1, - client1, + sdk_pool_handle, + sdk_wallet_client, patch_monitors): npr = getNonPrimaryReplicas(txnPoolNodeSet, 0)[0] slow_node = npr.node @@ -42,8 +44,10 @@ def test_slow_node_has_warn_unordered_log_msg(looper, 'all nodes do not have warnings before test' for i in range(monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM): - req = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) looper.runFor(monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC) others = [node for node in txnPoolNodeSet if node.name != slow_node.name] @@ -55,8 +59,10 @@ def test_slow_node_has_warn_unordered_log_msg(looper, ordered_requests_keys_len_before = len(monitor.ordered_requests_keys) # wait at least windows time looper.runFor(monitor.WARN_NOT_PARTICIPATING_WINDOW_MINS * 60) - req = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) assert no_any_warn(*others), 'others do not have warning' assert no_last_warn(slow_node), \ 'the last call of warn_has_lot_unordered_requests returned False ' \ diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py index a411b1f4cf..d6cfea9a81 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py @@ -1,13 +1,16 @@ +import pytest + +from plenum.common.exceptions import PoolLedgerTimeoutException from plenum.test import waits -from plenum.test.helper import checkViewNoForNodes, sendRandomRequest, \ - waitForSufficientRepliesForRequests, \ - verify_request_not_replied_and_not_ordered +from plenum.test.helper import checkViewNoForNodes, \ + sdk_send_random_and_check, sdk_send_random_requests, sdk_get_replies, \ + sdk_check_reply from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.test_node import ensureElectionsDone, getRequiredInstances from plenum.test.view_change.helper import start_stopped_node -TestRunningTimeLimitSec = 200 +TestRunningTimeLimitSec = 400 nodeCount = 5 @@ -19,56 +22,79 @@ def stop_node(node_to_stop, looper, pool_nodes): def test_quorum_after_f_plus_2_nodes_but_not_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, - txnPoolNodeSet, wallet1, client1): + txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): nodes = txnPoolNodeSet - request1 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request1]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) stop_node(nodes[4], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:4], expectedViewNo=0) - request2 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request2]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) stop_node(nodes[3], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:3], expectedViewNo=0) - request3 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request3, looper, client1, nodes) + sdk_reqs3 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs3) + sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:2], expectedViewNo=0) - request4 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request4, looper, client1, nodes) + sdk_reqs4 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs4) + sdk_check_reply(req_res[0]) nodes[4] = start_stopped_node(nodes[4], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:2] + nodes[4:], expectedViewNo=0) - request5 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request5, looper, client1, nodes) + sdk_reqs5 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs5) + sdk_check_reply(req_res[0]) nodes[3] = start_stopped_node(nodes[3], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[:2] + nodes[3:], numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes[:2] + nodes[3:], expectedViewNo=0) - request6 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests( - looper, client1, requests=[request3, request4, request5, request6]) + sdk_reqs6 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + sdk_get_replies(looper, [sdk_reqs3[0], sdk_reqs4[0], sdk_reqs5[0], sdk_reqs6[0]]) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes, expectedViewNo=0) - request7 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request7]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py index a0a7f754fe..4e5e22027e 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py @@ -1,7 +1,10 @@ +import pytest + +from plenum.common.exceptions import PoolLedgerTimeoutException from plenum.test import waits from plenum.test.helper import waitForViewChange, checkViewNoForNodes, \ - sendRandomRequest, waitForSufficientRepliesForRequests, \ - verify_request_not_replied_and_not_ordered + sdk_send_random_and_check, sdk_send_random_requests, sdk_get_replies, \ + sdk_check_reply from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.test_node import ensureElectionsDone, getRequiredInstances @@ -19,55 +22,80 @@ def stop_node(node_to_stop, looper, pool_nodes): def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, - txnPoolNodeSet, wallet1, client1): + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client): nodes = txnPoolNodeSet - request1 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request1]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) stop_node(nodes[0], looper, nodes) waitForViewChange(looper, nodes[1:], expectedViewNo=1) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) - request2 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request2]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) stop_node(nodes[1], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[2:], expectedViewNo=1) - request3 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request3, looper, client1, nodes) + sdk_reqs3 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs3) + sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) - request4 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request4, looper, client1, nodes) + sdk_reqs4 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs4) + sdk_check_reply(req_res[0]) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) - request5 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request5, looper, client1, nodes) + sdk_reqs5 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs5) + sdk_check_reply(req_res[0]) nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes[1:], expectedViewNo=1) - request6 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request6]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) nodes[0] = start_stopped_node(nodes[0], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes, expectedViewNo=1) - request7 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request7]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) diff --git a/plenum/test/pool_transactions/test_suspend_node.py b/plenum/test/pool_transactions/test_suspend_node.py index fd610def2e..2d582e8996 100644 --- a/plenum/test/pool_transactions/test_suspend_node.py +++ b/plenum/test/pool_transactions/test_suspend_node.py @@ -45,7 +45,7 @@ def testStewardSuspendsNode(looper, txnPoolNodeSet, looper.run(eventually(checkNodeNotInNodeReg, client, newNode.name)) # Check a client can send request and receive replies - req = sendRandomRequest(newStewardWallet, newSteward) + # req = sendRandomRequest(newStewardWallet, newSteward) waitForSufficientRepliesForRequests(looper, newSteward, requests=[req]) diff --git a/plenum/test/replica/test_replica_reject_same_pre_prepare.py b/plenum/test/replica/test_replica_reject_same_pre_prepare.py index 0fb79c6872..30d3b9201d 100644 --- a/plenum/test/replica/test_replica_reject_same_pre_prepare.py +++ b/plenum/test/replica/test_replica_reject_same_pre_prepare.py @@ -9,7 +9,8 @@ from plenum.test import waits from plenum.test.helper import checkPrePrepareReqSent, \ checkPrePrepareReqRecvd, \ - checkPrepareReqSent, check_sufficient_replies_received, sendRandomRequest + checkPrepareReqSent, check_sufficient_replies_received, sendRandomRequest, sdk_send_random_requests, \ + sdk_json_to_request_object, sdk_get_replies from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica whitelist = ['doing nothing for now', @@ -20,7 +21,7 @@ # noinspection PyIncorrectDocstring -def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1): +def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and @@ -39,7 +40,11 @@ def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1 for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(delay=c_delay, instId=1)) - request1 = sendRandomRequest(wallet1, client1) + req1 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1)[0] + request1 = sdk_json_to_request_object(req1[0]) for npr in nonPrimaryReplicas: looper.run(eventually(checkPrepareReqSent, npr, @@ -68,7 +73,11 @@ def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1 "one...") primaryRepl._lastPrePrepareSeqNo -= 1 view_no = primaryRepl.viewNo - request2 = sendRandomRequest(wallet1, client1) + request2 = sdk_json_to_request_object( + sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1)[0][0]) timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=timeout)) @@ -118,8 +127,5 @@ def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1 timeout=timeout)) timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + c_delay - result1 = \ - looper.run(eventually(check_sufficient_replies_received, - client1, request1.identifier, request1.reqId, - retryWait=1, timeout=timeout)) + result1 = sdk_get_replies(looper, [req1])[0][1] logger.debug("request {} gives result {}".format(request1, result1)) diff --git a/plenum/test/test_node_request.py b/plenum/test/test_node_request.py index 75938e78cb..fa1bf9de7d 100644 --- a/plenum/test/test_node_request.py +++ b/plenum/test/test_node_request.py @@ -14,7 +14,7 @@ from plenum.test.helper import setupNodesAndClient, \ sendRandomRequest, setupClient, \ assertLength, addNodeBack, waitForSufficientRepliesForRequests, \ - getPendingRequestsForReplica, checkRequestReturnedToNode + getPendingRequestsForReplica, checkRequestReturnedToNode, sdk_send_random_and_check from plenum.test.profiler import profile_this from plenum.test.test_node import TestNode, TestNodeSet, checkPoolReady, \ ensureElectionsDone, genNodeReg, prepareNodeSet @@ -23,31 +23,29 @@ logger = getlogger() -def testReqExecWhenReturnedByMaster(tdir_for_func, tconf_for_func): - with TestNodeSet(tconf_for_func, count=4, tmpdir=tdir_for_func) as nodeSet: - with Looper(nodeSet) as looper: - client1, wallet1 = setupNodesAndClient(looper, - nodeSet, - tmpdir=tdir_for_func) - req = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, - requests=[req]) - - async def chk(): - for node in nodeSet: - entries = node.spylog.getAll( - node.processOrdered.__name__) - for entry in entries: - arg = entry.params['ordered'] - result = entry.result - if arg.instId == node.instances.masterId: - assert result - else: - assert result is False - - timeout = waits.expectedOrderingTime( - nodeSet.nodes['Alpha'].instances.count) - looper.run(eventually(chk, timeout=timeout)) +def testReqExecWhenReturnedByMaster(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client): + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) + + async def chk(): + for node in txnPoolNodeSet: + entries = node.spylog.getAll( + node.processOrdered.__name__) + for entry in entries: + arg = entry.params['ordered'] + result = entry.result + if arg.instId == node.instances.masterId: + assert result + else: + assert result is False + + timeout = waits.expectedOrderingTime( + txnPoolNodeSet[0].instances.count) + looper.run(eventually(chk, timeout=timeout)) @pytest.mark.skip('Since primary is selected immediately now') @@ -78,8 +76,8 @@ def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): checkPoolReady(looper=looper, nodes=nodeSet) - client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) - request = sendRandomRequest(wal, client1) + #client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) + #request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 @@ -134,18 +132,6 @@ async def checkIfPropagateRecvdFromNode(recvrNode: TestNode, assert senderNode.name in recvrNode.requests[key].propagates -def testClientSendingSameRequestAgainBeforeFirstIsProcessed(looper, - txnPoolNodeSet, - wallet1, - client1): - size = len(client1.inBox) - req = sendRandomRequest(wallet1, client1) - client1.submitReqs(req) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) - # Only REQACK will be sent twice by the node but not REPLY - assert len(client1.inBox) == size + 12 - - def snapshotStats(*nodes): return {n.name: n.nodestack.stats.copy() for n in nodes} diff --git a/plenum/test/validator_info/test_validator_info.py b/plenum/test/validator_info/test_validator_info.py index 23fb486b67..297215a77c 100644 --- a/plenum/test/validator_info/test_validator_info.py +++ b/plenum/test/validator_info/test_validator_info.py @@ -13,7 +13,7 @@ from plenum.server.validator_info_tool import ValidatorNodeInfoTool from plenum.test import waits from plenum.test.helper import waitForSufficientRepliesForRequests, \ - sendRandomRequest, check_sufficient_replies_received + sendRandomRequest, check_sufficient_replies_received, sdk_send_random_and_check # noinspection PyUnresolvedReferences from plenum.test.node_catchup.helper import ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected @@ -246,14 +246,15 @@ def read_wrapped(txn_type): @pytest.fixture def write_txn_and_get_latest_info(txnPoolNodesLooper, - client_and_wallet, + sdk_pool_handle, + sdk_wallet_client, patched_dump_info_period, info_path): - client, wallet = client_and_wallet - def write_wrapped(): - req = sendRandomRequest(wallet, client) - waitForSufficientRepliesForRequests(txnPoolNodesLooper, client, requests=[req]) + sdk_send_random_and_check(txnPoolNodesLooper, range(nodeCount), + sdk_pool_handle, + sdk_wallet_client, + 1) txnPoolNodesLooper.runFor(patched_dump_info_period) return load_info(info_path) From f508ca4843adc82f0be6bb6e55b8d2c1f5b43c13 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 21 Mar 2018 10:49:35 +0300 Subject: [PATCH 31/73] Review changes Signed-off-by: ArtObr --- plenum/test/client/test_client.py | 83 -------------------------- plenum/test/test_verif_merkle_proof.py | 21 ------- 2 files changed, 104 deletions(-) delete mode 100644 plenum/test/test_verif_merkle_proof.py diff --git a/plenum/test/client/test_client.py b/plenum/test/client/test_client.py index 040caeffa8..fa293d6e97 100644 --- a/plenum/test/client/test_client.py +++ b/plenum/test/client/test_client.py @@ -222,86 +222,3 @@ def chk(): responseTimeout = waits.expectedTransactionExecutionTime(nodeCount) looper.run(eventually(chk, retryWait=1, timeout=responseTimeout)) - - -# noinspection PyIncorrectDocstring -@pytest.mark.skip('Unskip and complete this test while sdk integration') -def testReplyMatchesRequest(looper, txnPoolNodeSet, client_tdir): - ''' - This tests does check following things: - - wallet works correctly when used by multiple clients - - clients do receive responses for exactly the same request they sent - ''' - - def makeClient(id): - client, wallet = genTestClient(txnPoolNodeSet, - tmpdir=client_tdir, - name="client-{}".format(id)) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) - return client, wallet - - # creating clients - numOfClients = 3 - numOfRequests = 1 - - clients = set() - sharedWallet = None - for i in range(numOfClients): - client, wallet = makeClient(i) - if sharedWallet is None: - sharedWallet = wallet - clients.add(client) - - for i in range(1, numOfRequests + 1): - # sending requests - requests = {} - for client in clients: - op = randomOperation() - req = sharedWallet.signOp(op) - - request = client.submitReqs(req)[0][0] - requests[client] = (request.reqId, request.operation['amount']) - - # checking results - responseTimeout = waits.expectedTransactionExecutionTime(nodeCount) - for client, (reqId, sentAmount) in requests.items(): - looper.run(eventually(checkResponseRecvdFromNodes, - client, - nodeCount, - reqId, - retryWait=1, - timeout=responseTimeout)) - - print("Expected amount for request {} is {}". - format(reqId, sentAmount)) - - # This looks like it fails on some python versions - # replies = [r[0]['result']['amount'] - # for r in client.inBox - # if r[0]['op'] == 'REPLY' - # and r[0]['result']['reqId'] == reqId] - - replies = [] - for r in client.inBox: - if r[0]['op'] == 'REPLY' and r[0]['result']['reqId'] == reqId: - if 'amount' not in r[0]['result']: - logger.debug('{} cannot find amount in {}'. - format(client, r[0]['result'])) - replies.append(r[0]['result']['amount']) - - assert all(replies[0] == r for r in replies) - assert replies[0] == sentAmount - - -@pytest.mark.skip('Unskip and complete this test while sdk integration') -def testReplyReceivedOnlyByClientWhoSentRequest(looper, txnPoolNodeSet, client_tdir, - client1, wallet1): - newClient, _ = genTestClient(txnPoolNodeSet, tmpdir=client_tdir) - looper.add(newClient) - looper.run(newClient.ensureConnectedToNodes()) - client1InboxSize = len(client1.inBox) - newClientInboxSize = len(newClient.inBox) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, newClient, 1) - assert len(client1.inBox) == client1InboxSize - assert len(newClient.inBox) > newClientInboxSize diff --git a/plenum/test/test_verif_merkle_proof.py b/plenum/test/test_verif_merkle_proof.py deleted file mode 100644 index ff67c5c37a..0000000000 --- a/plenum/test/test_verif_merkle_proof.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from plenum.client.client import Client -from plenum.test.helper import waitForSufficientRepliesForRequests, \ - sendRandomRequest -from plenum.test.test_client import TestClient - - -@pytest.mark.skip('Wait for decision') -def testMerkleProofForFirstLeaf(client1: TestClient, replied1): - replies = client1.getRepliesFromAllNodes(*replied1.key).values() - assert Client.verifyMerkleProof(*replies) - - -@pytest.mark.skip('Wait for decision') -def testMerkleProofForNonFirstLeaf( - looper, txnPoolNodeSet, wallet1, client1, replied1): - req2 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[req2]) - replies = client1.getRepliesFromAllNodes(*req2.key).values() - assert Client.verifyMerkleProof(*replies) From d1946db0dfc6791f14cf11e79091c964bb20c80f Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 21 Mar 2018 10:49:35 +0300 Subject: [PATCH 32/73] Review changes Signed-off-by: ArtObr --- plenum/test/client/test_client.py | 83 ------------------------ plenum/test/input_validation/conftest.py | 0 plenum/test/test_verif_merkle_proof.py | 21 ------ 3 files changed, 104 deletions(-) delete mode 100644 plenum/test/input_validation/conftest.py delete mode 100644 plenum/test/test_verif_merkle_proof.py diff --git a/plenum/test/client/test_client.py b/plenum/test/client/test_client.py index 040caeffa8..fa293d6e97 100644 --- a/plenum/test/client/test_client.py +++ b/plenum/test/client/test_client.py @@ -222,86 +222,3 @@ def chk(): responseTimeout = waits.expectedTransactionExecutionTime(nodeCount) looper.run(eventually(chk, retryWait=1, timeout=responseTimeout)) - - -# noinspection PyIncorrectDocstring -@pytest.mark.skip('Unskip and complete this test while sdk integration') -def testReplyMatchesRequest(looper, txnPoolNodeSet, client_tdir): - ''' - This tests does check following things: - - wallet works correctly when used by multiple clients - - clients do receive responses for exactly the same request they sent - ''' - - def makeClient(id): - client, wallet = genTestClient(txnPoolNodeSet, - tmpdir=client_tdir, - name="client-{}".format(id)) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) - return client, wallet - - # creating clients - numOfClients = 3 - numOfRequests = 1 - - clients = set() - sharedWallet = None - for i in range(numOfClients): - client, wallet = makeClient(i) - if sharedWallet is None: - sharedWallet = wallet - clients.add(client) - - for i in range(1, numOfRequests + 1): - # sending requests - requests = {} - for client in clients: - op = randomOperation() - req = sharedWallet.signOp(op) - - request = client.submitReqs(req)[0][0] - requests[client] = (request.reqId, request.operation['amount']) - - # checking results - responseTimeout = waits.expectedTransactionExecutionTime(nodeCount) - for client, (reqId, sentAmount) in requests.items(): - looper.run(eventually(checkResponseRecvdFromNodes, - client, - nodeCount, - reqId, - retryWait=1, - timeout=responseTimeout)) - - print("Expected amount for request {} is {}". - format(reqId, sentAmount)) - - # This looks like it fails on some python versions - # replies = [r[0]['result']['amount'] - # for r in client.inBox - # if r[0]['op'] == 'REPLY' - # and r[0]['result']['reqId'] == reqId] - - replies = [] - for r in client.inBox: - if r[0]['op'] == 'REPLY' and r[0]['result']['reqId'] == reqId: - if 'amount' not in r[0]['result']: - logger.debug('{} cannot find amount in {}'. - format(client, r[0]['result'])) - replies.append(r[0]['result']['amount']) - - assert all(replies[0] == r for r in replies) - assert replies[0] == sentAmount - - -@pytest.mark.skip('Unskip and complete this test while sdk integration') -def testReplyReceivedOnlyByClientWhoSentRequest(looper, txnPoolNodeSet, client_tdir, - client1, wallet1): - newClient, _ = genTestClient(txnPoolNodeSet, tmpdir=client_tdir) - looper.add(newClient) - looper.run(newClient.ensureConnectedToNodes()) - client1InboxSize = len(client1.inBox) - newClientInboxSize = len(newClient.inBox) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, newClient, 1) - assert len(client1.inBox) == client1InboxSize - assert len(newClient.inBox) > newClientInboxSize diff --git a/plenum/test/input_validation/conftest.py b/plenum/test/input_validation/conftest.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/plenum/test/test_verif_merkle_proof.py b/plenum/test/test_verif_merkle_proof.py deleted file mode 100644 index ff67c5c37a..0000000000 --- a/plenum/test/test_verif_merkle_proof.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from plenum.client.client import Client -from plenum.test.helper import waitForSufficientRepliesForRequests, \ - sendRandomRequest -from plenum.test.test_client import TestClient - - -@pytest.mark.skip('Wait for decision') -def testMerkleProofForFirstLeaf(client1: TestClient, replied1): - replies = client1.getRepliesFromAllNodes(*replied1.key).values() - assert Client.verifyMerkleProof(*replies) - - -@pytest.mark.skip('Wait for decision') -def testMerkleProofForNonFirstLeaf( - looper, txnPoolNodeSet, wallet1, client1, replied1): - req2 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[req2]) - replies = client1.getRepliesFromAllNodes(*req2.key).values() - assert Client.verifyMerkleProof(*replies) From 6cf0553385e472e20f63c8993dc8efcdac516955 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Thu, 22 Mar 2018 18:06:58 +0300 Subject: [PATCH 33/73] Deletion of sendRandomRequest. Part 2. Signed-off-by: ArtObr --- plenum/test/client/test_client.py | 74 +------ plenum/test/client/test_client_retry.py | 190 ------------------ .../test/client/test_state_proof_verified.py | 33 --- plenum/test/helper.py | 4 - .../monitoring/test_warn_unordered_log_msg.py | 27 ++- ...plus_2_nodes_but_not_primary_off_and_on.py | 66 ++++-- ...us_2_nodes_including_primary_off_and_on.py | 62 ++++-- .../pool_transactions/test_suspend_node.py | 5 +- .../test_replica_reject_same_pre_prepare.py | 22 +- plenum/test/test_node_request.py | 83 +++----- .../validator_info/test_validator_info.py | 15 +- .../test_queueing_req_from_future_view.py | 14 +- 12 files changed, 169 insertions(+), 426 deletions(-) delete mode 100644 plenum/test/client/test_client_retry.py delete mode 100644 plenum/test/client/test_state_proof_verified.py diff --git a/plenum/test/client/test_client.py b/plenum/test/client/test_client.py index fa293d6e97..55e30c6257 100644 --- a/plenum/test/client/test_client.py +++ b/plenum/test/client/test_client.py @@ -10,10 +10,9 @@ from plenum.common.types import f from plenum.server.node import Node from plenum.test import waits -from plenum.test.helper import checkResponseCorrectnessFromNodes, \ - randomOperation, checkLastClientReqForNode, getRepliesFromClientInbox, \ - sendRandomRequest, waitForSufficientRepliesForRequests, assertLength, \ - sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import randomOperation, \ + checkLastClientReqForNode, getRepliesFromClientInbox, \ + waitForSufficientRepliesForRequests, assertLength from plenum.test.test_client import genTestClient @@ -32,28 +31,6 @@ logger = getlogger() -def checkResponseRecvdFromNodes(client, expectedCount: int, - expectedReqId: int): - # Checks if the client has unique `expectedCount` number of REQACKs and - # REPLYs from nodes. It ignores more than one REQACK or REPLY since a - # client might be retrying - acks = set() - replies = set() - for (resp, nodeNm) in client.inBox: - op = resp.get(OP_FIELD_NAME) - if op == REPLY: - reqId = resp.get(f.RESULT.nm, {}).get(f.REQ_ID.nm) - coll = replies - elif op == REQACK: - reqId = resp.get(f.REQ_ID.nm) - coll = acks - else: - continue - if reqId == expectedReqId: - coll.add(nodeNm) - assert len(replies) == len(acks) == expectedCount - - # noinspection PyIncorrectDocstring def testClientShouldNotBeAbleToConnectToNodesNodeStack(pool): """ @@ -149,51 +126,6 @@ def chk(): looper.run(eventually(chk)) -# noinspection PyIncorrectDocstring -def testReplyWhenRepliesFromAllNodesAreSame(looper, client1, wallet1): - """ - When there are not faulty nodes, the client must get a reply from all the - nodes. - """ - request = sendRandomRequest(wallet1, client1) - responseTimeout = waits.expectedTransactionExecutionTime(nodeCount) - looper.run( - eventually(checkResponseRecvdFromNodes, client1, - nodeCount, request.reqId, - retryWait=1, timeout=responseTimeout)) - checkResponseCorrectnessFromNodes(client1.inBox, request.reqId, F) - - -# noinspection PyIncorrectDocstring -def testReplyWhenRepliesFromExactlyFPlusOneNodesAreSame(looper, - client1, - wallet1): - """ - When only :math:`f+1` replies from the nodes are matching, the client - would accept the reply - """ - request = sendRandomRequest(wallet1, client1) - # exactly f + 1 => (3) nodes have correct responses - # modify some (numOfResponses of type REPLY - (f + 1)) => 4 responses to - # have a different operations - responseTimeout = waits.expectedTransactionExecutionTime(nodeCount) - looper.run( - eventually(checkResponseRecvdFromNodes, client1, - nodeCount, request.reqId, - retryWait=1, timeout=responseTimeout)) - - replies = (msg for msg, frm in client1.inBox - if msg[OP_FIELD_NAME] == REPLY and - msg[f.RESULT.nm][f.REQ_ID.nm] == request.reqId) - - # change two responses to something different - for i in range(2): - msg = next(replies) - msg[f.RESULT.nm][f.SIG.nm] = str(i) + "Some random id" - - checkResponseCorrectnessFromNodes(client1.inBox, request.reqId, F) - - # noinspection PyIncorrectDocstring def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, client1, sent1): """ diff --git a/plenum/test/client/test_client_retry.py b/plenum/test/client/test_client_retry.py deleted file mode 100644 index 128fb12900..0000000000 --- a/plenum/test/client/test_client_retry.py +++ /dev/null @@ -1,190 +0,0 @@ -import types -from functools import partial - -import pytest -import time - -from stp_core.loop.eventually import eventually, eventuallyAll -from plenum.common.request import Request -from plenum.common.messages.node_messages import RequestNack, Reply -from plenum.test.helper import sendRandomRequest, checkReqAck, wait_for_replies -from plenum.test import waits - -whitelist = ['AlphaC unable to send message', ] - - -def testClientRetryRequestWhenAckNotReceived(looper, txnPoolNodeSet, client1, wallet1): - """ - The client gets disconnected from node say Alpha but does not know it. - It sends request to all nodes including Alpha, expects ACK and REPLY from - Alpha too, does not get it, so reconnects to Alpha and sends request again - and gets REPLY - """ - alpha = txnPoolNodeSet[0] - - skipped = False - origPr = alpha.processRequest - - def skipReqOnce(msg, remoteName): - nonlocal skipped - if isinstance(msg, Request) and not skipped: - skipped = True - return - origPr(msg, remoteName) - - alpha.clientMsgRouter.routes[Request] = skipReqOnce - - req = sendRandomRequest(wallet1, client1) - - def chkAcks(): - for node in txnPoolNodeSet: - if node != alpha: - checkReqAck(client1, node, *req.key) - else: - with pytest.raises(AssertionError): - checkReqAck(client1, node, *req.key) - - timeout = waits.expectedReqAckQuorumTime() - looper.run(eventually(chkAcks, retryWait=1, timeout=timeout)) - idr, reqId = req.key - wait_for_replies(looper, client1, idr, reqId, 4) - - -def testClientRetryRequestWhenReplyNotReceived(looper, txnPoolNodeSet, client1, - wallet1, tconf): - """ - A node say Alpha sends ACK but doesn't send REPLY. The client resends the - request and gets REPLY - """ - - alpha = txnPoolNodeSet[0] - skipped = False - origTrans = alpha.transmitToClient - - def skipReplyOnce(msg, remoteName): - nonlocal skipped - if isinstance(msg, Reply) and not skipped: - skipped = True - return - origTrans(msg, remoteName) - - alpha.transmitToClient = skipReplyOnce - req = sendRandomRequest(wallet1, client1) - coros = [partial(checkReqAck, client1, node, *req.key) for node in txnPoolNodeSet] - timeout = waits.expectedReqAckQuorumTime() - start = time.perf_counter() - looper.run(eventuallyAll(*coros, retryWait=.5, totalTimeout=timeout)) - idr, reqId = req.key - # Client should get only 3 replies till the retry timeout since one node - # is not sending any replies - wait_for_replies(looper, client1, idr, reqId, 3, - custom_timeout=tconf.CLIENT_REPLY_TIMEOUT - 1) - end = time.perf_counter() - # Client should wait till the retry timeout but after that should - # get the reply from the remaining node - looper.runFor(tconf.CLIENT_REPLY_TIMEOUT - (end - start)) - wait_for_replies(looper, client1, idr, reqId, 4) - - -def testClientNotRetryRequestWhenReqnackReceived(looper, txnPoolNodeSet, client1, wallet1): - """ - A node sends REQNACK. The client does not resend Request. - """ - - numOfNodes = len(txnPoolNodeSet) - - alpha = txnPoolNodeSet[0] - origProcReq = alpha.processRequest - origTrans = alpha.transmitToClient - - def nackReq(self, req, frm): - self.transmitToClient(RequestNack(*req.key, "testing"), frm) - - def onlyTransNack(msg, remoteName): - if not isinstance(msg, RequestNack): - return - origTrans(msg, remoteName) - - alpha.clientMsgRouter.routes[Request] = types.MethodType(nackReq, alpha) - alpha.transmitToClient = onlyTransNack - - totalResends = client1.spylog.count(client1.resendRequests.__name__) - req = sendRandomRequest(wallet1, client1) - - reqAckTimeout = waits.expectedReqAckQuorumTime() - executionTimeout = waits.expectedTransactionExecutionTime(numOfNodes) - - # Wait till ACK timeout - looper.runFor(reqAckTimeout + 1) - assert client1.spylog.count( - client1.resendRequests.__name__) == totalResends - - # Wait till REPLY timeout - retryTimeout = executionTimeout - reqAckTimeout + 1 - looper.runFor(retryTimeout) - - assert client1.spylog.count( - client1.resendRequests.__name__) == totalResends - idr, reqId = req.key - wait_for_replies(looper, client1, idr, reqId, 3) - - alpha.clientMsgRouter.routes[Request] = origProcReq - alpha.transmitToClient = origTrans - - -@pytest.fixture(scope="function") -def withFewerRetryReq(tconf, request): - oldRetryReplyCount = tconf.CLIENT_MAX_RETRY_REPLY - oldRetryReplyTimeout = tconf.CLIENT_REPLY_TIMEOUT - tconf.CLIENT_MAX_RETRY_REPLY = 3 - tconf.CLIENT_REPLY_TIMEOUT = 5 - - def reset(): - tconf.CLIENT_MAX_RETRY_REPLY = oldRetryReplyCount - tconf.CLIENT_REPLY_TIMEOUT = oldRetryReplyTimeout - - request.addfinalizer(reset) - return tconf - - -def testClientNotRetryingRequestAfterMaxTriesDone(looper, - txnPoolNodeSet, - client1, - wallet1, - withFewerRetryReq): - """ - A client sends Request to a node but the node never responds to client. - The client resends the request but only the number of times defined in its - configuration and no more - """ - - alpha = txnPoolNodeSet[0] - origTrans = alpha.transmitToClient - - def dontTransmitReply(msg, remoteName): - if isinstance(msg, Reply): - return - origTrans(msg, remoteName) - - alpha.transmitToClient = dontTransmitReply - - totalResends = client1.spylog.count(client1.resendRequests.__name__) - req = sendRandomRequest(wallet1, client1) - - # Wait for more than REPLY timeout - # +1 because we have to wait one more retry timeout to make sure what - # client cleaned his buffers (expectingAcksFor, expectingRepliesFor) - retryTime = withFewerRetryReq.CLIENT_REPLY_TIMEOUT * \ - (withFewerRetryReq.CLIENT_MAX_RETRY_REPLY + 1) - timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + retryTime - - looper.runFor(timeout) - - idr, reqId = req.key - wait_for_replies(looper, client1, idr, reqId, 3) - - assert client1.spylog.count(client1.resendRequests.__name__) == \ - (totalResends + withFewerRetryReq.CLIENT_MAX_RETRY_REPLY) - assert req.key not in client1.expectingAcksFor - assert req.key not in client1.expectingRepliesFor - alpha.transmitToClient = origTrans diff --git a/plenum/test/client/test_state_proof_verified.py b/plenum/test/client/test_state_proof_verified.py deleted file mode 100644 index 3f3568bb4a..0000000000 --- a/plenum/test/client/test_state_proof_verified.py +++ /dev/null @@ -1,33 +0,0 @@ -from plenum.common.util import getMaxFailures -from plenum.test import waits -from plenum.test.helper import sendRandomRequest, \ - checkResponseCorrectnessFromNodes -from stp_core.loop.eventually import eventually - -nodeCount = 4 -nodes_wth_bls = 4 - -F = getMaxFailures(nodeCount) - - -def check_proved_reply_received(client, identifier, request_id): - client.take_one_quorumed = lambda a, b: False - reply, _ = client.getReply(identifier, request_id) - assert reply - - -def test_state_proof_checked_in_client_request(looper, txnPoolNodeSet, - client1, wallet1): - """ - Checks that client cat use state proofs instead of quorum for replies. - - To implement this test state proof is added to replies for - 'greeting' requests in TestNode. Parsing added to TestClient. - """ - request = sendRandomRequest(wallet1, client1) - responseTimeout = waits.expectedTransactionExecutionTime(nodeCount) - looper.run( - eventually(check_proved_reply_received, - client1, request.identifier, request.reqId, - retryWait=1, timeout=responseTimeout)) - checkResponseCorrectnessFromNodes(client1.inBox, request.reqId, F) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 0b58068065..b36f7ae3ee 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -317,10 +317,6 @@ def send_signed_requests(client: Client, signed_reqs: Sequence): return client.submitReqs(*signed_reqs)[0] -def sendRandomRequest(wallet: Wallet, client: Client): - return sendRandomRequests(wallet, client, 1)[0] - - def sendRandomRequests(wallet: Wallet, client: Client, count: int): logger.debug('Sending {} random requests'.format(count)) return send_signed_requests(client, diff --git a/plenum/test/monitoring/test_warn_unordered_log_msg.py b/plenum/test/monitoring/test_warn_unordered_log_msg.py index a1c3b3dcb1..b7270b41c9 100644 --- a/plenum/test/monitoring/test_warn_unordered_log_msg.py +++ b/plenum/test/monitoring/test_warn_unordered_log_msg.py @@ -3,8 +3,7 @@ from plenum.test.malicious_behaviors_node import delaysCommitProcessing from plenum.test.test_node import getNonPrimaryReplicas from stp_core.common.log import getlogger -from plenum.test.helper import sendRandomRequest, \ - waitForSufficientRepliesForRequests +from plenum.test.helper import sdk_send_random_and_check nodeCount = 4 logger = getlogger() @@ -12,13 +11,15 @@ # noinspection PyIncorrectDocstring def test_working_has_no_warn_log_msg(looper, txnPoolNodeSet, - wallet1, client1, patch_monitors): + sdk_pool_handle, sdk_wallet_client, patch_monitors): monitor = txnPoolNodeSet[0].monitor assert no_any_warn(*txnPoolNodeSet) for i in range(monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM): - req = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) looper.runFor(monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC) assert no_any_warn(*txnPoolNodeSet) @@ -27,8 +28,8 @@ def test_working_has_no_warn_log_msg(looper, txnPoolNodeSet, # noinspection PyIncorrectDocstring def test_slow_node_has_warn_unordered_log_msg(looper, txnPoolNodeSet, - wallet1, - client1, + sdk_pool_handle, + sdk_wallet_client, patch_monitors): npr = getNonPrimaryReplicas(txnPoolNodeSet, 0)[0] slow_node = npr.node @@ -42,8 +43,10 @@ def test_slow_node_has_warn_unordered_log_msg(looper, 'all nodes do not have warnings before test' for i in range(monitor.WARN_NOT_PARTICIPATING_UNORDERED_NUM): - req = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) looper.runFor(monitor.WARN_NOT_PARTICIPATING_MIN_DIFF_SEC) others = [node for node in txnPoolNodeSet if node.name != slow_node.name] @@ -55,8 +58,10 @@ def test_slow_node_has_warn_unordered_log_msg(looper, ordered_requests_keys_len_before = len(monitor.ordered_requests_keys) # wait at least windows time looper.runFor(monitor.WARN_NOT_PARTICIPATING_WINDOW_MINS * 60) - req = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) assert no_any_warn(*others), 'others do not have warning' assert no_last_warn(slow_node), \ 'the last call of warn_has_lot_unordered_requests returned False ' \ diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py index a411b1f4cf..d6cfea9a81 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py @@ -1,13 +1,16 @@ +import pytest + +from plenum.common.exceptions import PoolLedgerTimeoutException from plenum.test import waits -from plenum.test.helper import checkViewNoForNodes, sendRandomRequest, \ - waitForSufficientRepliesForRequests, \ - verify_request_not_replied_and_not_ordered +from plenum.test.helper import checkViewNoForNodes, \ + sdk_send_random_and_check, sdk_send_random_requests, sdk_get_replies, \ + sdk_check_reply from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.test_node import ensureElectionsDone, getRequiredInstances from plenum.test.view_change.helper import start_stopped_node -TestRunningTimeLimitSec = 200 +TestRunningTimeLimitSec = 400 nodeCount = 5 @@ -19,56 +22,79 @@ def stop_node(node_to_stop, looper, pool_nodes): def test_quorum_after_f_plus_2_nodes_but_not_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, - txnPoolNodeSet, wallet1, client1): + txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): nodes = txnPoolNodeSet - request1 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request1]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) stop_node(nodes[4], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:4], expectedViewNo=0) - request2 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request2]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) stop_node(nodes[3], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:3], expectedViewNo=0) - request3 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request3, looper, client1, nodes) + sdk_reqs3 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs3) + sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:2], expectedViewNo=0) - request4 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request4, looper, client1, nodes) + sdk_reqs4 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs4) + sdk_check_reply(req_res[0]) nodes[4] = start_stopped_node(nodes[4], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[:2] + nodes[4:], expectedViewNo=0) - request5 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request5, looper, client1, nodes) + sdk_reqs5 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs5) + sdk_check_reply(req_res[0]) nodes[3] = start_stopped_node(nodes[3], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[:2] + nodes[3:], numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes[:2] + nodes[3:], expectedViewNo=0) - request6 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests( - looper, client1, requests=[request3, request4, request5, request6]) + sdk_reqs6 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + sdk_get_replies(looper, [sdk_reqs3[0], sdk_reqs4[0], sdk_reqs5[0], sdk_reqs6[0]]) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes, expectedViewNo=0) - request7 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request7]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py index a0a7f754fe..4e5e22027e 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py @@ -1,7 +1,10 @@ +import pytest + +from plenum.common.exceptions import PoolLedgerTimeoutException from plenum.test import waits from plenum.test.helper import waitForViewChange, checkViewNoForNodes, \ - sendRandomRequest, waitForSufficientRepliesForRequests, \ - verify_request_not_replied_and_not_ordered + sdk_send_random_and_check, sdk_send_random_requests, sdk_get_replies, \ + sdk_check_reply from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.test_node import ensureElectionsDone, getRequiredInstances @@ -19,55 +22,80 @@ def stop_node(node_to_stop, looper, pool_nodes): def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( looper, allPluginsPath, tdir, tconf, - txnPoolNodeSet, wallet1, client1): + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client): nodes = txnPoolNodeSet - request1 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request1]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) stop_node(nodes[0], looper, nodes) waitForViewChange(looper, nodes[1:], expectedViewNo=1) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) - request2 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request2]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) stop_node(nodes[1], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[2:], expectedViewNo=1) - request3 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request3, looper, client1, nodes) + sdk_reqs3 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs3) + sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) looper.runFor(tconf.ToleratePrimaryDisconnection + waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) - request4 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request4, looper, client1, nodes) + sdk_reqs4 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs4) + sdk_check_reply(req_res[0]) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) looper.runFor(waits.expectedPoolElectionTimeout(len(nodes))) checkViewNoForNodes(nodes[3:], expectedViewNo=1) - request5 = sendRandomRequest(wallet1, client1) - verify_request_not_replied_and_not_ordered(request5, looper, client1, nodes) + sdk_reqs5 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1) + with pytest.raises(PoolLedgerTimeoutException): + req_res = sdk_get_replies(looper, sdk_reqs5) + sdk_check_reply(req_res[0]) nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes[1:], numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes[1:], expectedViewNo=1) - request6 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request6]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) nodes[0] = start_stopped_node(nodes[0], looper, tconf, tdir, allPluginsPath) ensureElectionsDone(looper, nodes, numInstances=getRequiredInstances(nodeCount)) checkViewNoForNodes(nodes, expectedViewNo=1) - request7 = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, requests=[request7]) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) diff --git a/plenum/test/pool_transactions/test_suspend_node.py b/plenum/test/pool_transactions/test_suspend_node.py index fd610def2e..46e76cfe99 100644 --- a/plenum/test/pool_transactions/test_suspend_node.py +++ b/plenum/test/pool_transactions/test_suspend_node.py @@ -4,8 +4,7 @@ from plenum.common.constants import CLIENT_STACK_SUFFIX from plenum.common.util import hexToFriendly from plenum.server.node import Node -from plenum.test.helper import sendRandomRequest, \ - waitForSufficientRepliesForRequests +from plenum.test.helper import waitForSufficientRepliesForRequests from plenum.test.node_catchup.helper import \ ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.pool_transactions.helper import suspendNode, \ @@ -45,7 +44,7 @@ def testStewardSuspendsNode(looper, txnPoolNodeSet, looper.run(eventually(checkNodeNotInNodeReg, client, newNode.name)) # Check a client can send request and receive replies - req = sendRandomRequest(newStewardWallet, newSteward) + # req = sendRandomRequest(newStewardWallet, newSteward) waitForSufficientRepliesForRequests(looper, newSteward, requests=[req]) diff --git a/plenum/test/replica/test_replica_reject_same_pre_prepare.py b/plenum/test/replica/test_replica_reject_same_pre_prepare.py index 0fb79c6872..c4daf6e88c 100644 --- a/plenum/test/replica/test_replica_reject_same_pre_prepare.py +++ b/plenum/test/replica/test_replica_reject_same_pre_prepare.py @@ -9,7 +9,8 @@ from plenum.test import waits from plenum.test.helper import checkPrePrepareReqSent, \ checkPrePrepareReqRecvd, \ - checkPrepareReqSent, check_sufficient_replies_received, sendRandomRequest + checkPrepareReqSent, sdk_send_random_requests, \ + sdk_json_to_request_object, sdk_get_replies from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica whitelist = ['doing nothing for now', @@ -20,7 +21,7 @@ # noinspection PyIncorrectDocstring -def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1): +def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Replicas should not accept PRE-PREPARE for view "v" and prepare sequence number "n" if it has already accepted a request with view number "v" and @@ -39,7 +40,11 @@ def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1 for node in txnPoolNodeSet: node.nodeIbStasher.delay(cDelay(delay=c_delay, instId=1)) - request1 = sendRandomRequest(wallet1, client1) + req1 = sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1)[0] + request1 = sdk_json_to_request_object(req1[0]) for npr in nonPrimaryReplicas: looper.run(eventually(checkPrepareReqSent, npr, @@ -68,7 +73,11 @@ def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1 "one...") primaryRepl._lastPrePrepareSeqNo -= 1 view_no = primaryRepl.viewNo - request2 = sendRandomRequest(wallet1, client1) + request2 = sdk_json_to_request_object( + sdk_send_random_requests(looper, + sdk_pool_handle, + sdk_wallet_client, + 1)[0][0]) timeout = waits.expectedPrePrepareTime(len(txnPoolNodeSet)) looper.run(eventually(checkPrePrepareReqSent, primaryRepl, request2, retryWait=1, timeout=timeout)) @@ -118,8 +127,5 @@ def testReplicasRejectSamePrePrepareMsg(looper, txnPoolNodeSet, client1, wallet1 timeout=timeout)) timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) + c_delay - result1 = \ - looper.run(eventually(check_sufficient_replies_received, - client1, request1.identifier, request1.reqId, - retryWait=1, timeout=timeout)) + result1 = sdk_get_replies(looper, [req1])[0][1] logger.debug("request {} gives result {}".format(request1, result1)) diff --git a/plenum/test/test_node_request.py b/plenum/test/test_node_request.py index 75938e78cb..a80855fa51 100644 --- a/plenum/test/test_node_request.py +++ b/plenum/test/test_node_request.py @@ -1,53 +1,44 @@ -from pprint import pprint - import pytest -from plenum import config from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger from stp_core.loop.looper import Looper -from plenum.common.messages.node_messages import Primary, PrePrepare, Prepare, Commit -from plenum.common.util import getMaxFailures +from plenum.common.messages.node_messages import \ + PrePrepare, Prepare, Commit from plenum.test import waits -from plenum.test.delayers import delayerMsgTuple from plenum.test.greek import genNodeNames -from plenum.test.helper import setupNodesAndClient, \ - sendRandomRequest, setupClient, \ - assertLength, addNodeBack, waitForSufficientRepliesForRequests, \ - getPendingRequestsForReplica, checkRequestReturnedToNode -from plenum.test.profiler import profile_this -from plenum.test.test_node import TestNode, TestNodeSet, checkPoolReady, \ - ensureElectionsDone, genNodeReg, prepareNodeSet +from plenum.test.helper import assertLength, addNodeBack, \ + getPendingRequestsForReplica, sdk_send_random_and_check +from plenum.test.test_node import TestNode, TestNodeSet, \ + checkPoolReady, genNodeReg, prepareNodeSet whitelist = ['cannot process incoming PREPARE'] logger = getlogger() -def testReqExecWhenReturnedByMaster(tdir_for_func, tconf_for_func): - with TestNodeSet(tconf_for_func, count=4, tmpdir=tdir_for_func) as nodeSet: - with Looper(nodeSet) as looper: - client1, wallet1 = setupNodesAndClient(looper, - nodeSet, - tmpdir=tdir_for_func) - req = sendRandomRequest(wallet1, client1) - waitForSufficientRepliesForRequests(looper, client1, - requests=[req]) - - async def chk(): - for node in nodeSet: - entries = node.spylog.getAll( - node.processOrdered.__name__) - for entry in entries: - arg = entry.params['ordered'] - result = entry.result - if arg.instId == node.instances.masterId: - assert result - else: - assert result is False - - timeout = waits.expectedOrderingTime( - nodeSet.nodes['Alpha'].instances.count) - looper.run(eventually(chk, timeout=timeout)) +def testReqExecWhenReturnedByMaster(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client): + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) + + async def chk(): + for node in txnPoolNodeSet: + entries = node.spylog.getAll( + node.processOrdered.__name__) + for entry in entries: + arg = entry.params['ordered'] + result = entry.result + if arg.instId == node.instances.masterId: + assert result + else: + assert result is False + + timeout = waits.expectedOrderingTime( + txnPoolNodeSet[0].instances.count) + looper.run(eventually(chk, timeout=timeout)) @pytest.mark.skip('Since primary is selected immediately now') @@ -78,8 +69,8 @@ def testPrePrepareWhenPrimaryStatusIsUnknown(tdir_for_func): checkPoolReady(looper=looper, nodes=nodeSet) - client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) - request = sendRandomRequest(wal, client1) + # client1, wal = setupClient(looper, nodeSet, tmpdir=tdir_for_func) + # request = sendRandomRequest(wal, client1) # TODO Rethink this instNo = 0 @@ -134,18 +125,6 @@ async def checkIfPropagateRecvdFromNode(recvrNode: TestNode, assert senderNode.name in recvrNode.requests[key].propagates -def testClientSendingSameRequestAgainBeforeFirstIsProcessed(looper, - txnPoolNodeSet, - wallet1, - client1): - size = len(client1.inBox) - req = sendRandomRequest(wallet1, client1) - client1.submitReqs(req) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) - # Only REQACK will be sent twice by the node but not REPLY - assert len(client1.inBox) == size + 12 - - def snapshotStats(*nodes): return {n.name: n.nodestack.stats.copy() for n in nodes} diff --git a/plenum/test/validator_info/test_validator_info.py b/plenum/test/validator_info/test_validator_info.py index 23fb486b67..020d5bc245 100644 --- a/plenum/test/validator_info/test_validator_info.py +++ b/plenum/test/validator_info/test_validator_info.py @@ -12,8 +12,8 @@ from plenum.common.util import getTimeBasedId from plenum.server.validator_info_tool import ValidatorNodeInfoTool from plenum.test import waits -from plenum.test.helper import waitForSufficientRepliesForRequests, \ - sendRandomRequest, check_sufficient_replies_received +from plenum.test.helper import check_sufficient_replies_received, \ + sdk_send_random_and_check # noinspection PyUnresolvedReferences from plenum.test.node_catchup.helper import ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected @@ -246,14 +246,15 @@ def read_wrapped(txn_type): @pytest.fixture def write_txn_and_get_latest_info(txnPoolNodesLooper, - client_and_wallet, + sdk_pool_handle, + sdk_wallet_client, patched_dump_info_period, info_path): - client, wallet = client_and_wallet - def write_wrapped(): - req = sendRandomRequest(wallet, client) - waitForSufficientRepliesForRequests(txnPoolNodesLooper, client, requests=[req]) + sdk_send_random_and_check(txnPoolNodesLooper, range(nodeCount), + sdk_pool_handle, + sdk_wallet_client, + 1) txnPoolNodesLooper.runFor(patched_dump_info_period) return load_info(info_path) diff --git a/plenum/test/view_change/test_queueing_req_from_future_view.py b/plenum/test/view_change/test_queueing_req_from_future_view.py index 6973452729..902c89db1a 100644 --- a/plenum/test/view_change/test_queueing_req_from_future_view.py +++ b/plenum/test/view_change/test_queueing_req_from_future_view.py @@ -1,17 +1,11 @@ -import pytest - -from plenum.test.view_change.helper import provoke_and_wait_for_view_change, ensure_view_change +from plenum.test.view_change.helper import ensure_view_change from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger -from plenum.common.util import getMaxFailures -from plenum.test import waits -from plenum.test.delayers import ppDelay, icDelay, vcd_delay -from plenum.test.helper import sendRandomRequest, \ - sendReqsToNodesAndVerifySuffReplies, sendRandomRequests, \ +from plenum.test.delayers import icDelay, vcd_delay +from plenum.test.helper import sendRandomRequests, \ waitForSufficientRepliesForRequests, \ send_reqs_to_nodes_and_verify_all_replies -from plenum.test.test_node import TestReplica, getNonPrimaryReplicas, \ - checkViewChangeInitiatedForNode, get_last_master_non_primary_node +from plenum.test.test_node import get_last_master_non_primary_node nodeCount = 7 From 980f9c06941bd45f3aa40c29bc47894a8033ba36 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Thu, 22 Mar 2018 18:45:28 +0300 Subject: [PATCH 34/73] Little error fix Signed-off-by: ArtObr --- plenum/test/test_node_request.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/plenum/test/test_node_request.py b/plenum/test/test_node_request.py index 99c45a30b3..a80855fa51 100644 --- a/plenum/test/test_node_request.py +++ b/plenum/test/test_node_request.py @@ -124,22 +124,6 @@ async def checkIfPropagateRecvdFromNode(recvrNode: TestNode, assert key in recvrNode.requests assert senderNode.name in recvrNode.requests[key].propagates -<< << << < HEAD -== == == = - -def testClientSendingSameRequestAgainBeforeFirstIsProcessed(looper, - txnPoolNodeSet, - wallet1, - client1): - size = len(client1.inBox) - req = sendRandomRequest(wallet1, client1) - client1.submitReqs(req) - waitForSufficientRepliesForRequests(looper, client1, requests=[req]) - # Only REQACK will be sent twice by the node but not REPLY - assert len(client1.inBox) == size + 12 - ->> >> >> > base / master - def snapshotStats(*nodes): return {n.name: n.nodestack.stats.copy() for n in nodes} From aead353c86464a0147c3cba30bc7726332820b21 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 23 Mar 2018 08:34:15 +0300 Subject: [PATCH 35/73] waitReqNackFromPoolWithReason function deletion Signed-off-by: ArtObr --- plenum/test/helper.py | 15 ++++---- .../test_plugin_request_handling.py | 38 +++++++++++-------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index b36f7ae3ee..804f52fdbb 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -144,7 +144,6 @@ def send_reqs_to_nodes_and_verify_all_replies(looper: Looper, override_timeout_limit=False, total_timeout=None): requests = sendRandomRequests(wallet, client, numReqs) - nodeCount = len(client.nodeReg) # wait till more than nodeCount replies are received (that is all nodes # answered) waitForSufficientRepliesForRequests( @@ -616,12 +615,6 @@ def ensureRejectsRecvd(looper, nodes, client, reason, timeout=5): timeout=timeout)) -def waitReqNackFromPoolWithReason(looper, nodes, client, reason): - for node in nodes: - waitReqNackWithReason(looper, client, reason, - node.clientstack.name) - - def waitRejectFromPoolWithReason(looper, nodes, client, reason): for node in nodes: waitRejectWithReason(looper, client, reason, @@ -990,6 +983,14 @@ def sdk_sign_request_objects(looper, sdk_wallet, reqs: Sequence): return reqs +def sdk_sign_request_strings(looper, sdk_wallet, reqs: Sequence): + wallet_h, did = sdk_wallet + reqs_str = [json.dumps(req) for req in reqs] + reqs = [looper.loop.run_until_complete(sign_request(wallet_h, did, req)) + for req in reqs_str] + return reqs + + def sdk_signed_random_requests(looper, sdk_wallet, count): _, did = sdk_wallet reqs_obj = sdk_random_request_objects(count, identifier=did, diff --git a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py index 77cbe3fd2a..63f9143e58 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py @@ -1,10 +1,11 @@ import pytest from plenum.common.constants import TXN_TYPE, DATA -from plenum.test.helper import waitReqNackFromPoolWithReason, \ - send_signed_requests, sign_requests, \ +from plenum.common.exceptions import RequestNackedException +from plenum.test.helper import send_signed_requests, sign_requests, \ waitRejectFromPoolWithReason, sdk_gen_request, sdk_sign_and_submit_req_obj, \ - sdk_get_reply + sdk_get_reply, sdk_send_signed_requests, \ + sdk_sign_request_strings, sdk_get_and_check_replies from plenum.test.plugin.demo_plugin.constants import AMOUNT, PLACE_BID, \ AUCTION_START, AUCTION_END, AUCTION_LEDGER_ID @@ -16,8 +17,7 @@ def successful_op(looper, op, sdk_wallet, sdk_pool_handle): sdk_get_reply(looper, req) -def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, stewardWallet, - steward1, +def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, sdk_wallet_steward, sdk_pool_handle): """ Check plugin static validation fails and passes @@ -25,17 +25,21 @@ def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, stewa op = { TXN_TYPE: AUCTION_START } - send_signed_requests(steward1, sign_requests(stewardWallet, [op, ])) - waitReqNackFromPoolWithReason(looper, txn_pool_node_set_post_creation, steward1, - 'attribute is missing or not in proper format') + reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) + reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) + with pytest.raises(RequestNackedException) as exc_info: + sdk_get_and_check_replies(looper, reqs) + exc_info.match('attribute is missing or not in proper format') op = { TXN_TYPE: AUCTION_START, DATA: 'should be a dict but giving a string' } - send_signed_requests(steward1, sign_requests(stewardWallet, [op, ])) - waitReqNackFromPoolWithReason(looper, txn_pool_node_set_post_creation, steward1, - 'attribute is missing or not in proper format') + reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) + reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) + with pytest.raises(RequestNackedException) as exc_info: + sdk_get_and_check_replies(looper, reqs) + exc_info.match('attribute is missing or not in proper format') op = { TXN_TYPE: AUCTION_START, @@ -48,9 +52,11 @@ def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, stewa TXN_TYPE: PLACE_BID, DATA: {'id': 'abc', AMOUNT: -3} } - send_signed_requests(steward1, sign_requests(stewardWallet, [op, ])) - waitReqNackFromPoolWithReason(looper, txn_pool_node_set_post_creation, steward1, - 'must be present and should be a number') + reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) + reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) + with pytest.raises(RequestNackedException) as exc_info: + sdk_get_and_check_replies(looper, reqs) + exc_info.match('must be present and should be a number') op = { TXN_TYPE: PLACE_BID, @@ -60,7 +66,7 @@ def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, stewa def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, stewardWallet, - steward1, + steward1, sdk_wallet_steward, sdk_pool_handle): """ Check plugin dynamic validation fails and passes @@ -88,7 +94,7 @@ def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, stew @pytest.fixture(scope="module") def some_requests(txn_pool_node_set_post_creation, looper, stewardWallet, - steward1, + steward1, sdk_wallet_steward, sdk_pool_handle): op = { TXN_TYPE: AUCTION_START, From b165397428c9c4520781b066daec88726db1eddd Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 23 Mar 2018 08:42:49 +0300 Subject: [PATCH 36/73] waitRejectFromPoolWithReason function deletion Signed-off-by: ArtObr --- plenum/test/helper.py | 6 ------ .../test_plugin_request_handling.py | 16 +++++++-------- .../test_3pc_msgs_during_view_change.py | 20 +++++++++---------- 3 files changed, 18 insertions(+), 24 deletions(-) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 804f52fdbb..7591d7d76f 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -615,12 +615,6 @@ def ensureRejectsRecvd(looper, nodes, client, reason, timeout=5): timeout=timeout)) -def waitRejectFromPoolWithReason(looper, nodes, client, reason): - for node in nodes: - waitRejectWithReason(looper, client, reason, - node.clientstack.name) - - def checkViewNoForNodes(nodes: Iterable[TestNode], expectedViewNo: int = None): """ Checks if all the given nodes have the expected view no diff --git a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py index 63f9143e58..ff9c8eb584 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py @@ -2,9 +2,8 @@ from plenum.common.constants import TXN_TYPE, DATA from plenum.common.exceptions import RequestNackedException -from plenum.test.helper import send_signed_requests, sign_requests, \ - waitRejectFromPoolWithReason, sdk_gen_request, sdk_sign_and_submit_req_obj, \ - sdk_get_reply, sdk_send_signed_requests, \ +from plenum.test.helper import sdk_gen_request, \ + sdk_sign_and_submit_req_obj, sdk_get_reply, sdk_send_signed_requests, \ sdk_sign_request_strings, sdk_get_and_check_replies from plenum.test.plugin.demo_plugin.constants import AMOUNT, PLACE_BID, \ AUCTION_START, AUCTION_END, AUCTION_LEDGER_ID @@ -65,8 +64,7 @@ def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle) -def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, stewardWallet, - steward1, +def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, sdk_wallet_steward, sdk_pool_handle): """ Check plugin dynamic validation fails and passes @@ -75,9 +73,11 @@ def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, stew TXN_TYPE: AUCTION_END, DATA: {'id': 'abcdef'} } - send_signed_requests(steward1, sign_requests(stewardWallet, [op, ])) - waitRejectFromPoolWithReason(looper, txn_pool_node_set_post_creation, steward1, - 'unknown auction') + reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) + reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) + with pytest.raises(RequestNackedException) as exc_info: + sdk_get_and_check_replies(looper, reqs) + exc_info.match('unknown auction') op = { TXN_TYPE: AUCTION_START, diff --git a/plenum/test/view_change/test_3pc_msgs_during_view_change.py b/plenum/test/view_change/test_3pc_msgs_during_view_change.py index 933b81105a..a3007a9821 100644 --- a/plenum/test/view_change/test_3pc_msgs_during_view_change.py +++ b/plenum/test/view_change/test_3pc_msgs_during_view_change.py @@ -1,8 +1,10 @@ import pytest + +from plenum.common.exceptions import RequestRejectedException from plenum.test.delayers import ppgDelay from plenum.test.helper import send_pre_prepare, send_prepare, send_commit, \ - sendRandomRequests, waitRejectFromPoolWithReason, \ - waitForSufficientRepliesForRequests + sendRandomRequests, waitForSufficientRepliesForRequests, \ + sdk_send_random_and_check from plenum.test.test_node import getPrimaryReplica from plenum.test.view_change.helper import check_replica_queue_empty, \ check_all_replica_queue_empty @@ -10,17 +12,15 @@ @pytest.mark.skip('Currently we stash client requests during view change') def test_no_requests_processed_during_view_change(looper, txnPoolNodeSet, - client1, wallet1): + sdk_pool_handle, sdk_wallet_client): for node in txnPoolNodeSet: node.view_change_in_progress = True - sendRandomRequests(wallet1, client1, 10) - - waitRejectFromPoolWithReason( - looper, - txnPoolNodeSet, - client1, - 'Can not process requests when view change is in progress') + with pytest.raises(RequestRejectedException) as e: + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, 10) + assert 'Can not process requests when view change is in progress' in \ + e._excinfo[1].args[0] for node in txnPoolNodeSet: check_replica_queue_empty(node) From 0ec07f1658741e46b08bebe692d6c8d31131f1ae Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 23 Mar 2018 08:51:26 +0300 Subject: [PATCH 37/73] checkReqNack function deletion Signed-off-by: ArtObr --- .../test/client/test_client_request_nack.py | 22 +++++++++---------- plenum/test/helper.py | 11 ---------- 2 files changed, 10 insertions(+), 23 deletions(-) diff --git a/plenum/test/client/test_client_request_nack.py b/plenum/test/client/test_client_request_nack.py index 5e1b97edad..2f4ca3e445 100644 --- a/plenum/test/client/test_client_request_nack.py +++ b/plenum/test/client/test_client_request_nack.py @@ -2,10 +2,8 @@ import pytest -from stp_core.loop.eventually import eventuallyAll - -from plenum.test import waits -from plenum.test.helper import checkReqNack +from plenum.common.exceptions import RequestNackedException +from plenum.test.helper import sdk_send_random_and_check whitelist = ['discarding message'] @@ -32,15 +30,15 @@ def request1(wallet1): @pytest.mark.skip(reason="old style plugin") def testRequestFullRoundTrip(restrictiveVerifier, - client1, - sent1, + sdk_pool_handle, + sdk_wallet_client, looper, txnPoolNodeSet): update = {'reason': 'client request invalid: InvalidClientRequest() ' '[caused by amount too high\nassert 999 <= 100]'} - - coros2 = [partial(checkReqNack, client1, node, sent1.identifier, - sent1.reqId, update) - for node in txnPoolNodeSet] - timeout = waits.expectedReqAckQuorumTime() - looper.run(eventuallyAll(*coros2, totalTimeout=timeout)) + with pytest.raises(RequestNackedException) as e: + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, 1) + assert 'client request invalid: InvalidClientRequest() ' + '[caused by amount too high\nassert 999 <= 100]' in \ + e._excinfo[1].args[0] diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 7591d7d76f..30b7db5836 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -540,17 +540,6 @@ def checkReqAck(client, node, idr, reqId, update: Dict[str, str] = None): assert client.inBox.count(expected) > 0 -def checkReqNack(client, node, idr, reqId, update: Dict[str, str] = None): - rec = {OP_FIELD_NAME: REQNACK, f.REQ_ID.nm: reqId, f.IDENTIFIER.nm: idr} - if update: - rec.update(update) - expected = (rec, node.clientstack.name) - # More than one matching message could be present in the client's inBox - # since client on not receiving request under timeout might have retried - # the request - assert client.inBox.count(expected) > 0 - - def checkReplyCount(client, idr, reqId, count): senders = set() for msg, sdr in client.inBox: From e7ee2035d1c2fd540e14b5f345f1519f1e02e207 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 23 Mar 2018 08:52:57 +0300 Subject: [PATCH 38/73] Two more fucntions deletion Signed-off-by: ArtObr --- plenum/test/helper.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 30b7db5836..65149625f2 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -540,23 +540,6 @@ def checkReqAck(client, node, idr, reqId, update: Dict[str, str] = None): assert client.inBox.count(expected) > 0 -def checkReplyCount(client, idr, reqId, count): - senders = set() - for msg, sdr in client.inBox: - if msg[OP_FIELD_NAME] == REPLY and \ - msg[f.RESULT.nm][f.IDENTIFIER.nm] == idr and \ - msg[f.RESULT.nm][f.REQ_ID.nm] == reqId: - senders.add(sdr) - assertLength(senders, count) - - -def wait_for_replies(looper, client, idr, reqId, count, custom_timeout=None): - timeout = custom_timeout or waits.expectedTransactionExecutionTime( - len(client.nodeReg)) - looper.run(eventually(checkReplyCount, client, idr, reqId, count, - timeout=timeout)) - - def checkReqNackWithReason(client, reason: str, sender: str): found = False for msg, sdr in client.inBox: From 376d8d35bbabff8d89dc34bbe405a806435ffd92 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 23 Mar 2018 10:37:01 +0300 Subject: [PATCH 39/73] Deletion of sendRandomRequests. Part 1. Signed-off-by: ArtObr --- .../test_instance_change_with_Delta.py | 8 ++-- plenum/test/node_catchup/conftest.py | 23 +++++++++- .../node_catchup/test_catchup_scenarios.py | 21 +++++----- ...test_catchup_while_new_request_incoming.py | 33 ++++++++------- .../test_node_request_consistency_proof.py | 23 +++++----- .../test_node_request_missing_transactions.py | 18 ++++---- .../test_catchup_multiple_rounds.py | 22 +++++----- plenum/test/view_change/helper.py | 42 ++++++++++--------- ...ew_change_2_of_4_nodes_with_new_primary.py | 8 ++-- ...ew_change_2_of_4_nodes_with_non_primary.py | 8 ++-- ...e_2_of_4_nodes_with_old_and_new_primary.py | 8 ++-- ...ew_change_2_of_4_nodes_with_old_primary.py | 8 ++-- .../slow_nodes/test_view_change_all_nodes.py | 11 ++--- ...test_view_change_all_nodes_random_delay.py | 9 ++-- .../slow_nodes/test_view_change_complex.py | 14 +++---- ...t_view_change_gc_all_nodes_random_delay.py | 18 ++++---- .../test_3pc_msgs_during_view_change.py | 11 ++--- ..._master_primary_different_from_previous.py | 11 +++-- ...detecting_lag_from_view_change_messages.py | 5 ++- .../test_queueing_req_from_future_view.py | 13 +++--- 20 files changed, 175 insertions(+), 139 deletions(-) diff --git a/plenum/test/monitoring/test_instance_change_with_Delta.py b/plenum/test/monitoring/test_instance_change_with_Delta.py index 46b9523f40..5678d2edc1 100644 --- a/plenum/test/monitoring/test_instance_change_with_Delta.py +++ b/plenum/test/monitoring/test_instance_change_with_Delta.py @@ -5,7 +5,7 @@ from stp_core.common.util import adict from plenum.server.node import Node from plenum.test import waits -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.malicious_behaviors_node import slow_primary from plenum.test.test_node import getPrimaryReplica from plenum.test.view_change.helper import provoke_and_wait_for_view_change @@ -116,11 +116,11 @@ def step3(step2): @pytest.mark.skip(reason="SOV-1123 - fails intermittently") -def testInstChangeWithLowerRatioThanDelta(looper, step3, wallet1, client1): +def testInstChangeWithLowerRatioThanDelta(looper, step3, sdk_pool_handle, sdk_wallet_client): # from plenum.test.test_node import ensureElectionsDone # ensureElectionsDone(looper, []) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 9) + sdk_send_random_and_check(looper, step3.nodes, sdk_pool_handle, sdk_wallet_client, 9) # wait for every node to run another checkPerformance waitForNextPerfCheck(looper, step3.nodes, step3.perfChecks) - provoke_and_wait_for_view_change(looper, step3.nodes, 1, wallet1, client1) + provoke_and_wait_for_view_change(looper, step3.nodes, 1, sdk_pool_handle, sdk_wallet_client) diff --git a/plenum/test/node_catchup/conftest.py b/plenum/test/node_catchup/conftest.py index 643e66e24a..45dfed5f4e 100644 --- a/plenum/test/node_catchup/conftest.py +++ b/plenum/test/node_catchup/conftest.py @@ -4,11 +4,11 @@ from stp_core.common.log import getlogger from plenum.common.util import randomString from plenum.test.conftest import getValueFromModule -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ check_last_3pc_master from plenum.test.pool_transactions.helper import \ - addNewStewardAndNode, buildPoolClientAndWallet + addNewStewardAndNode, buildPoolClientAndWallet, sdk_add_new_steward_and_node from plenum.test.test_client import TestClient from plenum.test.test_node import checkNodesConnected @@ -50,6 +50,25 @@ def nodeCreatedAfterSomeTxns(looper, testNodeClass, do_post_node_creation, newStewardWallet +@pytest.yield_fixture("module") +def sdk_node_created_after_some_txns(looper, testNodeClass, do_post_node_creation, + sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward, + txnPoolNodeSet, tdir, tconf, allPluginsPath, request): + txnCount = getValueFromModule(request, "txnCount", 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + txnCount) + new_steward_name = randomString() + new_node_name = "Epsilon" + new_steward_wallet_handle, new_node = sdk_add_new_steward_and_node( + looper, sdk_pool_handle, sdk_wallet_steward, + new_steward_name, new_node_name, tdir, tconf, nodeClass=testNodeClass, + allPluginsPath=allPluginsPath, autoStart=True, + do_post_node_creation=do_post_node_creation) + yield looper, new_node, sdk_pool_handle, new_steward_wallet_handle + + @pytest.fixture("module") def nodeSetWithNodeAddedAfterSomeTxns( txnPoolNodeSet, nodeCreatedAfterSomeTxns): diff --git a/plenum/test/node_catchup/test_catchup_scenarios.py b/plenum/test/node_catchup/test_catchup_scenarios.py index e6409e3497..761103beab 100644 --- a/plenum/test/node_catchup/test_catchup_scenarios.py +++ b/plenum/test/node_catchup/test_catchup_scenarios.py @@ -1,12 +1,11 @@ import pytest +from plenum.test.node_request.helper import sdk_ensure_pool_functional from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger from plenum.common.startable import Mode from plenum.test.delayers import cqDelay -from plenum.test.helper import sendRandomRequests -from plenum.test.node_catchup.helper import \ - ensureClientConnectedToNodesAndPoolLedgerSame +from plenum.test.helper import sdk_send_random_requests from plenum.test.test_node import checkNodesConnected from plenum.test import waits @@ -16,19 +15,19 @@ @pytest.fixture("module") -def nodeStashingOrderedRequests(txnPoolNodeSet, nodeCreatedAfterSomeTxns): - looper, newNode, client, wallet, _, _ = nodeCreatedAfterSomeTxns +def nodeStashingOrderedRequests(txnPoolNodeSet, sdk_node_created_after_some_txns): + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = sdk_node_created_after_some_txns for node in txnPoolNodeSet: node.nodeIbStasher.delay(cqDelay(5)) - txnPoolNodeSet.append(newNode) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, - *txnPoolNodeSet[:-1]) - sendRandomRequests(wallet, client, 10) + txnPoolNodeSet.append(new_node) + + sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet_handle, sdk_pool_handle) + sdk_send_random_requests(looper, sdk_pool_handle, new_steward_wallet_handle, 10) looper.run(checkNodesConnected(txnPoolNodeSet)) def stashing(): - assert newNode.mode != Mode.participating - assert len(newNode.stashedOrderedReqs) > 0 + assert new_node.mode != Mode.participating + assert len(new_node.stashedOrderedReqs) > 0 # assert len(newNode.reqsFromCatchupReplies) > 0 timeout = waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) diff --git a/plenum/test/node_catchup/test_catchup_while_new_request_incoming.py b/plenum/test/node_catchup/test_catchup_while_new_request_incoming.py index 5e03ff075f..49e6ce9224 100644 --- a/plenum/test/node_catchup/test_catchup_while_new_request_incoming.py +++ b/plenum/test/node_catchup/test_catchup_while_new_request_incoming.py @@ -5,18 +5,19 @@ from plenum.common.messages.node_messages import CatchupReq from plenum.common.util import randomString from plenum.test.delayers import cqDelay -from plenum.test.helper import sendRandomRequests, \ - sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check, \ + sdk_send_random_requests from plenum.test.node_catchup.helper import checkNodeDataForEquality -from plenum.test.pool_transactions.helper import addNewStewardAndNode +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node from plenum.test.test_node import TestNode from stp_core.loop.eventually import eventually def testNewNodeCatchupWhileIncomingRequests(looper, txnPoolNodeSet, testNodeClass, tdir, - tdirWithClientPoolTxns, tconf, - steward1, stewardWallet, + tconf, + sdk_pool_handle, + sdk_wallet_steward, allPluginsPath): """ A new node joins while transactions are happening, its catchup requests @@ -24,7 +25,7 @@ def testNewNodeCatchupWhileIncomingRequests(looper, txnPoolNodeSet, node's ledger size. In the meantime, the new node will stash all requests """ - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) def chkAfterCall(self, req, frm): r = self.processCatchupReq(req, frm) @@ -40,18 +41,18 @@ def chkAfterCall(self, req, frm): node.nodeIbStasher.delay(cqDelay(3)) print('Sending 5 requests') - sendRandomRequests(stewardWallet, steward1, 5) + sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_steward, 5) looper.runFor(1) - newStewardName = randomString() - newNodeName = "Epsilon" - newStewardClient, newStewardWallet, newNode = addNewStewardAndNode( - looper, steward1, stewardWallet, newStewardName, newNodeName, - tdir, tdirWithClientPoolTxns, tconf, nodeClass=testNodeClass, + new_steward_name = randomString() + new_node_name = "Epsilon" + new_steward_wallet_handle, new_node = sdk_add_new_steward_and_node( + looper, sdk_pool_handle, sdk_wallet_steward, + new_steward_name, new_node_name, tdir, tconf, nodeClass=testNodeClass, allPluginsPath=allPluginsPath, autoStart=True) - txnPoolNodeSet.append(newNode) + txnPoolNodeSet.append(new_node) looper.runFor(2) - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) # TODO select or create a timeout for this case in 'waits' - looper.run(eventually(checkNodeDataForEquality, newNode, + looper.run(eventually(checkNodeDataForEquality, new_node, *txnPoolNodeSet[:-1], retryWait=1, timeout=80)) - assert newNode.spylog.count(TestNode.processStashedOrderedReqs) > 0 + assert new_node.spylog.count(TestNode.processStashedOrderedReqs) > 0 diff --git a/plenum/test/node_catchup/test_node_request_consistency_proof.py b/plenum/test/node_catchup/test_node_request_consistency_proof.py index 8f22946715..b0e1204295 100644 --- a/plenum/test/node_catchup/test_node_request_consistency_proof.py +++ b/plenum/test/node_catchup/test_node_request_consistency_proof.py @@ -6,9 +6,8 @@ count_msg_reqs_of_type from stp_core.common.log import getlogger from plenum.common.messages.node_messages import LedgerStatus -from plenum.test.helper import sendRandomRequests +from plenum.test.helper import sdk_send_random_requests from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.test_node import checkNodesConnected # Do not remove the next imports from plenum.test.node_catchup.conftest import whitelist @@ -22,7 +21,7 @@ def testNodeRequestingConsProof(tconf, txnPoolNodeSet, - nodeCreatedAfterSomeTxns): + sdk_node_created_after_some_txns): """ All of the 4 old nodes delay the processing of LEDGER_STATUS from the newly joined node while they are processing requests which results in them sending @@ -30,18 +29,18 @@ def testNodeRequestingConsProof(tconf, txnPoolNodeSet, conclude about the state of transactions in the system. So the new node requests consistency proof for a particular range from all nodes. """ - looper, newNode, client, wallet, _, _ = nodeCreatedAfterSomeTxns + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = sdk_node_created_after_some_txns # So nodes wont tell the clients about the newly joined node so they # dont send any request to the newly joined node for node in txnPoolNodeSet: node.sendPoolInfoToClients = types.MethodType(lambda x, y: None, node) - txnPoolNodeSet.append(newNode) + txnPoolNodeSet.append(new_node) # The new node sends different ledger statuses to every node so it # does not get enough similar consistency proofs next_size = 0 - origMethod = newNode.build_ledger_status + origMethod = new_node.build_ledger_status def build_broken_ledger_status(self, ledger_id): nonlocal next_size @@ -61,18 +60,20 @@ def build_broken_ledger_status(self, ledger_id): print("dl status {}".format(ledgerStatus)) return ledgerStatus - newNode.build_ledger_status = types.MethodType( - build_broken_ledger_status, newNode) + new_node.build_ledger_status = types.MethodType( + build_broken_ledger_status, new_node) + logger.debug( - 'Domain Ledger status sender of {} patched'.format(newNode)) + 'Domain Ledger status sender of {} patched'.format(new_node)) - sendRandomRequests(wallet, client, 10) + sdk_send_random_requests(looper, sdk_pool_handle, + new_steward_wallet_handle, 10) # wait more than `ConsistencyProofsTimeout` # TODO: apply configurable timeout here # `ConsistencyProofsTimeout` is set to 60 sec, so need to wait more than # 60 sec, hence large timeout. Dont reduce it. - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1], + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1], customTimeout=75) # Other nodes should have received a request for `CONSISTENCY_PROOF` and diff --git a/plenum/test/node_catchup/test_node_request_missing_transactions.py b/plenum/test/node_catchup/test_node_request_missing_transactions.py index 8190d3ac55..53dff818ea 100644 --- a/plenum/test/node_catchup/test_node_request_missing_transactions.py +++ b/plenum/test/node_catchup/test_node_request_missing_transactions.py @@ -5,7 +5,7 @@ from plenum.common.constants import DOMAIN_LEDGER_ID from stp_core.common.log import getlogger from plenum.common.messages.node_messages import CatchupReq -from plenum.test.helper import sendRandomRequests +from plenum.test.helper import sdk_send_random_requests from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.test_node import checkNodesConnected, getNonPrimaryReplicas from plenum.test import waits @@ -31,15 +31,15 @@ def reset(): def testNodeRequestingTxns(reduced_catchup_timeout_conf, txnPoolNodeSet, - nodeCreatedAfterSomeTxns): + sdk_node_created_after_some_txns, sdk_wallet_client): """ A newly joined node is catching up and sends catchup requests to other nodes but one of the nodes does not reply and the newly joined node cannot complete the process till the timeout and then requests the missing transactions. """ - looper, newNode, client, wallet, _, _ = nodeCreatedAfterSomeTxns - new_node_ledger = newNode.ledgerManager.ledgerRegistry[DOMAIN_LEDGER_ID] + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = sdk_node_created_after_some_txns + new_node_ledger = new_node.ledgerManager.ledgerRegistry[DOMAIN_LEDGER_ID] old_size = len(new_node_ledger.ledger) old_size_others = txnPoolNodeSet[0].ledgerManager.ledgerRegistry[DOMAIN_LEDGER_ID].ledger.size @@ -56,19 +56,19 @@ def ignoreCatchupReq(self, req, frm): npr = getNonPrimaryReplicas(txnPoolNodeSet, 0) badReplica = npr[0] badNode = badReplica.node - txnPoolNodeSet.append(newNode) + txnPoolNodeSet.append(new_node) badNode.nodeMsgRouter.routes[CatchupReq] = types.MethodType( ignoreCatchupReq, badNode.ledgerManager) more_requests = 10 - sendRandomRequests(wallet, client, more_requests) + sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, more_requests) looper.run(checkNodesConnected(txnPoolNodeSet)) # Since one of the nodes does not reply, this new node will experience a # timeout and retry catchup requests, hence a long test timeout. timeout = waits.expectedPoolGetReadyTimeout(len(txnPoolNodeSet)) + \ reduced_catchup_timeout_conf.CatchupTransactionsTimeout - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1], + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1], customTimeout=timeout) new_size = len(new_node_ledger.ledger) @@ -76,6 +76,6 @@ def ignoreCatchupReq(self, req, frm): # `more_request` transactions assert old_size_others - \ old_size <= new_node_ledger.num_txns_caught_up <= new_size - old_size - sendRandomRequests(wallet, client, 2) - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1], + sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 2) + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1], customTimeout=timeout) diff --git a/plenum/test/primary_selection/test_catchup_multiple_rounds.py b/plenum/test/primary_selection/test_catchup_multiple_rounds.py index 14c16a7746..df34f8b4c3 100644 --- a/plenum/test/primary_selection/test_catchup_multiple_rounds.py +++ b/plenum/test/primary_selection/test_catchup_multiple_rounds.py @@ -3,7 +3,8 @@ from plenum.common.constants import DOMAIN_LEDGER_ID from plenum.test.delayers import delay_3pc_messages, icDelay from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \ - sendRandomRequests, waitForSufficientRepliesForRequests, checkViewNoForNodes + sendRandomRequests, waitForSufficientRepliesForRequests, checkViewNoForNodes, sdk_send_random_and_check, \ + sdk_send_random_requests, sdk_get_replies, sdk_get_and_check_replies from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.batching_3pc.conftest import tconf @@ -21,8 +22,8 @@ def test_slow_nodes_catchup_before_selecting_primary_in_new_view( tconf, looper, txnPoolNodeSet, - client1, - wallet1, + sdk_pool_handle, + sdk_wallet_client, one_node_added): """ Delay 3PC messages to one node and view change messages to some others @@ -40,8 +41,8 @@ def test_slow_nodes_catchup_before_selecting_primary_in_new_view( delay_3pc = 100 delay_ic = 5 - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, - 2 * Max3PCBatchSize) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 2 * Max3PCBatchSize) delay_3pc_messages([slow_node], 0, delay_3pc) @@ -53,13 +54,14 @@ def start_count(): return sum([1 for e in slow_node.ledgerManager.spylog.getAll( if e.params['ledgerId'] == DOMAIN_LEDGER_ID]) s = start_count() - requests = sendRandomRequests(wallet1, client1, 10 * Max3PCBatchSize) + requests = sdk_send_random_requests(looper, sdk_pool_handle, + sdk_wallet_client, 10 * Max3PCBatchSize) ensure_view_change(looper, nodes=txnPoolNodeSet, exclude_from_check=nodes_slow_to_inst_chg) - waitForSufficientRepliesForRequests(looper, client1, - requests=requests) + sdk_get_and_check_replies(looper, requests) + waitNodeDataEquality(looper, slow_node, *txnPoolNodeSet[:-1]) e = start_count() @@ -68,7 +70,7 @@ def start_count(): return sum([1 for e in slow_node.ledgerManager.spylog.getAll( looper.run(eventually(checkViewNoForNodes, slow_node.viewNo)) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, - 2 * Max3PCBatchSize) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 2 * Max3PCBatchSize) waitNodeDataEquality(looper, new_node, *nodes_slow_to_inst_chg) diff --git a/plenum/test/view_change/helper.py b/plenum/test/view_change/helper.py index 3ea2359158..a136128c7c 100644 --- a/plenum/test/view_change/helper.py +++ b/plenum/test/view_change/helper.py @@ -4,7 +4,8 @@ from plenum.test.delayers import delayNonPrimaries, delay_3pc_messages, reset_delays_and_process_delayeds from plenum.test.helper import checkViewNoForNodes, sendRandomRequests, \ - sendReqsToNodesAndVerifySuffReplies, send_reqs_to_nodes_and_verify_all_replies + sendReqsToNodesAndVerifySuffReplies, send_reqs_to_nodes_and_verify_all_replies, sdk_send_random_requests, \ + sdk_send_random_and_check from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data @@ -34,7 +35,7 @@ def start_stopped_node(stopped_node, looper, tconf, return restarted_node -def provoke_and_check_view_change(nodes, newViewNo, wallet, client): +def provoke_and_check_view_change(looper, nodes, newViewNo, sdk_pool_handle, sdk_wallet_client): if {n.viewNo for n in nodes} == {newViewNo}: return True @@ -47,24 +48,25 @@ def provoke_and_check_view_change(nodes, newViewNo, wallet, client): else: logger.info('Master instance has not degraded yet, ' 'sending more requests') - sendRandomRequests(wallet, client, 10) + sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client) assert False def provoke_and_wait_for_view_change(looper, nodeSet, expectedViewNo, - wallet, - client, + sdk_pool_handle, + sdk_wallet_client, customTimeout=None): timeout = customTimeout or waits.expectedPoolViewChangeStartedTimeout( len(nodeSet)) # timeout *= 30 return looper.run(eventually(provoke_and_check_view_change, + looper, nodeSet, expectedViewNo, - wallet, - client, + sdk_pool_handle, + sdk_wallet_client, timeout=timeout)) @@ -278,12 +280,14 @@ def ensure_view_change_complete_by_primary_restart( return nodes -def view_change_in_between_3pc(looper, nodes, slow_nodes, wallet, client1, +def view_change_in_between_3pc(looper, nodes, slow_nodes, + sdk_pool_handle, + sdk_wallet_handle, slow_delay=1, wait=None): - send_reqs_to_nodes_and_verify_all_replies(looper, wallet, client1, 4) + sdk_send_random_and_check(looper, nodes, sdk_pool_handle, sdk_wallet_handle, 4) delay_3pc_messages(slow_nodes, 0, delay=slow_delay) - sendRandomRequests(wallet, client1, 10) + sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_handle, 10) if wait: looper.runFor(wait) @@ -291,28 +295,28 @@ def view_change_in_between_3pc(looper, nodes, slow_nodes, wallet, client1, reset_delays_and_process_delayeds(slow_nodes) - sendReqsToNodesAndVerifySuffReplies( - looper, wallet, client1, 5, total_timeout=30) - send_reqs_to_nodes_and_verify_all_replies( - looper, wallet, client1, 5, total_timeout=30) + sdk_send_random_and_check(looper, nodes, sdk_pool_handle, + sdk_wallet_handle, 5, total_timeout=30) + sdk_send_random_and_check(looper, nodes, sdk_pool_handle, + sdk_wallet_handle, 5, total_timeout=30) def view_change_in_between_3pc_random_delays( looper, nodes, slow_nodes, - wallet1, - client1, + sdk_pool_handle, + sdk_wallet_client, tconf, min_delay=0, max_delay=0): - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 4) + sdk_send_random_and_check(looper, nodes, sdk_pool_handle, sdk_wallet_client, 4) # max delay should not be more than catchup timeout. max_delay = max_delay or tconf.MIN_TIMEOUT_CATCHUPS_DONE_DURING_VIEW_CHANGE - 1 delay_3pc_messages(slow_nodes, 0, min_delay=min_delay, max_delay=max_delay) - sendRandomRequests(wallet1, client1, 10) + sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 10) ensure_view_change_complete(looper, nodes, @@ -321,4 +325,4 @@ def view_change_in_between_3pc_random_delays( reset_delays_and_process_delayeds(slow_nodes) - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 10) + sdk_send_random_and_check(looper, nodes, sdk_pool_handle, sdk_wallet_client, 10) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py index 394f076e4a..52724d4b89 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py @@ -8,7 +8,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_handle): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the the first and the last non-primary node @@ -18,11 +18,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client1) + sdk_pool_handle, sdk_wallet_handle) def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary_long_delay( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_handle): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the the first and the last non-primary node @@ -32,5 +32,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary_long_delay( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client1, + sdk_pool_handle, sdk_wallet_handle, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py index bf765107d8..6420e89b72 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_non_primary.py @@ -7,7 +7,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_non_primary( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for 2 of 4 node (2>f). - Both nodes are non-primary for master neither in this nor the next view @@ -15,11 +15,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_non_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client1) + sdk_pool_handle, sdk_wallet_client) def test_view_change_in_between_3pc_2_of_4_nodes_with_non_primary_long_delay( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for 2 of 4 node (2>f). - Both nodes are non-primary for master neither in this nor the next view @@ -27,5 +27,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_non_primary_long_delay( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client1, + sdk_pool_handle, sdk_wallet_client, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py index e6832d10e2..b068d153b6 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_and_new_primary.py @@ -8,7 +8,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_old_and_new_primary( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow both current and next primaries @@ -16,11 +16,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_old_and_new_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client1) + sdk_pool_handle, sdk_wallet_client) def test_view_change_in_between_3pc_2_of_4_nodes_with_old_and_new_primary_long_delay( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow both current and next primaries @@ -28,5 +28,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_old_and_new_primary_long_d """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client1, + sdk_pool_handle, sdk_wallet_client, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py index 4fdb869599..6c8d7c640a 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_old_primary.py @@ -8,7 +8,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_old_primary( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the current Primary node and the last non-primary node (it will not @@ -17,11 +17,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_old_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client1) + sdk_pool_handle, sdk_wallet_client) def test_view_change_in_between_3pc_2_of_4_nodes_with_old_primary_long_delay( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the current Primary node and the last non-primary node (it will not @@ -30,5 +30,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_old_primary_long_delay( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - wallet1, client1, + sdk_pool_handle, sdk_wallet_client, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py index 0cbc010156..f32f3c9745 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes.py @@ -2,22 +2,23 @@ def test_view_change_in_between_3pc_all_nodes(txnPoolNodeSet, looper, - wallet1, client1): + sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for all nodes - do view change """ - view_change_in_between_3pc(looper, txnPoolNodeSet, txnPoolNodeSet, wallet1, - client1) + view_change_in_between_3pc(looper, txnPoolNodeSet, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client) def test_view_change_in_between_3pc_all_nodes_long_delay( - txnPoolNodeSet, looper, wallet1, client1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for all nodes - do view change """ view_change_in_between_3pc(looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, client1, + sdk_pool_handle, sdk_wallet_client, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py index 153fe5fea7..ef361d6fde 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_all_nodes_random_delay.py @@ -4,23 +4,24 @@ def test_view_change_in_between_3pc_all_nodes_random_delays( - txnPoolNodeSet, tconf, looper, wallet1, client1): + txnPoolNodeSet, tconf, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for all nodes randomly - do view change """ view_change_in_between_3pc_random_delays(looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, client1, tconf) + sdk_pool_handle, + sdk_wallet_client, tconf) def test_view_change_in_between_3pc_all_nodes_random_delays_long_delay( - txnPoolNodeSet, looper, wallet1, client1, tconf): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client, tconf): """ - Slow processing 3PC messages for all nodes randomly - do view change """ view_change_in_between_3pc_random_delays(looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, client1, tconf, + sdk_pool_handle, sdk_wallet_client, tconf, min_delay=5) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_complex.py b/plenum/test/view_change/slow_nodes/test_view_change_complex.py index ddfc8bb274..54b991aeb5 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_complex.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_complex.py @@ -5,7 +5,7 @@ def test_view_change_complex( - txnPoolNodeSet, looper, wallet1, client1, tconf): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client, tconf): """ - Complex scenario with multiple view changes """ @@ -35,8 +35,8 @@ def test_view_change_complex( looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, - client1, + sdk_pool_handle, + sdk_wallet_client, tconf, min_delay=0, max_delay=10) @@ -44,8 +44,8 @@ def test_view_change_complex( looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, - client1, + sdk_pool_handle, + sdk_wallet_client, tconf, min_delay=1, max_delay=5) @@ -53,7 +53,7 @@ def test_view_change_complex( looper, txnPoolNodeSet, txnPoolNodeSet, - wallet1, - client1, + sdk_pool_handle, + sdk_wallet_client, tconf, min_delay=5) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py index 2366130d83..9e689d7083 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py @@ -7,7 +7,8 @@ send_reqs_to_nodes_and_verify_all_replies, \ waitForSufficientRepliesForRequests, \ checkViewNoForNodes, \ - check_last_ordered_3pc + check_last_ordered_3pc, sdk_send_random_requests, sdk_send_random_request, sdk_get_replies, \ + sdk_send_random_and_check from plenum.test.delayers import delay_3pc_messages, \ reset_delays_and_process_delayeds from plenum.test.view_change.helper import ensure_view_change_complete @@ -26,14 +27,14 @@ def check_nodes_requests_size(nodes, size): def test_view_change_gc_in_between_3pc_all_nodes_delays( - looper, txnPoolNodeSet, wallet1, client1): + looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): """ Test that garbage collector compares the whole 3PC key (viewNo, ppSeqNo) and does not remove messages from node's queues that have higher viewNo than last ordered one even if their ppSeqNo are less or equal """ - numNodes = len(client1.nodeReg) + numNodes = len(txnPoolNodeSet) viewNo = checkViewNoForNodes(txnPoolNodeSet) # 1 send two messages one by one separately to make @@ -43,8 +44,10 @@ def test_view_change_gc_in_between_3pc_all_nodes_delays( # for master instances only cause non-master ones have # specific logic of its management which we don't care in # the test, see Replica::_setup_for_non_master) - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 1) - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 1) last_ordered_3pc = (viewNo, 2) check_nodes_last_ordered_3pc(txnPoolNodeSet, last_ordered_3pc) @@ -72,7 +75,7 @@ def test_view_change_gc_in_between_3pc_all_nodes_delays( delay_3pc_messages(txnPoolNodeSet, 1, delay=propagationTimeout * 2) - requests = sendRandomRequests(wallet1, client1, 1) + requests = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) def checkPrePrepareSentAtLeastByPrimary(): for node in txnPoolNodeSet: @@ -102,8 +105,7 @@ def checkPrePrepareSentAtLeastByPrimary(): # -> they should be ordered # -> last_ordered_3pc = (+2, 1) reset_delays_and_process_delayeds(txnPoolNodeSet) - waitForSufficientRepliesForRequests(looper, client1, - requests=requests) + sdk_get_replies(looper, requests) checkViewNoForNodes(txnPoolNodeSet, viewNo) last_ordered_3pc = (viewNo, 1) diff --git a/plenum/test/view_change/test_3pc_msgs_during_view_change.py b/plenum/test/view_change/test_3pc_msgs_during_view_change.py index a3007a9821..ebff6f82dd 100644 --- a/plenum/test/view_change/test_3pc_msgs_during_view_change.py +++ b/plenum/test/view_change/test_3pc_msgs_during_view_change.py @@ -3,8 +3,7 @@ from plenum.common.exceptions import RequestRejectedException from plenum.test.delayers import ppgDelay from plenum.test.helper import send_pre_prepare, send_prepare, send_commit, \ - sendRandomRequests, waitForSufficientRepliesForRequests, \ - sdk_send_random_and_check + sdk_send_random_and_check, sdk_send_random_requests, sdk_get_replies from plenum.test.test_node import getPrimaryReplica from plenum.test.view_change.helper import check_replica_queue_empty, \ check_all_replica_queue_empty @@ -50,7 +49,8 @@ def test_no_new_view_3pc_messages_processed_during_view_change( @pytest.mark.skip('The filter is not enabled now') def test_old_view_requests_processed_during_view_change(looper, txnPoolNodeSet, - client1, wallet1): + sdk_wallet_handle, + sdk_wallet_client): """ Make sure that requests sent before view change started are processed and replies are returned: - delay Propogates (to make sure that requests are not ordered before view change is started) @@ -61,10 +61,11 @@ def test_old_view_requests_processed_during_view_change(looper, txnPoolNodeSet, node.view_change_in_progress = False node.nodeIbStasher.delay(ppgDelay(3, 0)) - reqs = sendRandomRequests(wallet1, client1, 2) + requests = sdk_send_random_requests(looper, sdk_wallet_handle, + sdk_wallet_client, 2) looper.runFor(1) for node in txnPoolNodeSet: node.view_change_in_progress = True - waitForSufficientRepliesForRequests(looper, client1, requests=reqs) + sdk_get_replies(looper, requests) diff --git a/plenum/test/view_change/test_master_primary_different_from_previous.py b/plenum/test/view_change/test_master_primary_different_from_previous.py index af396404de..840b50e961 100644 --- a/plenum/test/view_change/test_master_primary_different_from_previous.py +++ b/plenum/test/view_change/test_master_primary_different_from_previous.py @@ -41,7 +41,7 @@ def test_master_primary_different_from_previous(txnPoolNodeSet, looper, @pytest.mark.skip(reason='Nodes use round robin primary selection') def test_master_primary_different_from_previous_view_for_itself( - txnPoolNodeSet, looper, client1, wallet1): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ After a view change, primary must be different from previous primary for master instance, it does not matter for other instance. Break it into @@ -67,8 +67,8 @@ def _get_undecided_inst_id(self): provoke_and_wait_for_view_change(looper, txnPoolNodeSet, old_view_no + 1, - wallet1, - client1) + sdk_pool_handle, + sdk_wallet_client) # Elections done ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) @@ -82,4 +82,7 @@ def _get_undecided_inst_id(self): 'of master in previous view too') == 1 # The new primary can still process requests - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 5) diff --git a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py index d5d13b3e12..f447aa135b 100644 --- a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py +++ b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py @@ -5,7 +5,7 @@ from plenum.common.util import compare_3PC_keys from plenum.test.delayers import delay_3pc_messages, icDelay, cDelay from plenum.test.helper import send_reqs_batches_and_get_suff_replies, \ - sendRandomRequests + sendRandomRequests, sdk_send_random_requests from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.spy_helpers import get_count from plenum.test.test_node import getNonPrimaryReplicas @@ -47,7 +47,8 @@ def test_node_detecting_lag_from_view_change_done_messages(txnPoolNodeSet, reqs = [] for i in range(10): - reqs = reqs + sendRandomRequests(wallet1, client1, 2) + # fix if unskip + reqs = reqs + sdk_send_random_requests() looper.runFor(.2) def chk1(): diff --git a/plenum/test/view_change/test_queueing_req_from_future_view.py b/plenum/test/view_change/test_queueing_req_from_future_view.py index 902c89db1a..a049334864 100644 --- a/plenum/test/view_change/test_queueing_req_from_future_view.py +++ b/plenum/test/view_change/test_queueing_req_from_future_view.py @@ -4,7 +4,7 @@ from plenum.test.delayers import icDelay, vcd_delay from plenum.test.helper import sendRandomRequests, \ waitForSufficientRepliesForRequests, \ - send_reqs_to_nodes_and_verify_all_replies + send_reqs_to_nodes_and_verify_all_replies, sdk_send_random_requests, sdk_get_replies, sdk_send_random_and_check from plenum.test.test_node import get_last_master_non_primary_node nodeCount = 7 @@ -14,7 +14,7 @@ # noinspection PyIncorrectDocstring def testQueueingReqFromFutureView(delayed_perf_chk, looper, txnPoolNodeSet, - wallet1, client1): + sdk_pool_handle, sdk_wallet_client): """ Test if every node queues 3 Phase requests(PRE-PREPARE, PREPARE and COMMIT) that come from a view which is greater than the current view. @@ -52,13 +52,13 @@ def chk_fut_view(view_no, is_empty): # send more requests that will be queued for the lagged node # sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 3) - reqs = sendRandomRequests(wallet1, client1, 5) + reqs = sdk_send_random_requests(looper, sdk_pool_handle, + sdk_wallet_client, 5) l = looper.run(eventually(chk_fut_view, old_view_no + 1, False, retryWait=1)) logger.debug('{} has {} messages for future views' .format(lagging_node, l)) - - waitForSufficientRepliesForRequests(looper, client1, requests=reqs) + sdk_get_replies(looper, reqs) # reset delays for the lagging_node node so that it finally makes view # change lagging_node.reset_delays_and_process_delayeds() @@ -69,4 +69,5 @@ def chk_fut_view(view_no, is_empty): logger.debug('{} exhausted pending messages for future views' .format(lagging_node)) - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 2) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, 2) From 6e63409e6ab55bc84cba1b7f03283b48f1a56506 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 26 Mar 2018 07:41:11 +0300 Subject: [PATCH 40/73] Deletion of sendRandomRequests. Part 2. send_reqs_to_nodes_and_verify_all_replies deleted. Signed-off-by: ArtObr --- plenum/test/helper.py | 22 ---------------- .../test_timestamp_post_view_change.py | 3 +-- plenum/test/view_change/helper.py | 7 +++--- ...t_view_change_gc_all_nodes_random_delay.py | 7 ++---- .../test_new_node_joins_after_view_change.py | 25 ++++++++++--------- .../test_queueing_req_from_future_view.py | 5 ++-- .../zstack_tests/test_zstack_reconnection.py | 13 ++++++---- 7 files changed, 30 insertions(+), 52 deletions(-) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 65149625f2..3b0da8e98f 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -135,28 +135,6 @@ def sendReqsToNodesAndVerifySuffReplies(looper: Looper, return requests -def send_reqs_to_nodes_and_verify_all_replies(looper: Looper, - wallet: Wallet, - client: TestClient, - numReqs: int, - customTimeoutPerReq: float = None, - add_delay_to_timeout: float = 0, - override_timeout_limit=False, - total_timeout=None): - requests = sendRandomRequests(wallet, client, numReqs) - # wait till more than nodeCount replies are received (that is all nodes - # answered) - waitForSufficientRepliesForRequests( - looper, - client, - requests=requests, - customTimeoutPerReq=customTimeoutPerReq, - add_delay_to_timeout=add_delay_to_timeout, - override_timeout_limit=override_timeout_limit, - total_timeout=total_timeout) - return requests - - def send_reqs_batches_and_get_suff_replies( looper: Looper, wallet: Wallet, diff --git a/plenum/test/node_request/test_timestamp/test_timestamp_post_view_change.py b/plenum/test/node_request/test_timestamp/test_timestamp_post_view_change.py index 42f6c950b5..ca88b8959a 100644 --- a/plenum/test/node_request/test_timestamp/test_timestamp_post_view_change.py +++ b/plenum/test/node_request/test_timestamp/test_timestamp_post_view_change.py @@ -1,7 +1,6 @@ from stp_core.loop.eventually import eventually -from plenum.test.helper import send_reqs_to_nodes_and_verify_all_replies, \ - sendRandomRequests, checkViewNoForNodes +from plenum.test.helper import checkViewNoForNodes from plenum.test.node_request.test_timestamp.helper import make_clock_faulty, \ get_timestamp_suspicion_count from plenum.test.test_node import ensureElectionsDone, getNonPrimaryReplicas diff --git a/plenum/test/view_change/helper.py b/plenum/test/view_change/helper.py index a136128c7c..f9d9419ec7 100644 --- a/plenum/test/view_change/helper.py +++ b/plenum/test/view_change/helper.py @@ -2,9 +2,10 @@ from stp_core.types import HA -from plenum.test.delayers import delayNonPrimaries, delay_3pc_messages, reset_delays_and_process_delayeds -from plenum.test.helper import checkViewNoForNodes, sendRandomRequests, \ - sendReqsToNodesAndVerifySuffReplies, send_reqs_to_nodes_and_verify_all_replies, sdk_send_random_requests, \ +from plenum.test.delayers import delayNonPrimaries, delay_3pc_messages, \ + reset_delays_and_process_delayeds +from plenum.test.helper import checkViewNoForNodes, \ + sendReqsToNodesAndVerifySuffReplies, sdk_send_random_requests, \ sdk_send_random_and_check from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected diff --git a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py index 9e689d7083..0bef2be19a 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py @@ -3,11 +3,8 @@ from stp_core.loop.eventually import eventually from plenum.test import waits -from plenum.test.helper import sendRandomRequests, \ - send_reqs_to_nodes_and_verify_all_replies, \ - waitForSufficientRepliesForRequests, \ - checkViewNoForNodes, \ - check_last_ordered_3pc, sdk_send_random_requests, sdk_send_random_request, sdk_get_replies, \ +from plenum.test.helper import checkViewNoForNodes, \ + check_last_ordered_3pc, sdk_send_random_request, sdk_get_replies, \ sdk_send_random_and_check from plenum.test.delayers import delay_3pc_messages, \ reset_delays_and_process_delayeds diff --git a/plenum/test/view_change/test_new_node_joins_after_view_change.py b/plenum/test/view_change/test_new_node_joins_after_view_change.py index e42be2a5f1..8b8c6f3dbc 100644 --- a/plenum/test/view_change/test_new_node_joins_after_view_change.py +++ b/plenum/test/view_change/test_new_node_joins_after_view_change.py @@ -9,8 +9,7 @@ from plenum.test.view_change.helper import ensure_view_change, start_stopped_node from stp_core.loop.eventually import eventually -from plenum.test.helper import send_reqs_to_nodes_and_verify_all_replies, \ - checkViewNoForNodes, stopNodes, sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check from plenum.test.pool_transactions.conftest import nodeThetaAdded from plenum.test.primary_selection.conftest import one_node_added @@ -23,12 +22,10 @@ def all_nodes_view_change( looper, txnPoolNodeSet, - stewardWallet, - steward1, - client1, - wallet1): + sdk_pool_handle, + sdk_wallet_client): for _ in range(5): - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 2) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 2) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper, txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) @@ -36,7 +33,7 @@ def all_nodes_view_change( @pytest.fixture(scope='module') def new_node_in_correct_view(all_nodes_view_change, looper, txnPoolNodeSet, - one_node_added, wallet1, client1): + one_node_added, sdk_pool_handle, sdk_wallet_client): new_node = one_node_added looper.run(eventually(checkViewNoForNodes, txnPoolNodeSet, retryWait=1, timeout=10)) @@ -44,7 +41,8 @@ def new_node_in_correct_view(all_nodes_view_change, looper, txnPoolNodeSet, new_node.view_changer._start_view_change_if_possible, compare_val_to=True)) > 0 assert not new_node.view_changer._next_view_indications - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 2) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 2) def test_new_node_has_same_view_as_others(new_node_in_correct_view): @@ -57,7 +55,8 @@ def test_old_non_primary_restart_after_view_change(new_node_in_correct_view, looper, txnPoolNodeSet, tdir, allPluginsPath, tconf, - wallet1, client1): + sdk_pool_handle, + sdk_wallet_client): """ An existing non-primary node crashes and then view change happens, the crashed node comes back up after view change @@ -72,11 +71,13 @@ def test_old_non_primary_restart_after_view_change(new_node_in_correct_view, remaining_nodes = list(set(txnPoolNodeSet) - {node_to_stop}) # Send some requests before view change - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 5) ensure_view_change(looper, remaining_nodes, custom_timeout=tconf.VIEW_CHANGE_TIMEOUT) ensureElectionsDone(looper, remaining_nodes) # Send some requests after view change - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 5) restarted_node = start_stopped_node(node_to_stop, looper, tconf, tdir, allPluginsPath) diff --git a/plenum/test/view_change/test_queueing_req_from_future_view.py b/plenum/test/view_change/test_queueing_req_from_future_view.py index a049334864..33b042992e 100644 --- a/plenum/test/view_change/test_queueing_req_from_future_view.py +++ b/plenum/test/view_change/test_queueing_req_from_future_view.py @@ -2,9 +2,8 @@ from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger from plenum.test.delayers import icDelay, vcd_delay -from plenum.test.helper import sendRandomRequests, \ - waitForSufficientRepliesForRequests, \ - send_reqs_to_nodes_and_verify_all_replies, sdk_send_random_requests, sdk_get_replies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_requests, \ + sdk_get_replies, sdk_send_random_and_check from plenum.test.test_node import get_last_master_non_primary_node nodeCount = 7 diff --git a/plenum/test/zstack_tests/test_zstack_reconnection.py b/plenum/test/zstack_tests/test_zstack_reconnection.py index f351ead6fd..c948b4fbe6 100644 --- a/plenum/test/zstack_tests/test_zstack_reconnection.py +++ b/plenum/test/zstack_tests/test_zstack_reconnection.py @@ -3,8 +3,7 @@ from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, stopNodes, \ - send_reqs_to_nodes_and_verify_all_replies +from plenum.test.helper import stopNodes, sdk_send_random_and_check from plenum.test.test_node import TestNode, ensureElectionsDone from plenum.common.config_helper import PNodeConfigHelper @@ -19,9 +18,12 @@ def tconf(tconf): return tconf -def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet, client1, wallet1, +def testZStackNodeReconnection(tconf, looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, tdir): - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 1) npr = [n for n in txnPoolNodeSet if not n.hasPrimary] nodeToCrash = npr[0] @@ -60,4 +62,5 @@ def checkFlakyConnected(conn=True): ensureElectionsDone(looper, txnPoolNodeSet, retryWait=2) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) - send_reqs_to_nodes_and_verify_all_replies(looper, wallet1, client1, 10) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 10) From 93fb14459d21f9fc5e25467dfb792437201f1195 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 26 Mar 2018 17:20:32 +0300 Subject: [PATCH 41/73] Deletion of sendRandomRequests. Part 3. Signed-off-by: ArtObr --- .../test_checkpoint_bounds_after_catchup.py | 29 +++++++------- ..._checkpoint_stabilization_after_catchup.py | 32 ++++++++------- .../test_ordering_after_catchup.py | 24 ++++++------ plenum/test/client/test_client_observer.py | 2 + .../test_client_sends_to_f_plus_one_nodes.py | 4 +- .../test_instance_change_with_Delta.py | 4 +- plenum/test/node_catchup/conftest.py | 10 +++++ ...us_2_nodes_including_primary_off_and_on.py | 2 +- .../test_plugin_request_handling.py | 18 ++++----- ...est_add_node_to_pool_with_large_ppseqno.py | 35 +++++++++++------ .../test_catchup_after_view_change.py | 13 ++++--- .../test_new_node_accepts_chosen_primary.py | 13 ++++--- plenum/test/test_dirty_read.py | 36 ----------------- ...ew_change_2_of_4_nodes_with_new_primary.py | 8 ++-- ...t_view_change_gc_all_nodes_random_delay.py | 2 +- ...in_after_view_change_by_primary_restart.py | 39 +++++++++---------- 16 files changed, 134 insertions(+), 137 deletions(-) diff --git a/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py index 87c4bde305..c03af015e1 100644 --- a/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py @@ -1,30 +1,33 @@ -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.pool_transactions.helper import addNewStewardAndNode +from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected CHK_FREQ = 5 def test_upper_bound_of_checkpoint_after_catchup_is_divisible_by_chk_freq( - chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet, - client1, wallet1, tdir, client_tdir, tconf, + chkFreqPatched, looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, sdk_wallet_client, tdir, + client_tdir, tconf, allPluginsPath): - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 4) - _, _, epsilon = addNewStewardAndNode(looper, steward1, stewardWallet, - 'EpsilonSteward', 'Epsilon', - tdir, client_tdir, tconf, - allPluginsPath) - txnPoolNodeSet.append(epsilon) + _, new_node = sdk_add_new_steward_and_node( + looper, sdk_pool_handle, sdk_wallet_steward, + 'EpsilonSteward', 'Epsilon', tdir, tconf, + allPluginsPath=allPluginsPath) + txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) - waitNodeDataEquality(looper, epsilon, *txnPoolNodeSet[:-1]) + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) # Epsilon did not participate in ordering of the batch with EpsilonSteward # NYM transaction and the batch with Epsilon NODE transaction. # Epsilon got these transactions via catch-up. - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 1) - for replica in epsilon.replicas: + for replica in new_node.replicas: assert len(replica.checkpoints) == 1 assert next(iter(replica.checkpoints)) == (7, 10) diff --git a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py index ce154503bd..6aad40dbf9 100644 --- a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py @@ -1,7 +1,7 @@ from plenum.test import waits -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.pool_transactions.helper import addNewStewardAndNode +from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected CHK_FREQ = 5 @@ -9,32 +9,33 @@ def test_second_checkpoint_after_catchup_can_be_stabilized( - chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet, - client1, wallet1, tdir, client_tdir, tconf, + chkFreqPatched, looper, txnPoolNodeSet, steward1, sdk_wallet_steward, + sdk_wallet_client, sdk_pool_handle, tdir, client_tdir, tconf, allPluginsPath): - _, _, epsilon = addNewStewardAndNode(looper, steward1, stewardWallet, - 'EpsilonSteward', 'Epsilon', - tdir, client_tdir, tconf, - allPluginsPath) - txnPoolNodeSet.append(epsilon) + _, new_node = sdk_add_new_steward_and_node( + looper, sdk_pool_handle, sdk_wallet_steward, + 'EpsilonSteward', 'Epsilon', tdir, tconf, + allPluginsPath=allPluginsPath) + txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) - waitNodeDataEquality(looper, epsilon, *txnPoolNodeSet[:-1]) + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) # Epsilon did not participate in ordering of the batch with EpsilonSteward # NYM transaction and the batch with Epsilon NODE transaction. # Epsilon got these transactions via catch-up. - for replica in epsilon.replicas: + for replica in new_node.replicas: assert len(replica.checkpoints) == 0 assert replica.h == 2 assert replica.H == 17 - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 7) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, 7) stabilization_timeout = \ waits.expectedTransactionExecutionTime(len(txnPoolNodeSet)) looper.runFor(stabilization_timeout) - for replica in epsilon.replicas: + for replica in new_node.replicas: assert len(replica.checkpoints) == 2 keys_iter = iter(replica.checkpoints) @@ -51,10 +52,11 @@ def test_second_checkpoint_after_catchup_can_be_stabilized( assert replica.h == 2 assert replica.H == 17 - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_client, 1) looper.runFor(stabilization_timeout) - for replica in epsilon.replicas: + for replica in new_node.replicas: assert len(replica.checkpoints) == 1 keys_iter = iter(replica.checkpoints) diff --git a/plenum/test/checkpoints/test_ordering_after_catchup.py b/plenum/test/checkpoints/test_ordering_after_catchup.py index 09ad59d2dc..bb75cf4ff3 100644 --- a/plenum/test/checkpoints/test_ordering_after_catchup.py +++ b/plenum/test/checkpoints/test_ordering_after_catchup.py @@ -1,10 +1,9 @@ -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \ - checkViewNoForNodes +from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ ensure_all_nodes_have_same_data from plenum.common.util import randomString from plenum.test.test_node import checkNodesConnected -from plenum.test.pool_transactions.helper import addNewStewardAndNode +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node CHK_FREQ = 6 LOG_SIZE = 3 * CHK_FREQ @@ -12,16 +11,16 @@ nodeCount = 4 -def add_new_node(looper, pool_nodes, steward, steward_wallet, - tdir, client_tdir, tconf, all_plugins_path): +def add_new_node(looper, pool_nodes, sdk_pool_handle, sdk_wallet_steward, + tdir, tconf, all_plugins_path): name = randomString(6) node_name = "Node-" + name new_steward_name = "Steward-" + name - _, _, new_node = addNewStewardAndNode(looper, steward, steward_wallet, - new_steward_name, node_name, - tdir, client_tdir, tconf, - all_plugins_path) + _, new_node = sdk_add_new_steward_and_node( + looper, sdk_pool_handle, sdk_wallet_steward, + new_steward_name, node_name, tdir, tconf, + allPluginsPath=all_plugins_path) pool_nodes.append(new_node) looper.run(checkNodesConnected(pool_nodes)) waitNodeDataEquality(looper, new_node, *pool_nodes[:-1]) @@ -33,7 +32,7 @@ def add_new_node(looper, pool_nodes, steward, steward_wallet, def test_ordering_after_more_than_f_nodes_caught_up( - chkFreqPatched, looper, txnPoolNodeSet, steward1, stewardWallet, + chkFreqPatched, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tdir, client_tdir, tconf, allPluginsPath): """ Verifies that more than LOG_SIZE batches can be ordered in one view @@ -43,10 +42,11 @@ def test_ordering_after_more_than_f_nodes_caught_up( initial_view_no = txnPoolNodeSet[0].viewNo for _ in range(2): - add_new_node(looper, txnPoolNodeSet, steward1, stewardWallet, + add_new_node(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tdir, client_tdir, tconf, allPluginsPath) checkViewNoForNodes(txnPoolNodeSet, initial_view_no) - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 20) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, 20) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) checkViewNoForNodes(txnPoolNodeSet, initial_view_no) diff --git a/plenum/test/client/test_client_observer.py b/plenum/test/client/test_client_observer.py index 08cab0549e..ffb561e1de 100644 --- a/plenum/test/client/test_client_observer.py +++ b/plenum/test/client/test_client_observer.py @@ -3,6 +3,7 @@ from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +# TODO: probably delete def test_observer_registration(looper, txnPoolNodeSet, client1): def callable1(*args, **kwargs): print(1) @@ -48,6 +49,7 @@ def callable2(*args, **kwargs): assert len(client1._observers) == 1 +# TODO: probably delete def test_observer_execution(looper, txnPoolNodeSet, client1, wallet1): resp1 = [] resp2 = [] diff --git a/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py b/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py index b94b227189..e5e559ac82 100644 --- a/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py +++ b/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py @@ -1,6 +1,6 @@ from plenum.test.client.conftest import passThroughReqAcked1 -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.malicious_behaviors_client import \ genDoesntSendRequestToSomeNodes from plenum.test.node_catchup.helper import waitNodeDataEquality @@ -10,6 +10,7 @@ reqAcked1 = passThroughReqAcked1 +# TODO: probably delete def testReplyWhenRequestSentToMoreThanFPlusOneNodes(looper, txnPoolNodeSet, fClient, replied1, wallet1): @@ -41,6 +42,7 @@ def chk(req_count=1): more_reqs_count = 5 sendReqsToNodesAndVerifySuffReplies(looper, wallet1, fClient, more_reqs_count, 1) + sdk_send_random_and_check # Ledger is same for all nodes waitNodeDataEquality(looper, alpha, *other_nodes) chk(6) # Since one request is already sent as part of `replied1` diff --git a/plenum/test/monitoring/test_instance_change_with_Delta.py b/plenum/test/monitoring/test_instance_change_with_Delta.py index 5678d2edc1..bea1dc0afb 100644 --- a/plenum/test/monitoring/test_instance_change_with_Delta.py +++ b/plenum/test/monitoring/test_instance_change_with_Delta.py @@ -66,7 +66,7 @@ def ensureAnotherPerfCheck(): @pytest.fixture(scope="module") -def step1(looper, txnPoolNodeSet, wallet1, client1): +def step1(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): startedNodes = txnPoolNodeSet """ stand up a pool of nodes and send 5 requests to client @@ -74,7 +74,7 @@ def step1(looper, txnPoolNodeSet, wallet1, client1): # the master instance has a primary replica, call it P P = getPrimaryReplica(startedNodes) - requests = sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client, 5) # profile_this(sendReqsToNodesAndVerifySuffReplies, looper, client1, 5) return adict(P=P, diff --git a/plenum/test/node_catchup/conftest.py b/plenum/test/node_catchup/conftest.py index 45dfed5f4e..37da7b9394 100644 --- a/plenum/test/node_catchup/conftest.py +++ b/plenum/test/node_catchup/conftest.py @@ -81,6 +81,16 @@ def nodeSetWithNodeAddedAfterSomeTxns( return looper, newNode, client, wallet, newStewardClient, newStewardWallet +@pytest.fixture("module") +def sdk_node_set_with_node_added_after_some_txns( + txnPoolNodeSet, sdk_node_created_after_some_txns): + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_created_after_some_txns + txnPoolNodeSet.append(new_node) + looper.run(checkNodesConnected(txnPoolNodeSet)) + return looper, new_node, sdk_pool_handle, new_steward_wallet_handle + + @pytest.fixture("module") def newNodeCaughtUp(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): looper, newNode, _, _, _, _ = nodeSetWithNodeAddedAfterSomeTxns diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py index 4e5e22027e..b9e995d84c 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py @@ -10,7 +10,7 @@ from plenum.test.test_node import ensureElectionsDone, getRequiredInstances from plenum.test.view_change.helper import start_stopped_node -TestRunningTimeLimitSec = 200 +TestRunningTimeLimitSec = 400 nodeCount = 5 diff --git a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py index ff9c8eb584..1bbb43bf5b 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py @@ -1,7 +1,7 @@ import pytest from plenum.common.constants import TXN_TYPE, DATA -from plenum.common.exceptions import RequestNackedException +from plenum.common.exceptions import RequestNackedException, CommonSdkIOException from plenum.test.helper import sdk_gen_request, \ sdk_sign_and_submit_req_obj, sdk_get_reply, sdk_send_signed_requests, \ sdk_sign_request_strings, sdk_get_and_check_replies @@ -26,9 +26,9 @@ def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, } reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) - with pytest.raises(RequestNackedException) as exc_info: + with pytest.raises(CommonSdkIOException) as exc_info: sdk_get_and_check_replies(looper, reqs) - exc_info.match('attribute is missing or not in proper format') + exc_info.match('Got an error with code 113') op = { TXN_TYPE: AUCTION_START, @@ -36,9 +36,9 @@ def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, } reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) - with pytest.raises(RequestNackedException) as exc_info: + with pytest.raises(CommonSdkIOException) as exc_info: sdk_get_and_check_replies(looper, reqs) - exc_info.match('attribute is missing or not in proper format') + exc_info.match('Got an error with code 113') op = { TXN_TYPE: AUCTION_START, @@ -53,9 +53,9 @@ def test_plugin_static_validation(txn_pool_node_set_post_creation, looper, } reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) - with pytest.raises(RequestNackedException) as exc_info: + with pytest.raises(CommonSdkIOException) as exc_info: sdk_get_and_check_replies(looper, reqs) - exc_info.match('must be present and should be a number') + exc_info.match('Got an error with code 113') op = { TXN_TYPE: PLACE_BID, @@ -75,9 +75,9 @@ def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, } reqs = sdk_sign_request_strings(looper, sdk_wallet_steward, [op, ]) reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) - with pytest.raises(RequestNackedException) as exc_info: + with pytest.raises(CommonSdkIOException) as exc_info: sdk_get_and_check_replies(looper, reqs) - exc_info.match('unknown auction') + exc_info.match('Got an error with code 113') op = { TXN_TYPE: AUCTION_START, diff --git a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py index e936f1af41..03e255abd7 100644 --- a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py +++ b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py @@ -1,8 +1,10 @@ import pytest +from plenum.test.node_request.helper import sdk_ensure_pool_functional + from plenum.common.util import randomString -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.view_change.helper import ensure_several_view_change -from plenum.test.pool_transactions.helper import addNewStewardAndNode +from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected from plenum.test.node_catchup.helper import ensureClientConnectedToNodesAndPoolLedgerSame, waitNodeDataEquality @@ -25,8 +27,8 @@ def _set_ppseqno(nodes, new_ppsn): @pytest.mark.parametrize('do_view_change', [0, 1]) -def test_add_node_to_pool_with_large_ppseqno_diff_views(do_view_change, looper, txnPoolNodeSet, tconf, steward1, - stewardWallet, tdir, client_tdir, allPluginsPath): +def test_add_node_to_pool_with_large_ppseqno_diff_views(do_view_change, looper, txnPoolNodeSet, tconf, sdk_pool_handle, + sdk_wallet_steward, tdir, client_tdir, allPluginsPath): """ Adding a node to the pool while ppSeqNo is big caused a node to stash all the requests because of incorrect watermarks limits set. @@ -44,27 +46,36 @@ def test_add_node_to_pool_with_large_ppseqno_diff_views(do_view_change, looper, assert (big_ppseqno > cur_ppseqno) # ensure pool is working properly - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, 3) assert (cur_ppseqno < _get_ppseqno(txnPoolNodeSet)) _set_ppseqno(txnPoolNodeSet, big_ppseqno) cur_ppseqno = _get_ppseqno(txnPoolNodeSet) assert (big_ppseqno == cur_ppseqno) - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, 3) + assert (cur_ppseqno < _get_ppseqno(txnPoolNodeSet)) new_steward_name = "testClientSteward" + randomString(4) new_node_name = "TestTheta" + randomString(4) - new_steward, new_steward_wallet, new_node = \ - addNewStewardAndNode(looper, steward1, stewardWallet, new_steward_name, - new_node_name, tdir, client_tdir, tconf, allPluginsPath) + new_steward_wallet_handle, new_node = sdk_add_new_steward_and_node( + looper, sdk_pool_handle, sdk_wallet_steward, + new_steward_name, new_node_name, tdir, tconf, + allPluginsPath=allPluginsPath) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, new_steward, *txnPoolNodeSet) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + sdk_wallet_steward, + sdk_pool_handle) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + new_steward_wallet_handle, + sdk_pool_handle) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, 3) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) diff --git a/plenum/test/primary_selection/test_catchup_after_view_change.py b/plenum/test/primary_selection/test_catchup_after_view_change.py index 33c762f608..4bd740d83b 100644 --- a/plenum/test/primary_selection/test_catchup_after_view_change.py +++ b/plenum/test/primary_selection/test_catchup_after_view_change.py @@ -1,6 +1,6 @@ import pytest -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data, \ waitNodeDataInequality from plenum.test.delayers import cr_delay, ppDelay, pDelay, \ @@ -27,7 +27,7 @@ def slow_node(request, txnPoolNodeSet): @pytest.mark.skip(reasone="It's an intermittent test, INDY-722") def test_slow_nodes_catchup_before_selecting_primary_in_new_view( - looper, txnPoolNodeSet, steward1, stewardWallet, tconf, slow_node): + looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tconf, slow_node): """ Delay 3PC to 1 node and then cause view change so by the time the view change happens(each node gets >n-f `INSTANCE_CHANGE`s), the slow node is @@ -43,8 +43,9 @@ def test_slow_nodes_catchup_before_selecting_primary_in_new_view( slow_node.nodeIbStasher.delay(pDelay(2 * delay, 0)) slow_node.nodeIbStasher.delay(cDelay(3 * delay, 0)) for i in range(2): - sendReqsToNodesAndVerifySuffReplies( - looper, stewardWallet, steward1, 20) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_steward, 20) waitNodeDataInequality(looper, slow_node, *fast_nodes) catchup_reply_counts = {n.name: n.ledgerManager.spylog.count( @@ -100,5 +101,7 @@ def slow_node_processed_some(): slow_node.reset_delays_and_process_delayeds() # Make sure pool is functional - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_steward, 5) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) diff --git a/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py b/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py index de4ec93517..f5a5e67632 100644 --- a/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py +++ b/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py @@ -2,13 +2,13 @@ from stp_core.common.log import getlogger -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.test_node import TestNode, TestViewChanger from plenum.test.view_change.helper import ensure_view_change_complete -from plenum.test.node_catchup.conftest import nodeCreatedAfterSomeTxns, \ - nodeSetWithNodeAddedAfterSomeTxns +from plenum.test.node_catchup.conftest import sdk_node_created_after_some_txns, \ + sdk_node_set_with_node_added_after_some_txns from plenum.test.node_catchup.helper import waitNodeDataEquality logger = getlogger() @@ -66,8 +66,8 @@ def txnPoolNodeSet(txnPoolNodeSet, looper, client1, wallet1, def test_new_node_accepts_chosen_primary( - txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): - looper, new_node, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns + txnPoolNodeSet, sdk_node_set_with_node_added_after_some_txns): + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = sdk_node_set_with_node_added_after_some_txns logger.debug("Ensure nodes data equality".format(txnPoolNodeSet[0].viewNo)) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) @@ -87,7 +87,8 @@ def test_new_node_accepts_chosen_primary( logger.debug("Send requests to ensure that pool is working properly, " "viewNo: {}".format(txnPoolNodeSet[0].viewNo)) - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 3) logger.debug("Ensure nodes data equality".format(txnPoolNodeSet[0].viewNo)) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) diff --git a/plenum/test/test_dirty_read.py b/plenum/test/test_dirty_read.py index e386ae0de1..8084cfb85d 100644 --- a/plenum/test/test_dirty_read.py +++ b/plenum/test/test_dirty_read.py @@ -20,39 +20,3 @@ async def serviceReplicas(limit): return await old(limit) node.serviceReplicas = serviceReplicas - - -def test_dirty_read(looper, txnPoolNodeSet, client1, wallet1): - """ - Tests the case when read request comes before write request is - not executed on some nodes - """ - - slow_nodes = list(txnPoolNodeSet)[2:4] - for node in slow_nodes: - logger.debug("Making node {} slow".format(node)) - make_node_slow(node) - - set_request = sendReqsToNodesAndVerifySuffReplies(looper, - wallet1, - client1, - numReqs=1)[0] - - received_replies = getRepliesFromClientInbox(inbox=client1.inBox, - reqId=set_request.reqId) - - seq_no = received_replies[0]["result"]["seqNo"] - get_request = [wallet1.signOp({ - TXN_TYPE: GET_TXN, - f.LEDGER_ID.nm: DOMAIN_LEDGER_ID, - DATA: seq_no - })] - send_signed_requests(client1, get_request) - waitForSufficientRepliesForRequests(looper, - client1, - requests=get_request) - received_replies = getRepliesFromClientInbox(inbox=client1.inBox, - reqId=get_request[0].reqId) - results = [str(reply['result'][DATA]) for reply in received_replies] - - assert len(set(results)) == 1 diff --git a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py index 52724d4b89..7140cb3bf8 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_2_of_4_nodes_with_new_primary.py @@ -8,7 +8,7 @@ def slow_nodes(node_set): def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary( - txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_handle): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the the first and the last non-primary node @@ -18,11 +18,11 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - sdk_pool_handle, sdk_wallet_handle) + sdk_pool_handle, sdk_wallet_client) def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary_long_delay( - txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_handle): + txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_client): """ - Slow processing 3PC messages for 2 of 4 node (2>f) - Slow the the first and the last non-primary node @@ -32,5 +32,5 @@ def test_view_change_in_between_3pc_2_of_4_nodes_with_new_primary_long_delay( """ view_change_in_between_3pc(looper, txnPoolNodeSet, slow_nodes(txnPoolNodeSet), - sdk_pool_handle, sdk_wallet_handle, + sdk_pool_handle, sdk_wallet_client, slow_delay=20) diff --git a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py index 0bef2be19a..8a535c3b60 100644 --- a/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py +++ b/plenum/test/view_change/slow_nodes/test_view_change_gc_all_nodes_random_delay.py @@ -102,7 +102,7 @@ def checkPrePrepareSentAtLeastByPrimary(): # -> they should be ordered # -> last_ordered_3pc = (+2, 1) reset_delays_and_process_delayeds(txnPoolNodeSet) - sdk_get_replies(looper, requests) + sdk_get_replies(looper, [requests]) checkViewNoForNodes(txnPoolNodeSet, viewNo) last_ordered_3pc = (viewNo, 1) diff --git a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py index d81167e21e..dc93cf60ce 100644 --- a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py +++ b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py @@ -4,7 +4,7 @@ from plenum.test.view_change.helper import ensure_all_nodes_have_same_data, \ ensure_view_change_by_primary_restart, start_stopped_node from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState, POOL_LEDGER_ID -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually @@ -12,7 +12,7 @@ waitNodeDataEquality from plenum.common.util import randomString from plenum.test.test_node import checkNodesConnected -from plenum.test.pool_transactions.helper import addNewStewardAndNode +from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node from plenum.test import waits from plenum.common.startable import Mode @@ -41,19 +41,14 @@ def catchuped(node): assert node.mode == Mode.participating -def add_new_node(looper, nodes, steward, steward_wallet, +def add_new_node(looper, nodes, sdk_pool_handle, sdk_wallet_steward, tdir, client_tdir, tconf, all_plugins_path, name=None): node_name = name or "Psi" new_steward_name = "testClientSteward" + randomString(3) - new_steward, new_steward_wallet, new_node = addNewStewardAndNode(looper, - steward, - steward_wallet, - new_steward_name, - node_name, - tdir, - client_tdir, - tconf, - all_plugins_path) + _, new_node = sdk_add_new_steward_and_node( + looper, sdk_pool_handle, sdk_wallet_steward, + new_steward_name, node_name, tdir, tconf, + allPluginsPath=all_plugins_path) nodes.append(new_node) looper.run(checkNodesConnected(nodes)) timeout = waits.expectedPoolCatchupTime(nodeCount=len(nodes)) @@ -64,7 +59,8 @@ def add_new_node(looper, nodes, steward, steward_wallet, def test_6th_node_join_after_view_change_by_master_restart( looper, txnPoolNodeSet, tdir, tconf, - allPluginsPath, steward1, stewardWallet, + allPluginsPath, sdk_pool_handle, + sdk_wallet_steward, client_tdir, limitTestRunningTime): """ Test steps: @@ -88,18 +84,20 @@ def test_6th_node_join_after_view_change_by_master_restart( for node in pool_of_nodes: looper.run(eventually(catchuped, node, timeout=2 * timeout)) ensure_all_nodes_have_same_data(looper, pool_of_nodes, custom_timeout=timeout) - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, 5) new_epsilon_node = add_new_node(looper, pool_of_nodes, - steward1, - stewardWallet, + sdk_pool_handle, + sdk_wallet_steward, tdir, client_tdir, tconf, allPluginsPath, name='Epsilon') - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, 5) """ check that pool and domain ledgers for new node are in synced state """ @@ -120,11 +118,12 @@ def test_6th_node_join_after_view_change_by_master_restart( timeout = waits.expectedPoolCatchupTime(nodeCount=len(pool_of_nodes)) for node in pool_of_nodes: looper.run(eventually(catchuped, node, timeout=2 * timeout)) - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 2) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, 2) new_psi_node = add_new_node(looper, pool_of_nodes, - steward1, - stewardWallet, + sdk_pool_handle, + sdk_wallet_steward, tdir, client_tdir, tconf, From 4b649c16ee332cc7ebbe3648d73fd70c69a79e47 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 26 Mar 2018 18:11:46 +0300 Subject: [PATCH 42/73] Little fix Signed-off-by: ArtObr --- ...lus_2_nodes_including_primary_off_and_on.py | 18 +++++++++--------- plenum/test/view_change/helper.py | 10 +++++----- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py index b9e995d84c..d2f4efb729 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_including_primary_off_and_on.py @@ -1,16 +1,15 @@ import pytest -from plenum.common.exceptions import PoolLedgerTimeoutException from plenum.test import waits from plenum.test.helper import waitForViewChange, checkViewNoForNodes, \ sdk_send_random_and_check, sdk_send_random_requests, sdk_get_replies, \ - sdk_check_reply + sdk_check_reply, sdk_eval_timeout from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.test_node import ensureElectionsDone, getRequiredInstances from plenum.test.view_change.helper import start_stopped_node -TestRunningTimeLimitSec = 400 +TestRunningTimeLimitSec = 200 nodeCount = 5 @@ -25,6 +24,7 @@ def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): + timeout = sdk_eval_timeout(1, len(txnPoolNodeSet)) nodes = txnPoolNodeSet sdk_send_random_and_check(looper, txnPoolNodeSet, @@ -51,8 +51,8 @@ def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( sdk_pool_handle, sdk_wallet_client, 1) - with pytest.raises(PoolLedgerTimeoutException): - req_res = sdk_get_replies(looper, sdk_reqs3) + with pytest.raises(TimeoutError): + req_res = sdk_get_replies(looper, sdk_reqs3, timeout=timeout) sdk_check_reply(req_res[0]) stop_node(nodes[2], looper, nodes) @@ -64,8 +64,8 @@ def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( sdk_pool_handle, sdk_wallet_client, 1) - with pytest.raises(PoolLedgerTimeoutException): - req_res = sdk_get_replies(looper, sdk_reqs4) + with pytest.raises(TimeoutError): + req_res = sdk_get_replies(looper, sdk_reqs4, timeout=timeout) sdk_check_reply(req_res[0]) nodes[2] = start_stopped_node(nodes[2], looper, tconf, tdir, allPluginsPath) @@ -76,8 +76,8 @@ def test_quorum_after_f_plus_2_nodes_including_primary_turned_off_and_later_on( sdk_pool_handle, sdk_wallet_client, 1) - with pytest.raises(PoolLedgerTimeoutException): - req_res = sdk_get_replies(looper, sdk_reqs5) + with pytest.raises(TimeoutError): + req_res = sdk_get_replies(looper, sdk_reqs5, timeout=timeout) sdk_check_reply(req_res[0]) nodes[1] = start_stopped_node(nodes[1], looper, tconf, tdir, allPluginsPath) diff --git a/plenum/test/view_change/helper.py b/plenum/test/view_change/helper.py index f9d9419ec7..ae8c1609c2 100644 --- a/plenum/test/view_change/helper.py +++ b/plenum/test/view_change/helper.py @@ -283,12 +283,12 @@ def ensure_view_change_complete_by_primary_restart( def view_change_in_between_3pc(looper, nodes, slow_nodes, sdk_pool_handle, - sdk_wallet_handle, + sdk_wallet_client, slow_delay=1, wait=None): - sdk_send_random_and_check(looper, nodes, sdk_pool_handle, sdk_wallet_handle, 4) + sdk_send_random_and_check(looper, nodes, sdk_pool_handle, sdk_wallet_client, 4) delay_3pc_messages(slow_nodes, 0, delay=slow_delay) - sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_handle, 10) + sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 10) if wait: looper.runFor(wait) @@ -297,9 +297,9 @@ def view_change_in_between_3pc(looper, nodes, slow_nodes, reset_delays_and_process_delayeds(slow_nodes) sdk_send_random_and_check(looper, nodes, sdk_pool_handle, - sdk_wallet_handle, 5, total_timeout=30) + sdk_wallet_client, 5, total_timeout=30) sdk_send_random_and_check(looper, nodes, sdk_pool_handle, - sdk_wallet_handle, 5, total_timeout=30) + sdk_wallet_client, 5, total_timeout=30) def view_change_in_between_3pc_random_delays( From 7ea516b009b55beaf60069a8558eed53701d125b Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 26 Mar 2018 19:54:55 +0300 Subject: [PATCH 43/73] Deletion of sendRandomRequest. Part 4. Signed-off-by: ArtObr --- .../test_ordering_after_catchup.py | 2 +- .../test_catchup_delayed_nodes.py | 44 +++++++++----- .../test/node_catchup/test_catchup_demoted.py | 55 ++++++++++-------- .../node_catchup/test_catchup_f_plus_one.py | 15 +++-- .../test_catchup_inlcuding_3PC.py | 2 +- .../test_primary_election_case6.py | 9 ++- .../test_new_node_accepts_chosen_primary.py | 5 +- .../test_primary_selection.py | 13 ++--- ..._selection_after_demoted_node_promotion.py | 57 +++++++++---------- ...after_primary_demotion_and_pool_restart.py | 32 ++++++----- ...on_after_primary_demotion_and_promotion.py | 40 +++++++------ ...after_primary_demotion_and_view_changes.py | 38 +++++++------ .../test_recover_more_than_f_failure.py | 11 ++-- .../test_recover_primary_no_view_change.py | 11 ++-- plenum/test/view_change/helper.py | 15 ++--- .../view_change/test_disable_view_change.py | 8 ++- plenum/test/view_change/test_view_change.py | 16 ++++-- .../test_view_change_min_cathup_timeout.py | 15 +++-- .../test_view_change_on_master_degraded.py | 15 +++-- 19 files changed, 230 insertions(+), 173 deletions(-) diff --git a/plenum/test/checkpoints/test_ordering_after_catchup.py b/plenum/test/checkpoints/test_ordering_after_catchup.py index bb75cf4ff3..287594b410 100644 --- a/plenum/test/checkpoints/test_ordering_after_catchup.py +++ b/plenum/test/checkpoints/test_ordering_after_catchup.py @@ -43,7 +43,7 @@ def test_ordering_after_more_than_f_nodes_caught_up( for _ in range(2): add_new_node(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, - tdir, client_tdir, tconf, allPluginsPath) + tdir, tconf, allPluginsPath) checkViewNoForNodes(txnPoolNodeSet, initial_view_no) sdk_send_random_and_check(looper, txnPoolNodeSet, diff --git a/plenum/test/node_catchup/test_catchup_delayed_nodes.py b/plenum/test/node_catchup/test_catchup_delayed_nodes.py index c88afeea24..b5ae678207 100644 --- a/plenum/test/node_catchup/test_catchup_delayed_nodes.py +++ b/plenum/test/node_catchup/test_catchup_delayed_nodes.py @@ -4,9 +4,9 @@ from plenum.test import waits from plenum.test.delayers import cpDelay -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.pool_transactions.helper import addNewStewardAndNode +from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected logger = getlogger() @@ -17,8 +17,11 @@ @pytest.mark.skip(reason="SOV-551. Incomplete implementation") -def testCatchupDelayedNodes(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns, - txnPoolCliNodeReg, tdirWithPoolTxns, tconf, +def testCatchupDelayedNodes(txnPoolNodeSet, + sdk_node_set_with_node_added_after_some_txns, + sdk_wallet_steward, + txnPoolCliNodeReg, tdirWithPoolTxns, + tconf, tdir, allPluginsPath): """ Node sends catchup request to other nodes for only those sequence numbers @@ -28,7 +31,8 @@ def testCatchupDelayedNodes(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns, not receive any catchup requests :return: """ - looper, _, _, _, client, wallet = nodeSetWithNodeAddedAfterSomeTxns + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_set_with_node_added_after_some_txns stewardXName = "testClientStewardX" nodeXName = "Zeta" stewardYName = "testClientStewardY" @@ -37,14 +41,25 @@ def testCatchupDelayedNodes(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns, nodeZName = "Theta" delayX = 45 delayY = 2 - stewardX, nodeX = addNewStewardAndNode(looper, client, stewardXName, - nodeXName, - tdirWithPoolTxns, tconf, - allPluginsPath, autoStart=False) - stewardY, nodeY = addNewStewardAndNode(looper, client, stewardYName, - nodeYName, - tdirWithPoolTxns, tconf, - allPluginsPath, autoStart=False) + stewardX, nodeX = sdk_add_new_steward_and_node(looper, + sdk_pool_handle, + sdk_wallet_steward, + stewardXName, + nodeXName, + tdir, + tconf, + autoStart=False, + allPluginsPath=allPluginsPath) + + stewardY, nodeY = sdk_add_new_steward_and_node(looper, + sdk_pool_handle, + sdk_wallet_steward, + stewardYName, + nodeYName, + tdir, + tconf, + autoStart=False, + allPluginsPath=allPluginsPath) nodeX.nodeIbStasher.delay(cpDelay(delayX)) nodeY.nodeIbStasher.delay(cpDelay(delayY)) looper.add(nodeX) @@ -60,7 +75,8 @@ def testCatchupDelayedNodes(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns, nodeX.stop() nodeY.stop() logger.debug("Sending requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 50) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 50) logger.debug("Starting the 2 stopped nodes, {} and {}".format(nodeX.name, nodeY.name)) nodeX.start(looper.loop) diff --git a/plenum/test/node_catchup/test_catchup_demoted.py b/plenum/test/node_catchup/test_catchup_demoted.py index 9f6734421a..0c50f68f41 100644 --- a/plenum/test/node_catchup/test_catchup_demoted.py +++ b/plenum/test/node_catchup/test_catchup_demoted.py @@ -1,9 +1,11 @@ +from plenum.common.util import hexToFriendly + from plenum.common.constants import ALIAS, SERVICES, VALIDATOR -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ checkNodeDataForInequality from plenum.test.pool_transactions.helper import \ - updateNodeData + updateNodeData, sdk_send_update_node from stp_core.common.log import getlogger from plenum.test.node_catchup.conftest import whitelist @@ -12,42 +14,45 @@ def test_catch_up_after_demoted( - txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): + txnPoolNodeSet, + sdk_node_set_with_node_added_after_some_txns, + sdk_wallet_client): logger.info( "1. add a new node after sending some txns and check that catch-up " "is done (the new node is up to date)") - looper, newNode, client, wallet, newStewardClient, \ - newStewardWallet = nodeSetWithNodeAddedAfterSomeTxns - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:4]) + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_set_with_node_added_after_some_txns + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4]) logger.info("2. turn the new node off (demote)") - node_data = { - ALIAS: newNode.name, - SERVICES: [] - } - updateNodeData(looper, newStewardClient, - newStewardWallet, newNode, - node_data) + node_dest = hexToFriendly(new_node.nodestack.verhex) + sdk_send_update_node(looper, new_steward_wallet_handle, + sdk_pool_handle, + node_dest, new_node.name, + None, None, + None, None, + services=[]) logger.info("3. send more requests, " "so that the new node's state is outdated") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) - checkNodeDataForInequality(newNode, *txnPoolNodeSet[:-1]) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 5) + checkNodeDataForInequality(new_node, *txnPoolNodeSet[:-1]) logger.info("4. turn the new node on") - node_data = { - ALIAS: newNode.name, - SERVICES: [VALIDATOR] - } - updateNodeData(looper, newStewardClient, - newStewardWallet, newNode, - node_data) + sdk_send_update_node(looper, new_steward_wallet_handle, + sdk_pool_handle, + node_dest, new_node.name, + None, None, + None, None, + services=[VALIDATOR]) logger.info("5. make sure catch-up is done " "(the new node is up to date again)") - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1]) + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) logger.info("6. send more requests and make sure " "that the new node participates in processing them") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 10) - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1]) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 10) + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) diff --git a/plenum/test/node_catchup/test_catchup_f_plus_one.py b/plenum/test/node_catchup/test_catchup_f_plus_one.py index 8988850126..6b0efcadd1 100644 --- a/plenum/test/node_catchup/test_catchup_f_plus_one.py +++ b/plenum/test/node_catchup/test_catchup_f_plus_one.py @@ -1,6 +1,6 @@ from stp_core.common.log import getlogger from plenum.common.config_helper import PNodeConfigHelper -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ waitNodeDataInequality, checkNodeDataForEquality from plenum.test.pool_transactions.helper import \ @@ -14,12 +14,15 @@ txnCount = 5 -def testNodeCatchupFPlusOne(txnPoolNodeSet, poolAfterSomeTxns, tconf, tdir, +def testNodeCatchupFPlusOne(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_steward, + tconf, tdir, tdirWithPoolTxns, allPluginsPath, testNodeClass): """ Check that f+1 nodes is enough for catchup """ - looper, client, wallet = poolAfterSomeTxns assert len(txnPoolNodeSet) == 4 @@ -33,7 +36,8 @@ def testNodeCatchupFPlusOne(txnPoolNodeSet, poolAfterSomeTxns, tconf, tdir, looper.removeProdable(node0) logger.debug("Sending requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 5) logger.debug("Stopping node1 with pool ledger size {}". format(node1.poolManager.txnSeqNo)) @@ -61,5 +65,6 @@ def testNodeCatchupFPlusOne(txnPoolNodeSet, poolAfterSomeTxns, tconf, tdir, waitNodeDataEquality(looper, node0, *txnPoolNodeSet[:-2]) logger.debug("Sending more requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 2) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 2) checkNodeDataForEquality(node0, *txnPoolNodeSet[:-2]) diff --git a/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py b/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py index b64478dd7c..5626825d04 100644 --- a/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py +++ b/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py @@ -43,7 +43,7 @@ def pre_check(tconf, looper, txnPoolNodeSet, tdirWithClientPoolTxns, looper.add(client) looper.run(client.ensureConnectedToNodes()) for i in range(tconf.ProcessedBatchMapsToKeep - 1): - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1) + sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1) # All node maintain the same map from txn range to 3PC looper.run(eventually(chk_if_equal_txn_to_3pc, txnPoolNodeSet)) diff --git a/plenum/test/primary_election/test_primary_election_case6.py b/plenum/test/primary_election/test_primary_election_case6.py index f658b9774b..ce25a60d1e 100644 --- a/plenum/test/primary_election/test_primary_election_case6.py +++ b/plenum/test/primary_election/test_primary_election_case6.py @@ -2,7 +2,7 @@ from plenum.common.messages.node_messages import Nomination, Reelection, Primary from plenum.test.delayers import delay -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.test_node import checkNodesConnected, \ checkProtocolInstanceSetup from stp_core.loop.eventually import eventually @@ -56,11 +56,14 @@ def chk(): @pytest.mark.skip('Nodes use round robin primary selection') -def test_primary_election_case6(elections_done, looper, client1, wallet1): +def test_primary_election_case6(elections_done, txnPoolNodeSet, looper, + sdk_pool_handle, + sdk_wallet_steward): """ A is disconnected with B so A does not get any Nomination/Primary from B (simulated by a large delay). A gets Nominations delayed due to which is sends Primary only after it has received Primary from other 2 nodes. A should still be able to select a primary and the pool should function. """ - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 5) diff --git a/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py b/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py index f5a5e67632..f78d924a44 100644 --- a/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py +++ b/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py @@ -40,7 +40,7 @@ def testNodeClass(patchPluginManager): @pytest.fixture("module") -def txnPoolNodeSet(txnPoolNodeSet, looper, client1, wallet1, +def txnPoolNodeSet(txnPoolNodeSet, looper, sdk_pool_handle, sdk_wallet_steward, tconf, tdirWithPoolTxns, allPluginsPath): logger.debug("Do several view changes to round the list of primaries") @@ -50,7 +50,8 @@ def txnPoolNodeSet(txnPoolNodeSet, looper, client1, wallet1, ensure_view_change_complete(looper, txnPoolNodeSet) logger.debug("Send requests to ensure that pool is working properly, " "viewNo: {}".format(txnPoolNodeSet[0].viewNo)) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) logger.debug("Pool is ready, current viewNo: {}".format(txnPoolNodeSet[0].viewNo)) diff --git a/plenum/test/primary_selection/test_primary_selection.py b/plenum/test/primary_selection/test_primary_selection.py index c4764350c5..bc9541acc8 100644 --- a/plenum/test/primary_selection/test_primary_selection.py +++ b/plenum/test/primary_selection/test_primary_selection.py @@ -2,7 +2,7 @@ import pytest -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.primary_selection.helper import \ check_rank_consistent_across_each_node from plenum.test.view_change.helper import ensure_view_change @@ -12,12 +12,6 @@ from plenum.test import waits from plenum.test.test_node import checkProtocolInstanceSetup, getPrimaryReplica, ensureElectionsDone -# noinspection PyUnresolvedReferences -from plenum.test.view_change.conftest import viewNo - -# noinspection PyUnresolvedReferences -from plenum.test.conftest import looper, client1, wallet1, clientAndWallet1 - nodeCount = 7 @@ -29,7 +23,7 @@ def primaryReplicas(txnPoolNodeSet): # noinspection PyIncorrectDocstring def testPrimarySelectionAfterPoolReady( - looper, txnPoolNodeSet, wallet1, client1): + looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward): """ Once the pool is ready(node has connected to at least 3 other nodes), appropriate primary replicas should be selected. @@ -72,7 +66,8 @@ def checkPrimaryPlacement(): # Check if every protocol instance has one and only one primary and any node # has no more than one primary checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 5) @pytest.fixture(scope='module') diff --git a/plenum/test/primary_selection/test_primary_selection_after_demoted_node_promotion.py b/plenum/test/primary_selection/test_primary_selection_after_demoted_node_promotion.py index 59d7e0f1c4..2e69a6fc92 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_demoted_node_promotion.py +++ b/plenum/test/primary_selection/test_primary_selection_after_demoted_node_promotion.py @@ -1,14 +1,15 @@ import pytest +from plenum.common.util import hexToFriendly from stp_core.common.log import getlogger -from plenum.common.constants import ALIAS, SERVICES, VALIDATOR +from plenum.common.constants import VALIDATOR -from plenum.test.helper import checkViewNoForNodes, \ - sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check -from plenum.test.pool_transactions.helper import updateNodeData, \ - disconnect_node_and_ensure_disconnected +from plenum.test.pool_transactions.helper import \ + disconnect_node_and_ensure_disconnected, sdk_send_update_node +from plenum.test.pool_transactions.conftest import sdk_node_theta_added from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.view_change.helper import ensure_view_change_complete, \ @@ -26,7 +27,8 @@ def check_all_nodes_the_same_pool_list(nodes): def test_primary_selection_after_demoted_node_promotion( - looper, txnPoolNodeSet, nodeThetaAdded, + looper, txnPoolNodeSet, sdk_node_theta_added, + sdk_pool_handle, tconf, tdir, allPluginsPath): """ Demote non-primary node @@ -41,41 +43,36 @@ def test_primary_selection_after_demoted_node_promotion( demoted / promoted node as a primary for some instanse """ - nodeThetaSteward, nodeThetaStewardWallet, nodeTheta = nodeThetaAdded + new_steward_wallet, new_node = sdk_node_theta_added # viewNo0 = checkViewNoForNodes(txnPoolNodeSet) check_all_nodes_the_same_pool_list(txnPoolNodeSet) logger.info("1. Demote node Theta") - node_data = { - ALIAS: nodeTheta.name, - SERVICES: [] - } - updateNodeData(looper, nodeThetaSteward, - nodeThetaStewardWallet, nodeTheta, node_data) - remainingNodes = list(set(txnPoolNodeSet) - {nodeTheta}) + node_dest = hexToFriendly(new_node.nodestack.verhex) + sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, + node_dest, new_node.name, None, None, None, None, + []) + remainingNodes = list(set(txnPoolNodeSet) - {new_node}) check_all_nodes_the_same_pool_list(remainingNodes) # ensure pool is working properly - sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, - nodeThetaSteward, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet, 3) # TODO view change might happen unexpectedly by unknown reason # checkViewNoForNodes(remainingNodes, expectedViewNo=viewNo0) logger.info("2. Promote node Theta back") - node_data = { - ALIAS: nodeTheta.name, - SERVICES: [VALIDATOR] - } - updateNodeData(looper, nodeThetaSteward, - nodeThetaStewardWallet, nodeTheta, node_data) + sdk_send_update_node(looper, new_steward_wallet, sdk_pool_handle, + node_dest, new_node.name, None, None, None, None, + [VALIDATOR]) check_all_nodes_the_same_pool_list(txnPoolNodeSet) # ensure pool is working properly - sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, - nodeThetaSteward, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet, 3) # checkViewNoForNodes(txnPoolNodeSet, expectedViewNo=viewNo0) logger.info("3. Restart one node") @@ -86,8 +83,8 @@ def test_primary_selection_after_demoted_node_promotion( looper.removeProdable(stopped_node) remainingNodes = list(set(txnPoolNodeSet) - {stopped_node}) # ensure pool is working properly - sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, - nodeThetaSteward, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet, 3) # checkViewNoForNodes(remainingNodes, expectedViewNo=viewNo0) # start node @@ -96,8 +93,8 @@ def test_primary_selection_after_demoted_node_promotion( txnPoolNodeSet = remainingNodes + [restartedNode] ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # ensure pool is working properly - sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, - nodeThetaSteward, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet, 3) # checkViewNoForNodes(txnPoolNodeSet, expectedViewNo=viewNo0) logger.info("4. Do view changes to check that nodeTheta will be chosen " @@ -105,5 +102,5 @@ def test_primary_selection_after_demoted_node_promotion( while txnPoolNodeSet[0].viewNo < 4: ensure_view_change_complete(looper, txnPoolNodeSet) # ensure pool is working properly - sendReqsToNodesAndVerifySuffReplies(looper, nodeThetaStewardWallet, - nodeThetaSteward, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet, 3) diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py index 3a9db246ef..435077e605 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py @@ -1,12 +1,14 @@ +from plenum.common.util import hexToFriendly + from stp_core.common.log import getlogger -from plenum.common.constants import ALIAS, SERVICES -from plenum.test.pool_transactions.helper import updateNodeData +from plenum.common.constants import ALIAS, SERVICES, VALIDATOR +from plenum.test.pool_transactions.helper import updateNodeData, sdk_send_update_node from plenum.test.test_node import TestNode, checkNodesConnected, \ ensureElectionsDone from plenum.test.helper import checkViewNoForNodes, \ - sendReqsToNodesAndVerifySuffReplies + sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.primary_selection.helper import getPrimaryNodesIdxs from plenum.common.config_helper import PNodeConfigHelper @@ -15,7 +17,9 @@ def test_primary_selection_after_primary_demotion_and_pool_restart(looper, - txnPoolNodeSet, stewardAndWalletForMasterNode, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_steward, txnPoolMasterNodes, tdir, tconf): """ @@ -25,19 +29,20 @@ def test_primary_selection_after_primary_demotion_and_pool_restart(looper, logger.info("1. turn off the node which has primary replica for master instanse") master_node = txnPoolMasterNodes[0] - client, wallet = stewardAndWalletForMasterNode - - node_data = { - ALIAS: master_node.name, - SERVICES: [] - } - updateNodeData(looper, client, wallet, master_node, node_data) + node_dest = hexToFriendly(master_node.nodestack.verhex) + sdk_send_update_node(looper, sdk_wallet_steward, + sdk_pool_handle, + node_dest, master_node.name, + None, None, + None, None, + services=[]) restNodes = [node for node in txnPoolNodeSet if node.name != master_node.name] ensureElectionsDone(looper, restNodes) # ensure pool is working properly - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) logger.info("2. restart pool") # Stopping existing nodes @@ -62,7 +67,8 @@ def test_primary_selection_after_primary_demotion_and_pool_restart(looper, looper.run(checkNodesConnected(restNodes)) ensureElectionsDone(looper, restNodes) checkViewNoForNodes(restNodes, 0) - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) primariesIdxs = getPrimaryNodesIdxs(restNodes) assert restNodes[primariesIdxs[0]].name != master_node.name diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_promotion.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_promotion.py index 58af8b3b61..a9da12e69e 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_promotion.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_promotion.py @@ -1,14 +1,16 @@ -from plenum.common.constants import ALIAS, SERVICES, VALIDATOR -from plenum.test.pool_transactions.helper import updateNodeData +from plenum.common.util import hexToFriendly + +from plenum.common.constants import VALIDATOR +from plenum.test.pool_transactions.helper import sdk_send_update_node from plenum.test.test_node import ensureElectionsDone -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.primary_selection.helper import getPrimaryNodesIdxs def test_primary_selection_after_demoted_primary_node_promotion( - looper, txnPoolNodeSet, stewardAndWalletForMasterNode, + looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, txnPoolMasterNodes): """ Demote primary of master instance, wait for view change and promote it back. @@ -23,14 +25,15 @@ def test_primary_selection_after_demoted_primary_node_promotion( assert primariesIdxs[1] == 1 master_node = txnPoolMasterNodes[0] - client, wallet = stewardAndWalletForMasterNode # Demote primary of master instance. - node_data = { - ALIAS: master_node.name, - SERVICES: [] - } - updateNodeData(looper, client, wallet, master_node, node_data) + node_dest = hexToFriendly(master_node.nodestack.verhex) + sdk_send_update_node(looper, sdk_wallet_steward, + sdk_pool_handle, + node_dest, master_node.name, + None, None, + None, None, + services=[]) restNodes = [node for node in txnPoolNodeSet if node.name != master_node.name] ensureElectionsDone(looper, restNodes) @@ -41,17 +44,20 @@ def test_primary_selection_after_demoted_primary_node_promotion( assert primariesIdxs[0] == 1 # Ensure pool is working properly. - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) # Promote demoted node back. - node_data = { - ALIAS: master_node.name, - SERVICES: [VALIDATOR] - } - updateNodeData(looper, client, wallet, master_node, node_data) + sdk_send_update_node(looper, sdk_wallet_steward, + sdk_pool_handle, + node_dest, master_node.name, + None, None, + None, None, + services=[VALIDATOR]) # Ensure pool is working properly. - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) # Check that there are two instances again, check their primaries. primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py index f846de58e1..2703eae639 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_view_changes.py @@ -1,11 +1,10 @@ -from stp_core.common.log import getlogger +from plenum.common.util import hexToFriendly -from plenum.common.constants import ALIAS, SERVICES +from stp_core.common.log import getlogger -from plenum.test.pool_transactions.helper import updateNodeData +from plenum.test.pool_transactions.helper import sdk_send_update_node -from plenum.test.helper import checkViewNoForNodes, \ - sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check from plenum.test.test_node import ensureElectionsDone from plenum.test.view_change.helper import ensure_view_change_complete @@ -13,7 +12,9 @@ def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPoolNodeSet, - stewardAndWalletForMasterNode, txnPoolMasterNodes): + sdk_pool_handle, + sdk_wallet_steward, + txnPoolMasterNodes): """ Demote primary and do multiple view changes forcing primaries rotation. Demoted primary should be skipped without additional view changes. @@ -24,12 +25,13 @@ def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPo logger.info("1. turn off the node which has primary replica for master instanse, " " this should trigger view change") master_node = txnPoolMasterNodes[0] - client, wallet = stewardAndWalletForMasterNode - node_data = { - ALIAS: master_node.name, - SERVICES: [] - } - updateNodeData(looper, client, wallet, master_node, node_data) + node_dest = hexToFriendly(master_node.nodestack.verhex) + sdk_send_update_node(looper, sdk_wallet_steward, + sdk_pool_handle, + node_dest, master_node.name, + None, None, + None, None, + services=[]) restNodes = [node for node in txnPoolNodeSet \ if node.name != master_node.name] @@ -43,7 +45,8 @@ def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPo assert restNodes[0].replicas[0].primaryName != master_node.name # ensure pool is working properly - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) logger.info("2. force view change 2 and check final viewNo") ensure_view_change_complete(looper, restNodes) @@ -52,7 +55,8 @@ def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPo assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo2 == viewNo1 + 1 - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) logger.info("3. force view change 3 and check final viewNo") ensure_view_change_complete(looper, restNodes) @@ -60,7 +64,8 @@ def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPo assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo3 == viewNo2 + 1 - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) logger.info("4. force view change 4 and check final viewNo") ensure_view_change_complete(looper, restNodes) @@ -68,4 +73,5 @@ def test_primary_selection_after_primary_demotion_and_view_changes(looper, txnPo assert restNodes[0].replicas[0].primaryName != master_node.name assert viewNo4 == viewNo3 + 1 - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=3) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) diff --git a/plenum/test/primary_selection/test_recover_more_than_f_failure.py b/plenum/test/primary_selection/test_recover_more_than_f_failure.py index fe1bd3e56a..9304214bc9 100644 --- a/plenum/test/primary_selection/test_recover_more_than_f_failure.py +++ b/plenum/test/primary_selection/test_recover_more_than_f_failure.py @@ -3,7 +3,7 @@ from stp_core.common.log import getlogger from plenum.test.helper import stopNodes, waitForViewChange, \ - sendReqsToNodesAndVerifySuffReplies + sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected @@ -14,7 +14,8 @@ def test_recover_stop_primaries(looper, checkpoint_size, txnPoolNodeSet, - allPluginsPath, tdir, tconf, client1, wallet1): + allPluginsPath, tdir, tconf, sdk_pool_handle, + sdk_wallet_steward): """ Test that we can recover after having more than f nodes disconnected: - stop current master primary (Alpha) @@ -38,7 +39,8 @@ def test_recover_stop_primaries(looper, checkpoint_size, txnPoolNodeSet, logger.info("send at least one checkpoint") assert nodes_do_not_have_checkpoints(*active_nodes) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=2 * checkpoint_size) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 2 * checkpoint_size) assert nodes_have_checkpoints(*active_nodes) ensure_all_nodes_have_same_data(looper, nodes=active_nodes) @@ -58,7 +60,8 @@ def test_recover_stop_primaries(looper, checkpoint_size, txnPoolNodeSet, ensure_all_nodes_have_same_data(looper, nodes=active_nodes) logger.info("Check if the pool is able to process requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=10 * checkpoint_size) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 10 * checkpoint_size) ensure_all_nodes_have_same_data(looper, nodes=active_nodes) assert nodes_have_checkpoints(*active_nodes) diff --git a/plenum/test/primary_selection/test_recover_primary_no_view_change.py b/plenum/test/primary_selection/test_recover_primary_no_view_change.py index 19ec82af8d..7d329e34ea 100644 --- a/plenum/test/primary_selection/test_recover_primary_no_view_change.py +++ b/plenum/test/primary_selection/test_recover_primary_no_view_change.py @@ -4,7 +4,7 @@ from plenum.test.conftest import getValueFromModule from plenum.test.helper import waitForViewChange, \ - sendReqsToNodesAndVerifySuffReplies + sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.test_node import ensureElectionsDone from plenum.test.view_change.helper import start_stopped_node @@ -24,7 +24,8 @@ def tconf(tconf): def test_recover_stop_primaries_no_view_change(looper, checkpoint_size, txnPoolNodeSet, - allPluginsPath, tdir, tconf, client1, wallet1): + allPluginsPath, tdir, tconf, sdk_pool_handle, + sdk_wallet_steward): """ Test that we can recover after having more than f nodes disconnected: - send txns @@ -39,7 +40,8 @@ def test_recover_stop_primaries_no_view_change(looper, checkpoint_size, txnPoolN logger.info("send at least one checkpoint") assert nodes_do_not_have_checkpoints(*active_nodes) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=2 * checkpoint_size) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 2 * checkpoint_size) assert nodes_have_checkpoints(*active_nodes) ensure_all_nodes_have_same_data(looper, nodes=active_nodes) @@ -59,6 +61,7 @@ def test_recover_stop_primaries_no_view_change(looper, checkpoint_size, txnPoolN ensure_all_nodes_have_same_data(looper, nodes=active_nodes) logger.info("Check if the pool is able to process requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, numReqs=10 * checkpoint_size) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 10 * checkpoint_size) ensure_all_nodes_have_same_data(looper, nodes=active_nodes) assert nodes_have_checkpoints(*active_nodes) diff --git a/plenum/test/view_change/helper.py b/plenum/test/view_change/helper.py index ae8c1609c2..a537e35674 100644 --- a/plenum/test/view_change/helper.py +++ b/plenum/test/view_change/helper.py @@ -71,13 +71,14 @@ def provoke_and_wait_for_view_change(looper, timeout=timeout)) -def simulate_slow_master(looper, txnPoolNodeSet, wallet, - client, delay=10, num_reqs=4): +def simulate_slow_master(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, delay=10, num_reqs=4): m_primary_node = get_master_primary_node(list(txnPoolNodeSet)) # Delay processing of PRE-PREPARE from all non primary replicas of master # so master's performance falls and view changes delayNonPrimaries(txnPoolNodeSet, 0, delay) - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, num_reqs) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, num_reqs) return m_primary_node @@ -229,14 +230,6 @@ def check_each_node_reaches_same_end_for_view(nodes, view_no): assert v == val -def do_vc(looper, nodes, client, wallet, old_view_no=None): - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) - new_view_no = ensure_view_change(looper, nodes) - if old_view_no: - assert new_view_no - old_view_no >= 1 - return new_view_no - - def disconnect_master_primary(nodes): pr_node = get_master_primary_node(nodes) for node in nodes: diff --git a/plenum/test/view_change/test_disable_view_change.py b/plenum/test/view_change/test_disable_view_change.py index 1fc79173c9..72dec53242 100644 --- a/plenum/test/view_change/test_disable_view_change.py +++ b/plenum/test/view_change/test_disable_view_change.py @@ -15,13 +15,15 @@ def test_disable_view_change( looper, txnPoolNodeSet, viewNo, - wallet1, - client1): + sdk_pool_handle, + sdk_wallet_steward): assert disable_view_change_config assert isinstance(disable_view_change_config.unsafe, set) assert 'disable_view_change' in disable_view_change_config.unsafe - simulate_slow_master(looper, txnPoolNodeSet, wallet1, client1) + simulate_slow_master(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_steward) with pytest.raises(AssertionError): waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) diff --git a/plenum/test/view_change/test_view_change.py b/plenum/test/view_change/test_view_change.py index d38a543ef4..d296c7efaf 100644 --- a/plenum/test/view_change/test_view_change.py +++ b/plenum/test/view_change/test_view_change.py @@ -1,4 +1,4 @@ -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.spy_helpers import get_count from plenum.test.test_node import ensureElectionsDone @@ -19,11 +19,13 @@ def test_view_change_on_empty_ledger(txnPoolNodeSet, looper): # noinspection PyIncorrectDocstring def test_view_change_after_some_txns(looper, txnPoolNodeSet, viewNo, - wallet1, client1): + sdk_pool_handle, + sdk_wallet_client): """ Check that view change is done after processing some of txns """ - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 3) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) @@ -32,17 +34,19 @@ def test_view_change_after_some_txns(looper, txnPoolNodeSet, viewNo, # noinspection PyIncorrectDocstring def test_send_more_after_view_change(looper, txnPoolNodeSet, - wallet1, client1): + sdk_pool_handle, sdk_wallet_client): """ Check that we can send more requests after view change """ - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 4) ensure_view_change(looper, txnPoolNodeSet) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 10) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 10) def test_node_notified_about_primary_election_result(txnPoolNodeSet, looper): diff --git a/plenum/test/view_change/test_view_change_min_cathup_timeout.py b/plenum/test/view_change/test_view_change_min_cathup_timeout.py index 3a6b928b1d..550917fe33 100644 --- a/plenum/test/view_change/test_view_change_min_cathup_timeout.py +++ b/plenum/test/view_change/test_view_change_min_cathup_timeout.py @@ -1,7 +1,9 @@ import types import pytest -from plenum.test.helper import waitForViewChange, sendReqsToNodesAndVerifySuffReplies +from plenum.test.node_request.helper import sdk_ensure_pool_functional + +from plenum.test.helper import waitForViewChange, sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.primary_selection.test_primary_selection_pool_txn import \ ensure_pool_functional @@ -21,7 +23,9 @@ def patched_has_ordered_till_last_prepared_certificate(self) -> bool: patched_has_ordered_till_last_prepared_certificate, node) -def test_view_change_min_catchup_timeout(txnPoolNodeSet, looper, wallet1, client1, +def test_view_change_min_catchup_timeout(txnPoolNodeSet, looper, + sdk_pool_handle, + sdk_wallet_client, tconf, viewNo): """ @@ -39,7 +43,8 @@ def test_view_change_min_catchup_timeout(txnPoolNodeSet, looper, wallet1, client """ # 1. Send some txns - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 4) # 2. make the only condition to finish catch-up is # MIN_TIMEOUT_CATCHUPS_DONE_DURING_VIEW_CHANGE @@ -65,4 +70,6 @@ def test_view_change_min_catchup_timeout(txnPoolNodeSet, looper, wallet1, client ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) # 6. ensure that the pool is still functional. - ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + sdk_wallet_client, + sdk_pool_handle) diff --git a/plenum/test/view_change/test_view_change_on_master_degraded.py b/plenum/test/view_change/test_view_change_on_master_degraded.py index a7ddbcee17..4213a5e175 100644 --- a/plenum/test/view_change/test_view_change_on_master_degraded.py +++ b/plenum/test/view_change/test_view_change_on_master_degraded.py @@ -5,7 +5,7 @@ from plenum.server.view_change.view_changer import ViewChanger from plenum.test.delayers import delayNonPrimaries from plenum.test.helper import waitForViewChange, \ - sendReqsToNodesAndVerifySuffReplies + sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.test_node import get_master_primary_node, getPrimaryReplica, \ ensureElectionsDone @@ -18,7 +18,8 @@ def test_view_change_on_performance_degraded(looper, txnPoolNodeSet, viewNo, - wallet1, client1): + sdk_pool_handle, + sdk_wallet_steward): """ Test that a view change is done when the performance of master goes down Send multiple requests from the client and delay some requests by master @@ -27,7 +28,8 @@ def test_view_change_on_performance_degraded(looper, txnPoolNodeSet, viewNo, """ old_primary_node = get_master_primary_node(list(txnPoolNodeSet)) - simulate_slow_master(looper, txnPoolNodeSet, wallet1, client1) + simulate_slow_master(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward) waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) ensureElectionsDone(looper=looper, nodes=txnPoolNodeSet) @@ -37,7 +39,9 @@ def test_view_change_on_performance_degraded(looper, txnPoolNodeSet, viewNo, def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper, - wallet1, client1, viewNo): + sdk_pool_handle, + sdk_wallet_steward, + viewNo): """ Node will change view even though it does not find the master to be degraded when a quorum of nodes agree that master performance degraded @@ -62,7 +66,8 @@ def test_view_change_on_quorum_of_master_degraded(txnPoolNodeSet, looper, relucatantNode.monitor.isMasterDegraded = types.MethodType( lambda x: False, relucatantNode.monitor) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 4) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 4) # Check that view change happened for all nodes waitForViewChange(looper, txnPoolNodeSet, expectedViewNo=viewNo + 1) From ff34a3104386454d5f0d9ea72de44c805c30c7ab Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 27 Mar 2018 19:45:04 +0300 Subject: [PATCH 44/73] Deletion of sendRandomRequests. Part 5. Deleted: ensure_pool_functional, NodeSetWithNodeAddedAfterSomeTxns, nodeCreatedAfterSomeTxns, newNodeCaughtUp. Signed-off-by: ArtObr --- .../test_checkpoint_bounds_after_catchup.py | 4 +- ..._checkpoint_stabilization_after_catchup.py | 4 +- plenum/test/client/test_client_observer.py | 83 ------------- .../test_client_sends_to_f_plus_one_nodes.py | 48 -------- plenum/test/helper.py | 44 ++----- .../test_instance_change_with_Delta.py | 2 +- plenum/test/node_catchup/conftest.py | 90 ++++---------- .../test_catchup_delayed_nodes.py | 6 +- .../test/node_catchup/test_catchup_demoted.py | 6 +- .../node_catchup/test_catchup_f_plus_one.py | 2 +- .../test_catchup_inlcuding_3PC.py | 27 ++--- .../test/node_catchup/test_config_ledger.py | 8 +- .../test/node_catchup/test_discard_view_no.py | 19 +-- .../test/node_catchup/test_large_catchup.py | 15 ++- .../node_catchup/test_new_node_catchup.py | 6 +- .../node_catchup/test_new_node_catchup2.py | 14 ++- .../test_no_catchup_if_got_from_3pc.py | 25 ++-- .../test_node_catchup_after_checkpoints.py | 31 ++--- .../test_node_catchup_after_disconnect.py | 31 ++--- ...test_node_catchup_after_lost_connection.py | 33 ++--- ...t_node_catchup_after_restart_after_txns.py | 29 +++-- ...test_node_catchup_after_restart_no_txns.py | 18 +-- .../test_node_catchup_causes_no_desync.py | 16 +-- ...tchup_when_3_not_primary_node_restarted.py | 12 +- ..._node_reject_invalid_txn_during_catchup.py | 8 +- .../test_remove_request_keys_post_catchup.py | 14 ++- .../test_revert_during_catchup.py | 20 ++-- .../test_timestamp/test_timestamp_new_node.py | 7 +- .../test/plugin/demo_plugin/test_catchup.py | 14 +-- .../test_plugin_request_handling.py | 1 + .../test_primary_election_case6.py | 2 +- plenum/test/primary_selection/conftest.py | 3 +- ...est_add_node_to_pool_with_large_ppseqno.py | 6 +- .../test_catchup_after_view_change.py | 2 +- .../test_catchup_multiple_rounds.py | 5 +- .../test_new_node_accepts_chosen_primary.py | 2 +- .../test_primary_selection.py | 2 +- ...after_primary_demotion_and_pool_restart.py | 8 +- .../test_primary_selection_pool_txn.py | 38 +++--- .../test_recover_more_than_f_failure.py | 6 +- .../test_recover_primary_no_view_change.py | 2 +- plenum/test/script/helper.py | 44 ++++--- .../script/test_change_non_primary_node_ha.py | 10 +- .../script/test_change_primary_node_ha.py | 9 +- plenum/test/test_current_state_propagation.py | 14 +-- plenum/test/test_dirty_read.py | 22 ---- plenum/test/test_memory_consumpion.py | 14 ++- plenum/test/test_performance.py | 113 +++++++----------- .../test_state_regenerated_from_ledger.py | 11 +- plenum/test/view_change/helper.py | 3 +- ...in_after_view_change_by_primary_restart.py | 7 +- ...detecting_lag_from_view_change_messages.py | 11 +- plenum/test/view_change/test_view_change.py | 2 +- .../test_view_change_min_cathup_timeout.py | 4 +- .../test_view_change_on_master_degraded.py | 4 +- .../view_change/test_view_change_timeout.py | 19 +-- 56 files changed, 380 insertions(+), 620 deletions(-) delete mode 100644 plenum/test/client/test_client_observer.py delete mode 100644 plenum/test/client/test_client_sends_to_f_plus_one_nodes.py delete mode 100644 plenum/test/test_dirty_read.py diff --git a/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py index c03af015e1..8b22e3c758 100644 --- a/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_bounds_after_catchup.py @@ -1,6 +1,6 @@ -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected CHK_FREQ = 5 diff --git a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py index 6aad40dbf9..0e35fc1eb2 100644 --- a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py @@ -1,7 +1,7 @@ from plenum.test import waits -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected CHK_FREQ = 5 diff --git a/plenum/test/client/test_client_observer.py b/plenum/test/client/test_client_observer.py deleted file mode 100644 index ffb561e1de..0000000000 --- a/plenum/test/client/test_client_observer.py +++ /dev/null @@ -1,83 +0,0 @@ -import pytest - -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies - - -# TODO: probably delete -def test_observer_registration(looper, txnPoolNodeSet, client1): - def callable1(*args, **kwargs): - print(1) - print(args) - print(kwargs) - - def callable2(*args, **kwargs): - print(2) - print(args) - print(kwargs) - - client1.registerObserver(callable1, name='first') - assert len(client1._observers) == 1 - assert client1.hasObserver(callable1) - - # Error when registering callable again - with pytest.raises(RuntimeError): - client1.registerObserver(callable1) - assert len(client1._observers) == 1 - - # Error when registering different callable with same name - with pytest.raises(RuntimeError): - client1.registerObserver(callable2, name='first') - assert len(client1._observers) == 1 - - # Register observer without name - client1.registerObserver(callable2) - assert len(client1._observers) == 2 - - with pytest.raises(RuntimeError): - client1.registerObserver(callable2) - assert len(client1._observers) == 2 - - client1.deregisterObserver('first') - assert len(client1._observers) == 1 - assert not client1.hasObserver(callable1) - - with pytest.raises(RuntimeError): - client1.deregisterObserver('first') - assert len(client1._observers) == 1 - with pytest.raises(RuntimeError): - client1.deregisterObserver('random_name') - assert len(client1._observers) == 1 - - -# TODO: probably delete -def test_observer_execution(looper, txnPoolNodeSet, client1, wallet1): - resp1 = [] - resp2 = [] - - def callable1(name, reqId, frm, result, numReplies): - resp1.append(reqId) - return reqId - - def callable2(name, reqId, frm, result, numReplies): - resp2.append(reqId) - return reqId - - client1.registerObserver(callable1, name='first') - client1.registerObserver(callable2) - - # Send 1 request - req, = sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) - # Each observer is called only once - assert len(resp1) == 1 - assert len(resp2) == 1 - assert resp1[0] == req.reqId - assert resp2[0] == req.reqId - - client1.deregisterObserver('first') - # Send another request - req1, = sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 1) - # Only 1 observer is called - assert len(resp1) == 1 - assert len(resp2) == 2 - assert resp1[-1] == req.reqId - assert resp2[-1] == req1.reqId diff --git a/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py b/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py deleted file mode 100644 index e5e559ac82..0000000000 --- a/plenum/test/client/test_client_sends_to_f_plus_one_nodes.py +++ /dev/null @@ -1,48 +0,0 @@ -from plenum.test.client.conftest import passThroughReqAcked1 - -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check -from plenum.test.malicious_behaviors_client import \ - genDoesntSendRequestToSomeNodes -from plenum.test.node_catchup.helper import waitNodeDataEquality - -nodeCount = 4 -clientFault = genDoesntSendRequestToSomeNodes("AlphaC") -reqAcked1 = passThroughReqAcked1 - - -# TODO: probably delete -def testReplyWhenRequestSentToMoreThanFPlusOneNodes(looper, txnPoolNodeSet, - fClient, replied1, - wallet1): - """ - Alpha would not be sent request but other nodes will be, so Alpha will - just rely on propagates from other nodes - """ - alpha = txnPoolNodeSet[0] - other_nodes = [n for n in txnPoolNodeSet if n != alpha] - - def chk(req_count=1): - for node in txnPoolNodeSet: - prc_req = node.processRequest.__name__ - prc_ppg = node.processPropagate.__name__ - if node != alpha: - # All nodes except alpha will receive requests from client - assert node.spylog.count(prc_req) == req_count - else: - # Alpha will not receive requests from client - assert node.spylog.count(prc_req) == 0 - - # All nodes will get propagates from others - assert node.spylog.count(prc_ppg) == req_count * (nodeCount - 1) - - # Ledger is same for all nodes - waitNodeDataEquality(looper, alpha, *other_nodes) - chk(1) - - more_reqs_count = 5 - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, fClient, - more_reqs_count, 1) - sdk_send_random_and_check - # Ledger is same for all nodes - waitNodeDataEquality(looper, alpha, *other_nodes) - chk(6) # Since one request is already sent as part of `replied1` diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 3b0da8e98f..c86652cc30 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -115,54 +115,30 @@ def waitForSufficientRepliesForRequests(looper, override_eventually_timeout=override_timeout_limit) -def sendReqsToNodesAndVerifySuffReplies(looper: Looper, - wallet: Wallet, - client: TestClient, - numReqs: int, - customTimeoutPerReq: float = None, - add_delay_to_timeout: float = 0, - override_timeout_limit=False, - total_timeout=None): - requests = sendRandomRequests(wallet, client, numReqs) - waitForSufficientRepliesForRequests( - looper, - client, - requests=requests, - customTimeoutPerReq=customTimeoutPerReq, - add_delay_to_timeout=add_delay_to_timeout, - override_timeout_limit=override_timeout_limit, - total_timeout=total_timeout) - return requests - - def send_reqs_batches_and_get_suff_replies( looper: Looper, - wallet: Wallet, - client: TestClient, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, num_reqs: int, num_batches=1, **kwargs): # This method assumes that `num_reqs` <= num_batches*MaxbatchSize if num_batches == 1: - return sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, - num_reqs, **kwargs) + return sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, num_reqs) else: requests = [] for _ in range(num_batches - 1): requests.extend( - sendReqsToNodesAndVerifySuffReplies( - looper, - wallet, - client, - num_reqs // num_batches, - **kwargs)) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, num_reqs // num_batches)) rem = num_reqs % num_batches if rem == 0: rem = num_reqs // num_batches - requests.extend(sendReqsToNodesAndVerifySuffReplies(looper, wallet, - client, - rem, - **kwargs)) + requests.extend( + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, rem)) return requests diff --git a/plenum/test/monitoring/test_instance_change_with_Delta.py b/plenum/test/monitoring/test_instance_change_with_Delta.py index bea1dc0afb..9ed992d12b 100644 --- a/plenum/test/monitoring/test_instance_change_with_Delta.py +++ b/plenum/test/monitoring/test_instance_change_with_Delta.py @@ -5,7 +5,7 @@ from stp_core.common.util import adict from plenum.server.node import Node from plenum.test import waits -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.malicious_behaviors_node import slow_primary from plenum.test.test_node import getPrimaryReplica from plenum.test.view_change.helper import provoke_and_wait_for_view_change diff --git a/plenum/test/node_catchup/conftest.py b/plenum/test/node_catchup/conftest.py index 37da7b9394..e25f534196 100644 --- a/plenum/test/node_catchup/conftest.py +++ b/plenum/test/node_catchup/conftest.py @@ -4,12 +4,11 @@ from stp_core.common.log import getlogger from plenum.common.util import randomString from plenum.test.conftest import getValueFromModule -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ check_last_3pc_master from plenum.test.pool_transactions.helper import \ - addNewStewardAndNode, buildPoolClientAndWallet, sdk_add_new_steward_and_node -from plenum.test.test_client import TestClient + sdk_add_new_steward_and_node, sdk_pool_refresh from plenum.test.test_node import checkNodesConnected @@ -20,36 +19,6 @@ def whitelist(): logger = getlogger() -@pytest.yield_fixture(scope="module") -def looper(txnPoolNodesLooper): - yield txnPoolNodesLooper - - -@pytest.yield_fixture("module") -def nodeCreatedAfterSomeTxns(looper, testNodeClass, do_post_node_creation, - txnPoolNodeSet, tdir, tdirWithClientPoolTxns, - poolTxnStewardData, tconf, allPluginsPath, request): - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, - tdirWithClientPoolTxns, - clientClass=TestClient) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) - txnCount = getValueFromModule(request, "txnCount", 5) - sendReqsToNodesAndVerifySuffReplies(looper, - wallet, - client, - txnCount) - newStewardName = randomString() - newNodeName = "Epsilon" - newStewardClient, newStewardWallet, newNode = addNewStewardAndNode( - looper, client, wallet, newStewardName, newNodeName, - tdir, tdirWithClientPoolTxns, tconf, nodeClass=testNodeClass, - allPluginsPath=allPluginsPath, autoStart=True, - do_post_node_creation=do_post_node_creation) - yield looper, newNode, client, wallet, newStewardClient, \ - newStewardWallet - - @pytest.yield_fixture("module") def sdk_node_created_after_some_txns(looper, testNodeClass, do_post_node_creation, sdk_pool_handle, sdk_wallet_client, sdk_wallet_steward, @@ -66,21 +35,10 @@ def sdk_node_created_after_some_txns(looper, testNodeClass, do_post_node_creatio new_steward_name, new_node_name, tdir, tconf, nodeClass=testNodeClass, allPluginsPath=allPluginsPath, autoStart=True, do_post_node_creation=do_post_node_creation) + sdk_pool_refresh(looper, sdk_pool_handle) yield looper, new_node, sdk_pool_handle, new_steward_wallet_handle -@pytest.fixture("module") -def nodeSetWithNodeAddedAfterSomeTxns( - txnPoolNodeSet, nodeCreatedAfterSomeTxns): - looper, newNode, client, wallet, newStewardClient, newStewardWallet = \ - nodeCreatedAfterSomeTxns - txnPoolNodeSet.append(newNode) - looper.run(checkNodesConnected(txnPoolNodeSet)) - looper.run(newStewardClient.ensureConnectedToNodes()) - looper.run(client.ensureConnectedToNodes()) - return looper, newNode, client, wallet, newStewardClient, newStewardWallet - - @pytest.fixture("module") def sdk_node_set_with_node_added_after_some_txns( txnPoolNodeSet, sdk_node_created_after_some_txns): @@ -88,18 +46,20 @@ def sdk_node_set_with_node_added_after_some_txns( sdk_node_created_after_some_txns txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) + sdk_pool_refresh(looper, sdk_pool_handle) return looper, new_node, sdk_pool_handle, new_steward_wallet_handle @pytest.fixture("module") -def newNodeCaughtUp(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): - looper, newNode, _, _, _, _ = nodeSetWithNodeAddedAfterSomeTxns - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:4]) - check_last_3pc_master(newNode, txnPoolNodeSet[:4]) +def sdk_new_node_caught_up(txnPoolNodeSet, + sdk_node_set_with_node_added_after_some_txns): + looper, new_node, _, _ = sdk_node_set_with_node_added_after_some_txns + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4]) + check_last_3pc_master(new_node, txnPoolNodeSet[:4]) # Check if catchup done once catchup_done_once = True - for li in newNode.ledgerManager.ledgerRegistry.values(): + for li in new_node.ledgerManager.ledgerRegistry.values(): catchup_done_once = catchup_done_once and (li.num_txns_caught_up > 0) if not catchup_done_once: @@ -109,34 +69,26 @@ def newNodeCaughtUp(txnPoolNodeSet, nodeSetWithNodeAddedAfterSomeTxns): assert max( getAllReturnVals( - newNode, - newNode.num_txns_caught_up_in_last_catchup)) > 0 + new_node, + new_node.num_txns_caught_up_in_last_catchup)) > 0 - for li in newNode.ledgerManager.ledgerRegistry.values(): + for li in new_node.ledgerManager.ledgerRegistry.values(): assert not li.receivedCatchUpReplies assert not li.recvdCatchupRepliesFrm - return newNode + return new_node @pytest.yield_fixture("module") def poolAfterSomeTxns( looper, - txnPoolNodesLooper, txnPoolNodeSet, - tdirWithClientPoolTxns, - poolTxnStewardData, - allPluginsPath, + sdk_pool_handle, + sdk_wallet_client, request): - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, - tdirWithClientPoolTxns, - clientClass=TestClient) - looper.run(checkNodesConnected(txnPoolNodeSet)) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) txnCount = getValueFromModule(request, "txnCount", 5) - sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, - wallet, - client, - txnCount) - yield looper, client, wallet + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + txnCount) + yield looper, sdk_pool_handle, sdk_wallet_client diff --git a/plenum/test/node_catchup/test_catchup_delayed_nodes.py b/plenum/test/node_catchup/test_catchup_delayed_nodes.py index b5ae678207..28f4df7141 100644 --- a/plenum/test/node_catchup/test_catchup_delayed_nodes.py +++ b/plenum/test/node_catchup/test_catchup_delayed_nodes.py @@ -4,9 +4,10 @@ from plenum.test import waits from plenum.test.delayers import cpDelay -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node, \ + sdk_pool_refresh from plenum.test.test_node import checkNodesConnected logger = getlogger() @@ -75,6 +76,7 @@ def testCatchupDelayedNodes(txnPoolNodeSet, nodeX.stop() nodeY.stop() logger.debug("Sending requests") + sdk_pool_refresh(looper, sdk_pool_handle) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 50) logger.debug("Starting the 2 stopped nodes, {} and {}".format(nodeX.name, diff --git a/plenum/test/node_catchup/test_catchup_demoted.py b/plenum/test/node_catchup/test_catchup_demoted.py index 0c50f68f41..ccb2056039 100644 --- a/plenum/test/node_catchup/test_catchup_demoted.py +++ b/plenum/test/node_catchup/test_catchup_demoted.py @@ -1,11 +1,11 @@ from plenum.common.util import hexToFriendly -from plenum.common.constants import ALIAS, SERVICES, VALIDATOR -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.common.constants import VALIDATOR +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ checkNodeDataForInequality from plenum.test.pool_transactions.helper import \ - updateNodeData, sdk_send_update_node + sdk_send_update_node from stp_core.common.log import getlogger from plenum.test.node_catchup.conftest import whitelist diff --git a/plenum/test/node_catchup/test_catchup_f_plus_one.py b/plenum/test/node_catchup/test_catchup_f_plus_one.py index 6b0efcadd1..4f5c2b49bb 100644 --- a/plenum/test/node_catchup/test_catchup_f_plus_one.py +++ b/plenum/test/node_catchup/test_catchup_f_plus_one.py @@ -1,6 +1,6 @@ from stp_core.common.log import getlogger from plenum.common.config_helper import PNodeConfigHelper -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ waitNodeDataInequality, checkNodeDataForEquality from plenum.test.pool_transactions.helper import \ diff --git a/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py b/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py index 5626825d04..0e74f1fa1f 100644 --- a/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py +++ b/plenum/test/node_catchup/test_catchup_inlcuding_3PC.py @@ -2,11 +2,9 @@ from plenum.common.constants import DOMAIN_LEDGER_ID from plenum.common.util import check_if_all_equal_in_list -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import check_last_3pc_master, \ waitNodeDataEquality -from plenum.test.pool_transactions.helper import buildPoolClientAndWallet -from plenum.test.test_client import TestClient from stp_core.loop.eventually import eventually TestRunningTimeLimitSec = 125 @@ -34,21 +32,18 @@ def reset(): @pytest.fixture("module") -def pre_check(tconf, looper, txnPoolNodeSet, tdirWithClientPoolTxns, - poolTxnStewardData): +def pre_check(tconf, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): # TODO: Maybe this needs to be extracted in another fixture - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, - tdirWithClientPoolTxns, - clientClass=TestClient) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) + for i in range(tconf.ProcessedBatchMapsToKeep - 1): - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 1) # All node maintain the same map from txn range to 3PC looper.run(eventually(chk_if_equal_txn_to_3pc, txnPoolNodeSet)) for i in range(3): - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 1) # All node maintain the same map from txn range to 3PC and its equal to # `tconf.ProcessedBatchMapsToKeep` even after sending more batches than @@ -59,13 +54,15 @@ def pre_check(tconf, looper, txnPoolNodeSet, tdirWithClientPoolTxns, def test_nodes_maintain_master_txn_3PC_map(looper, txnPoolNodeSet, pre_check, - nodeCreatedAfterSomeTxns): - _, new_node, client, wallet, _, _ = nodeCreatedAfterSomeTxns + sdk_node_created_after_some_txns): + _, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_created_after_some_txns txnPoolNodeSet.append(new_node) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4]) # Check the new node has set same `last_3pc_ordered` for master as others check_last_3pc_master(new_node, txnPoolNodeSet[:4]) chk_if_equal_txn_to_3pc(txnPoolNodeSet[:4]) # Requests still processed - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 2) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 2) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4]) diff --git a/plenum/test/node_catchup/test_config_ledger.py b/plenum/test/node_catchup/test_config_ledger.py index aab21bb1fc..f65c2fd333 100644 --- a/plenum/test/node_catchup/test_config_ledger.py +++ b/plenum/test/node_catchup/test_config_ledger.py @@ -114,11 +114,11 @@ def some_config_txns_done(looper, setup, txnPoolNodeSet, keys, def test_new_node_catchup_config_ledger(looper, some_config_txns_done, - txnPoolNodeSet, newNodeCaughtUp): + txnPoolNodeSet, sdk_new_node_caught_up): """ A new node catches up the config ledger too """ - assert len(newNodeCaughtUp.getLedger(CONFIG_LEDGER_ID)) >= \ + assert len(sdk_new_node_caught_up.getLedger(CONFIG_LEDGER_ID)) >= \ len(some_config_txns_done) @@ -127,12 +127,12 @@ def test_disconnected_node_catchup_config_ledger_txns(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, - newNodeCaughtUp, keys): + sdk_new_node_caught_up, keys): """ A node gets disconnected, a few config ledger txns happen, the disconnected node comes back up and catches up the config ledger """ - new_node = newNodeCaughtUp + new_node = sdk_new_node_caught_up disconnect_node_and_ensure_disconnected( looper, txnPoolNodeSet, new_node, stopNode=False) diff --git a/plenum/test/node_catchup/test_discard_view_no.py b/plenum/test/node_catchup/test_discard_view_no.py index 658ac45c99..d7625f96e3 100644 --- a/plenum/test/node_catchup/test_discard_view_no.py +++ b/plenum/test/node_catchup/test_discard_view_no.py @@ -15,16 +15,17 @@ def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet, - nodeSetWithNodeAddedAfterSomeTxns, - newNodeCaughtUp, - allPluginsPath): + sdk_node_set_with_node_added_after_some_txns, + sdk_new_node_caught_up, + allPluginsPath, wallet1): """ Node discards 3-phase or ViewChangeDone messages from view nos that it does not know of (view nos before it joined the pool) :return: """ - looper, nodeX, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns - viewNo = nodeX.viewNo + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_set_with_node_added_after_some_txns + viewNo = new_node.viewNo # Force two view changes: node discards msgs which have viewNo # at least two less than node's. Current protocol implementation @@ -32,11 +33,11 @@ def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet, # from the current view. for i in range(2): ensure_view_change(looper, txnPoolNodeSet) - waitNodeDataEquality(looper, nodeX, *txnPoolNodeSet[:-1]) + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) sender = txnPoolNodeSet[0] - rid_x_node = sender.nodestack.getRemote(nodeX.name).uid + rid_x_node = sender.nodestack.getRemote(new_node.name).uid messageTimeout = waits.expectedNodeToNodeMessageDeliveryTime() # 3 pc msg (PrePrepare) needs to be discarded @@ -46,7 +47,7 @@ def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet, viewNo, 10, get_utc_epoch(), - [[wallet.defaultId, Request.gen_req_id()]], + [[wallet1.defaultId, Request.gen_req_id()]], 1, "random digest", DOMAIN_LEDGER_ID, @@ -54,7 +55,7 @@ def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet, primaryRepl.txnRootHash(DOMAIN_LEDGER_ID), ) sender.send(three_pc, rid_x_node) - looper.run(eventually(checkDiscardMsg, [nodeX, ], three_pc, + looper.run(eventually(checkDiscardMsg, [new_node, ], three_pc, 'un-acceptable viewNo', retryWait=1, timeout=messageTimeout)) diff --git a/plenum/test/node_catchup/test_large_catchup.py b/plenum/test/node_catchup/test_large_catchup.py index fa022ccf87..98ad93fa14 100644 --- a/plenum/test/node_catchup/test_large_catchup.py +++ b/plenum/test/node_catchup/test_large_catchup.py @@ -4,7 +4,7 @@ from plenum.common.config_helper import PNodeConfigHelper from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality from stp_core.validators.message_length_validator import MessageLenValidator @@ -44,8 +44,8 @@ def test_large_catchup(tdir, tconf, looper, testNodeClass, txnPoolNodeSet, - wallet1, - client1, + sdk_pool_handle, + sdk_wallet_client, allPluginsPath): """ Checks that node can catchup large ledgers @@ -55,11 +55,9 @@ def test_large_catchup(tdir, tconf, rest_nodes = txnPoolNodeSet[:-1] all_nodes = txnPoolNodeSet - # Prepare client - client, wallet = client1, wallet1 - # Check that requests executed well - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=10) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 10) # Stop one node waitNodeDataEquality(looper, lagging_node, *rest_nodes) @@ -70,7 +68,8 @@ def test_large_catchup(tdir, tconf, looper.removeProdable(lagging_node) # Send more requests to active nodes - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, numReqs=100) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 100) waitNodeDataEquality(looper, *rest_nodes) # Make message size limit smaller to ensure that catchup response is diff --git a/plenum/test/node_catchup/test_new_node_catchup.py b/plenum/test/node_catchup/test_new_node_catchup.py index d10480ea50..37545de0a7 100644 --- a/plenum/test/node_catchup/test_new_node_catchup.py +++ b/plenum/test/node_catchup/test_new_node_catchup.py @@ -10,7 +10,7 @@ txnCount = 5 -def testNewNodeCatchup(newNodeCaughtUp): +def testNewNodeCatchup(sdk_new_node_caught_up): """ A new node that joins after some transactions are done should eventually get those transactions. @@ -22,7 +22,7 @@ def testNewNodeCatchup(newNodeCaughtUp): def testPoolLegerCatchupBeforeDomainLedgerCatchup(txnPoolNodeSet, - newNodeCaughtUp): + sdk_new_node_caught_up): """ For new node, this should be the sequence of events: 1. Pool ledger starts catching up. @@ -31,7 +31,7 @@ def testPoolLegerCatchupBeforeDomainLedgerCatchup(txnPoolNodeSet, 4. Domain ledger completes catching up Every node's pool ledger starts catching up before it """ - newNode = newNodeCaughtUp + newNode = sdk_new_node_caught_up starts = newNode.ledgerManager.spylog.getAll( TestLedgerManager.startCatchUpProcess.__name__) completes = newNode.ledgerManager.spylog.getAll( diff --git a/plenum/test/node_catchup/test_new_node_catchup2.py b/plenum/test/node_catchup/test_new_node_catchup2.py index 47770cb6b6..6e2d9c2629 100644 --- a/plenum/test/node_catchup/test_new_node_catchup2.py +++ b/plenum/test/node_catchup/test_new_node_catchup2.py @@ -2,7 +2,7 @@ from plenum.test import waits from plenum.test.delayers import cqDelay -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.node_request.node_request_helper import chk_commits_prepares_recvd from plenum.test.test_node import ensureElectionsDone @@ -28,18 +28,19 @@ def nodes_slow_to_process_catchup_reqs(txnPoolNodeSet): def testNodeDoesNotParticipateUntilCaughtUp(txnPoolNodeSet, nodes_slow_to_process_catchup_reqs, - nodeCreatedAfterSomeTxns): + sdk_node_created_after_some_txns): """ A new node that joins after some transactions should stash new transactions until it has caught up :return: """ - looper, new_node, client, wallet, newStewardClient, newStewardWallet = \ - nodeCreatedAfterSomeTxns + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_created_after_some_txns txnPoolNodeSet.append(new_node) old_nodes = txnPoolNodeSet[:-1] - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 5) chk_commits_prepares_recvd(0, old_nodes, new_node) for node in old_nodes: @@ -51,7 +52,8 @@ def testNodeDoesNotParticipateUntilCaughtUp(txnPoolNodeSet, ensureElectionsDone(looper, txnPoolNodeSet, customTimeout=timeout) waitNodeDataEquality(looper, new_node, *old_nodes) - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 2) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 2) # Commits and Prepares are received by all old nodes with pytest.raises(AssertionError): diff --git a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py index 9e14786c74..7cd38b51b3 100644 --- a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py +++ b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py @@ -1,3 +1,5 @@ +from plenum.test.node_request.helper import sdk_ensure_pool_functional + from plenum.common.constants import DOMAIN_LEDGER_ID from plenum.common.messages.node_messages import Commit, ConsistencyProof from plenum.test.delayers import cpDelay, cDelay @@ -5,21 +7,25 @@ from plenum.test.helper import send_reqs_batches_and_get_suff_replies from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data, \ waitNodeDataInequality, waitNodeDataEquality -from plenum.test.primary_selection.test_primary_selection_pool_txn import \ - ensure_pool_functional from plenum.test.spy_helpers import getAllReturnVals from plenum.test.test_node import getNonPrimaryReplicas from plenum.test.view_change.helper import ensure_view_change -def test_no_catchup_if_got_from_3pc(looper, txnPoolNodeSet, wallet1, client1): +def test_no_catchup_if_got_from_3pc(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client): """ A node is slow to receive COMMIT messages so after a view change it starts catchup. But before it can start requesting txns, the COMMITs messages are received and are ordered. The node should not request any transactions. :return: """ - send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 2 * 3, 3) + send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 2 * 3, + 3) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node other_nodes = [n for n in txnPoolNodeSet if n != slow_node] @@ -40,8 +46,11 @@ def domain_cr_count(): return sum(1 for entry in old_count = domain_cr_count() sent_batches = 10 - send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, - 2 * sent_batches, sent_batches) + send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 2 * sent_batches, + sent_batches) ensure_view_change(looper, nodes=txnPoolNodeSet) # After view change, the `slow_node` is behind @@ -64,4 +73,6 @@ def domain_cr_count(): return sum(1 for entry in rv = getAllReturnVals(slow_node, slow_node.processStashedOrderedReqs) assert sent_batches in rv - ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + sdk_wallet_client, + sdk_pool_handle) diff --git a/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py b/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py index 900ca02475..76bfe575ec 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py +++ b/plenum/test/node_catchup/test_node_catchup_after_checkpoints.py @@ -19,8 +19,8 @@ def test_node_catchup_after_checkpoints( looper, chk_freq_patched, txnPoolNodeSet, - wallet1, - client1, + sdk_pool_handle, + sdk_wallet_client, broken_node_and_others): """ For some reason a node misses 3pc messages but eventually the node stashes @@ -30,10 +30,11 @@ def test_node_catchup_after_checkpoints( completed_catchups_before = get_number_of_completed_catchups(broken_node) logger.info("Step 1: The node misses quite a lot requests") - send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, - num_reqs=chk_freq_patched + 1, - num_batches=chk_freq_patched + 1, - ) + send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + chk_freq_patched + 1, + chk_freq_patched + 1) waitNodeDataInequality(looper, broken_node, *other_nodes) logger.info( @@ -41,10 +42,11 @@ def test_node_catchup_after_checkpoints( "missed ones. But the nodes eventually stashes some amount checkpoints " "after that the node starts catch up") repaired_node = repair_broken_node(broken_node) - send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, - num_reqs=2 * chk_freq_patched, - num_batches=2 * chk_freq_patched - ) + send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 2 * chk_freq_patched, + 2 * chk_freq_patched) waitNodeDataEquality(looper, repaired_node, *other_nodes) # check if there was at least 1 catchup @@ -52,10 +54,11 @@ def test_node_catchup_after_checkpoints( assert completed_catchups_after >= completed_catchups_before + 1 logger.info("Step 3: Check if the node is able to process requests") - send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, - num_reqs=chk_freq_patched + 2, - num_batches=chk_freq_patched + 2 - ) + send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + chk_freq_patched + 2, + chk_freq_patched + 2) waitNodeDataEquality(looper, repaired_node, *other_nodes) diff --git a/plenum/test/node_catchup/test_node_catchup_after_disconnect.py b/plenum/test/node_catchup/test_node_catchup_after_disconnect.py index 00907cc612..8a8272cd11 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_disconnect.py +++ b/plenum/test/node_catchup/test_node_catchup_after_disconnect.py @@ -1,5 +1,5 @@ from stp_core.common.log import getlogger -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ waitNodeDataInequality, checkNodeDataForEquality from plenum.test.pool_transactions.helper import \ @@ -14,32 +14,35 @@ # TODO: Refactor tests to minimize module-scoped fixtures.They make tests # depend on each other -def testNodeCatchupAfterDisconnect(newNodeCaughtUp, txnPoolNodeSet, - nodeSetWithNodeAddedAfterSomeTxns): +def testNodeCatchupAfterDisconnect(sdk_new_node_caught_up, txnPoolNodeSet, + sdk_node_set_with_node_added_after_some_txns): """ A node that disconnects after some transactions should eventually get the transactions which happened while it was disconnected :return: """ - looper, newNode, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_set_with_node_added_after_some_txns logger.debug("Stopping node {} with pool ledger size {}". - format(newNode, newNode.poolManager.txnSeqNo)) + format(new_node, new_node.poolManager.txnSeqNo)) disconnect_node_and_ensure_disconnected( - looper, txnPoolNodeSet, newNode, stopNode=False) + looper, txnPoolNodeSet, new_node, stopNode=False) # TODO: Check if the node has really stopped processing requests? logger.debug("Sending requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 5) # Make sure new node got out of sync - waitNodeDataInequality(looper, newNode, *txnPoolNodeSet[:-1]) + waitNodeDataInequality(looper, new_node, *txnPoolNodeSet[:-1]) - logger.debug("Starting the stopped node, {}".format(newNode)) - reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, newNode) + logger.debug("Starting the stopped node, {}".format(new_node)) + reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, new_node) - logger.debug("Waiting for the node to catch up, {}".format(newNode)) - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1]) + logger.debug("Waiting for the node to catch up, {}".format(new_node)) + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) logger.debug("Sending more requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 10) - checkNodeDataForEquality(newNode, *txnPoolNodeSet[:-1]) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 10) + checkNodeDataForEquality(new_node, *txnPoolNodeSet[:-1]) diff --git a/plenum/test/node_catchup/test_node_catchup_after_lost_connection.py b/plenum/test/node_catchup/test_node_catchup_after_lost_connection.py index 1bf4921a95..b2331a10d3 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_lost_connection.py +++ b/plenum/test/node_catchup/test_node_catchup_after_lost_connection.py @@ -3,7 +3,7 @@ reconnect_node_and_ensure_connected from plenum.test.test_node import ensure_node_disconnected from stp_core.common.log import getlogger -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ waitNodeDataInequality, checkNodeDataForEquality @@ -16,36 +16,39 @@ # TODO: Refactor tests to minimize module-scoped fixtures.They make tests # depend on each other -def testNodeCatchupAfterLostConnection(newNodeCaughtUp, txnPoolNodeSet, - nodeSetWithNodeAddedAfterSomeTxns): +def testNodeCatchupAfterLostConnection(sdk_new_node_caught_up, txnPoolNodeSet, + sdk_node_set_with_node_added_after_some_txns): """ A node that has poor internet connection and got unsynced after some transactions should eventually get the transactions which happened while it was not accessible :return: """ - looper, newNode, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_set_with_node_added_after_some_txns logger.debug("Disconnecting node {}, ledger size {}". - format(newNode, newNode.domainLedger.size)) - disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, newNode, + format(new_node, new_node.domainLedger.size)) + disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node, stopNode=False) # TODO: Check if the node has really stopped processing requests? logger.debug("Sending requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 5) # Make sure new node got out of sync - waitNodeDataInequality(looper, newNode, *txnPoolNodeSet[:-1]) + waitNodeDataInequality(looper, new_node, *txnPoolNodeSet[:-1]) # logger.debug("Ensure node {} gets disconnected".format(newNode)) - ensure_node_disconnected(looper, newNode, txnPoolNodeSet[:-1]) + ensure_node_disconnected(looper, new_node, txnPoolNodeSet[:-1]) logger.debug("Connecting the node {} back, ledger size {}". - format(newNode, newNode.domainLedger.size)) - reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, newNode) + format(new_node, new_node.domainLedger.size)) + reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, new_node) - logger.debug("Waiting for the node to catch up, {}".format(newNode)) - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1]) + logger.debug("Waiting for the node to catch up, {}".format(new_node)) + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) logger.debug("Sending more requests") - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 10) - checkNodeDataForEquality(newNode, *txnPoolNodeSet[:-1]) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, 10) + checkNodeDataForEquality(new_node, *txnPoolNodeSet[:-1]) diff --git a/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py b/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py index 0f56311ea5..9df86c705e 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py +++ b/plenum/test/node_catchup/test_node_catchup_after_restart_after_txns.py @@ -4,18 +4,15 @@ from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState from plenum.test.delayers import cr_delay -from plenum.test.spy_helpers import get_count from stp_core.loop.eventually import eventually from plenum.common.types import HA from stp_core.common.log import getlogger -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \ - check_last_ordered_3pc +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ check_ledger_state from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected -from plenum.test.test_ledger_manager import TestLedgerManager from plenum.test.test_node import checkNodesConnected, TestNode from plenum.test import waits from plenum.common.config_helper import PNodeConfigHelper @@ -34,22 +31,23 @@ # and after prepares, respectively. Here is the pivotal link # https://www.pivotaltracker.com/story/show/127897273 def test_node_catchup_after_restart_with_txns( - newNodeCaughtUp, + sdk_new_node_caught_up, txnPoolNodeSet, tdir, tconf, - nodeSetWithNodeAddedAfterSomeTxns, + sdk_node_set_with_node_added_after_some_txns, allPluginsPath): """ A node that restarts after some transactions should eventually get the transactions which happened while it was down :return: """ - looper, newNode, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = \ + sdk_node_set_with_node_added_after_some_txns logger.debug("Stopping node {} with pool ledger size {}". - format(newNode, newNode.poolManager.txnSeqNo)) - disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, newNode) - looper.removeProdable(newNode) + format(new_node, new_node.poolManager.txnSeqNo)) + disconnect_node_and_ensure_disconnected(looper, txnPoolNodeSet, new_node) + looper.removeProdable(new_node) # for n in txnPoolNodeSet[:4]: # for r in n.nodestack.remotes.values(): # if r.name == newNode.name: @@ -59,12 +57,13 @@ def test_node_catchup_after_restart_with_txns( # TODO: Check if the node has really stopped processing requests? logger.debug("Sending requests") more_requests = 5 - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, more_requests) - logger.debug("Starting the stopped node, {}".format(newNode)) - nodeHa, nodeCHa = HA(*newNode.nodestack.ha), HA(*newNode.clientstack.ha) - config_helper = PNodeConfigHelper(newNode.name, tconf, chroot=tdir) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + new_steward_wallet_handle, more_requests) + logger.debug("Starting the stopped node, {}".format(new_node)) + nodeHa, nodeCHa = HA(*new_node.nodestack.ha), HA(*new_node.clientstack.ha) + config_helper = PNodeConfigHelper(new_node.name, tconf, chroot=tdir) newNode = TestNode( - newNode.name, + new_node.name, config_helper=config_helper, config=tconf, ha=nodeHa, diff --git a/plenum/test/node_catchup/test_node_catchup_after_restart_no_txns.py b/plenum/test/node_catchup/test_node_catchup_after_restart_no_txns.py index 805024236c..8fc02b1bf8 100644 --- a/plenum/test/node_catchup/test_node_catchup_after_restart_no_txns.py +++ b/plenum/test/node_catchup/test_node_catchup_after_restart_no_txns.py @@ -1,21 +1,13 @@ -import pytest - -from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState -from plenum.test.delayers import cr_delay from plenum.test.spy_helpers import get_count from stp_core.loop.eventually import eventually from plenum.common.types import HA from stp_core.common.log import getlogger -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \ - check_last_ordered_3pc -from plenum.test.node_catchup.helper import waitNodeDataEquality, \ - check_ledger_state +from plenum.test.helper import check_last_ordered_3pc +from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected -from plenum.test.test_ledger_manager import TestLedgerManager from plenum.test.test_node import checkNodesConnected, TestNode -from plenum.test import waits from plenum.common.config_helper import PNodeConfigHelper # Do not remove the next import @@ -26,18 +18,18 @@ def test_node_catchup_after_restart_no_txns( - newNodeCaughtUp, + sdk_new_node_caught_up, txnPoolNodeSet, tdir, tconf, - nodeSetWithNodeAddedAfterSomeTxns, + sdk_node_set_with_node_added_after_some_txns, tdirWithPoolTxns, allPluginsPath): """ A node restarts but no transactions have happened while it was down. It would then use the `LedgerStatus` to catchup """ - looper, new_node, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns + looper, new_node, sdk_pool_handle, new_steward_wallet_handle = sdk_node_set_with_node_added_after_some_txns waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) logger.debug("Stopping node {} with pool ledger size {}". diff --git a/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py b/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py index 8d60e0052e..15e8433e87 100644 --- a/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py +++ b/plenum/test/node_catchup/test_node_catchup_causes_no_desync.py @@ -3,7 +3,7 @@ from plenum.test.node_catchup.test_node_reject_invalid_txn_during_catchup import \ get_any_non_primary_node from stp_core.common.log import getlogger -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import \ waitNodeDataEquality, \ waitNodeDataInequality @@ -41,14 +41,13 @@ def replicas_synced(node): assert compare_last_ordered_3pc(node) == 0 -def test_node_catchup_causes_no_desync(looper, txnPoolNodeSet, client1, - wallet1, monkeypatch): +def test_node_catchup_causes_no_desync(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, monkeypatch): """ Checks that transactions received by catchup do not break performance monitoring """ - client, wallet = client1, wallet1 lagging_node = get_any_non_primary_node(txnPoolNodeSet) rest_nodes = set(txnPoolNodeSet).difference({lagging_node}) @@ -59,7 +58,8 @@ def test_node_catchup_causes_no_desync(looper, txnPoolNodeSet, client1, lambda *x, **y: None) # Send some requests and check that all replicas except master executed it - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 5) waitNodeDataInequality(looper, lagging_node, *rest_nodes) looper.run(eventually(backup_replicas_run_forward, lagging_node)) @@ -69,7 +69,8 @@ def test_node_catchup_causes_no_desync(looper, txnPoolNodeSet, client1, txnPoolNodeSet, lagging_node, stopNode=False) - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 5) reconnect_node_and_ensure_connected(looper, txnPoolNodeSet, lagging_node) # Check that catchup done @@ -77,7 +78,8 @@ def test_node_catchup_causes_no_desync(looper, txnPoolNodeSet, client1, # Send some more requests to ensure that backup and master replicas # are in the same state - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 5) looper.run(eventually(replicas_synced, lagging_node)) # Check that master is not considered to be degraded diff --git a/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py b/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py index 679154091c..26d62347b9 100644 --- a/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py +++ b/plenum/test/node_catchup/test_node_catchup_when_3_not_primary_node_restarted.py @@ -1,9 +1,7 @@ -import pytest - from plenum.test.view_change.helper import ensure_all_nodes_have_same_data, \ start_stopped_node from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState, POOL_LEDGER_ID -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually @@ -22,7 +20,7 @@ def catchuped(node): def test_node_catchup_when_3_not_primary_node_restarted( looper, txnPoolNodeSet, tdir, tconf, - allPluginsPath, steward1, stewardWallet): + allPluginsPath, sdk_wallet_steward, sdk_pool_handle): """ Test case: 1. Create pool of 4 nodes @@ -57,7 +55,8 @@ def start_stop_one_node(node_to_restart, pool_of_nodes): ensure_all_nodes_have_same_data(looper, remaining_nodes, custom_timeout=tconf.VIEW_CHANGE_TIMEOUT) - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 1) node_to_restart = start_stopped_node(node_to_restart, looper, tconf, @@ -83,7 +82,8 @@ def start_stop_one_node(node_to_restart, pool_of_nodes): node_to_restart = [n for n in pool_of_nodes if n.name == nodes_names[__]][0] assert not node_to_restart.has_master_primary pool_of_nodes = start_stop_one_node(node_to_restart, pool_of_nodes) - sendReqsToNodesAndVerifySuffReplies(looper, stewardWallet, steward1, 1) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 1) ensure_all_nodes_have_same_data(looper, pool_of_nodes, custom_timeout=tconf.VIEW_CHANGE_TIMEOUT) diff --git a/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py b/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py index 9488298023..0329d64c5f 100644 --- a/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py +++ b/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py @@ -19,7 +19,7 @@ def testNodeRejectingInvalidTxns(tconf, txnPoolNodeSet, patched_node, - nodeCreatedAfterSomeTxns): + sdk_node_created_after_some_txns): """ A newly joined node is catching up and sends catchup requests to other nodes but one of the nodes replies with incorrect transactions. The newly @@ -27,7 +27,7 @@ def testNodeRejectingInvalidTxns(tconf, txnPoolNodeSet, patched_node, the node. Ii thus cannot complete the process till the timeout and then requests the missing transactions. """ - looper, newNode, client, wallet, _, _ = nodeCreatedAfterSomeTxns + looper, new_node, _, _ = sdk_node_created_after_some_txns bad_node = patched_node do_not_tell_clients_about_newly_joined_node(txnPoolNodeSet) @@ -42,10 +42,10 @@ def testNodeRejectingInvalidTxns(tconf, txnPoolNodeSet, patched_node, # have to skip seqno_db check because the txns are not executed # on the new node - waitNodeDataEquality(looper, newNode, *txnPoolNodeSet[:-1], + waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1], customTimeout=timeout) - assert newNode.isNodeBlacklisted(bad_node.name) + assert new_node.isNodeBlacklisted(bad_node.name) @pytest.fixture diff --git a/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py b/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py index dc543d9195..6a1ce12f4e 100644 --- a/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py +++ b/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py @@ -5,7 +5,7 @@ from plenum.test.delayers import delay_3pc_messages, pDelay, cDelay, ppDelay, \ cr_delay from plenum.test.helper import send_reqs_batches_and_get_suff_replies, \ - check_last_ordered_3pc + check_last_ordered_3pc, sdk_json_couples_to_request_list from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.test_node import getNonPrimaryReplicas, ensureElectionsDone from plenum.test.view_change.helper import ensure_view_change @@ -26,15 +26,21 @@ def setup(request, looper, txnPoolNodeSet, client1, wallet1): def test_nodes_removes_request_keys_for_ordered(setup, looper, txnPoolNodeSet, - client1, wallet1): + sdk_pool_handle, + sdk_wallet_client): """ A node does not order requests since it is missing some 3PC messages, gets them from catchup. It then clears them from its request queues """ slow_node, fast_nodes = setup - reqs = send_reqs_batches_and_get_suff_replies( - looper, wallet1, client1, 10, 5) + reqs = sdk_json_couples_to_request_list( + send_reqs_batches_and_get_suff_replies( + looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 10, + 5)) ensure_all_nodes_have_same_data(looper, fast_nodes) assert slow_node.master_replica.last_ordered_3pc != \ fast_nodes[0].master_replica.last_ordered_3pc diff --git a/plenum/test/node_catchup/test_revert_during_catchup.py b/plenum/test/node_catchup/test_revert_during_catchup.py index 67e97a317d..ef92b5fe17 100644 --- a/plenum/test/node_catchup/test_revert_during_catchup.py +++ b/plenum/test/node_catchup/test_revert_during_catchup.py @@ -3,8 +3,8 @@ from plenum.common.constants import DOMAIN_LEDGER_ID, COMMIT from plenum.test import waits from plenum.test.delayers import cDelay, cr_delay -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \ - check_last_ordered_3pc, assertEquality +from plenum.test.helper import check_last_ordered_3pc, \ + assertEquality, sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataInequality, \ make_a_node_catchup_twice, ensure_all_nodes_have_same_data from plenum.test.spy_helpers import getAllReturnVals @@ -22,8 +22,8 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, txnPoolNodeSet, - client1, - wallet1): + sdk_pool_handle, + sdk_wallet_client): """ Delay COMMITs to a node such that when it needs to catchup, it needs to revert some unordered state. Also till this time the node should have @@ -34,8 +34,8 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, try to process delayed COMMITs, some COMMITs will be rejected but some will be processed since catchup was done for older ledger. """ - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, - 3 * Max3PCBatchSize) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 3 * Max3PCBatchSize) nprs = getNonPrimaryReplicas(txnPoolNodeSet, 0) slow_node = nprs[-1].node other_nodes = [n for n in txnPoolNodeSet if n != slow_node] @@ -47,8 +47,8 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, # Delay COMMITs to one node slow_node.nodeIbStasher.delay(cDelay(commit_delay, 0)) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, - 6 * Max3PCBatchSize) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 6 * Max3PCBatchSize) ensure_all_nodes_have_same_data(looper, other_nodes) waitNodeDataInequality(looper, slow_node, *other_nodes) @@ -122,6 +122,6 @@ def chk5(): checkProtocolInstanceSetup(looper, txnPoolNodeSet, retryWait=1) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, - 2 * Max3PCBatchSize) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 2 * Max3PCBatchSize) ensure_all_nodes_have_same_data(looper, nodes=txnPoolNodeSet) diff --git a/plenum/test/node_request/test_timestamp/test_timestamp_new_node.py b/plenum/test/node_request/test_timestamp/test_timestamp_new_node.py index 12360b819b..392cf0a722 100644 --- a/plenum/test/node_request/test_timestamp/test_timestamp_new_node.py +++ b/plenum/test/node_request/test_timestamp/test_timestamp_new_node.py @@ -8,8 +8,7 @@ txnCount = 20 Max3PCBatchSize = 4 -from plenum.test.node_catchup.conftest import nodeCreatedAfterSomeTxns, \ - nodeSetWithNodeAddedAfterSomeTxns +from plenum.test.node_catchup.conftest import sdk_node_created_after_some_txns from plenum.test.batching_3pc.conftest import tconf # lot of requests will be sent @@ -17,12 +16,12 @@ def test_new_node_accepts_timestamp(tconf, looper, txnPoolNodeSet, - nodeSetWithNodeAddedAfterSomeTxns, + sdk_node_created_after_some_txns, sdk_wallet_client, sdk_pool_handle): """ A new node joins the pool and is able to function properly without """ - _, new_node, _, _, _, _ = nodeSetWithNodeAddedAfterSomeTxns + _, new_node, _, _ = sdk_node_created_after_some_txns old_susp_count = get_timestamp_suspicion_count(new_node) # Don't wait for node to catchup, start sending requests sdk_send_random_and_check(looper, diff --git a/plenum/test/plugin/demo_plugin/test_catchup.py b/plenum/test/plugin/demo_plugin/test_catchup.py index 887d0e1ab0..3749f19e36 100644 --- a/plenum/test/plugin/demo_plugin/test_catchup.py +++ b/plenum/test/plugin/demo_plugin/test_catchup.py @@ -5,8 +5,8 @@ waitNodeDataEquality from plenum.test.plugin.demo_plugin import AUCTION_LEDGER_ID -from plenum.test.node_catchup.conftest import nodeCreatedAfterSomeTxns, \ - nodeSetWithNodeAddedAfterSomeTxns, newNodeCaughtUp +from plenum.test.node_catchup.conftest import sdk_node_created_after_some_txns, \ + sdk_node_set_with_node_added_after_some_txns, sdk_new_node_caught_up from plenum.test.plugin.demo_plugin.constants import AUCTION_END, PLACE_BID, \ AMOUNT, AUCTION_START from plenum.test.plugin.demo_plugin.test_plugin_request_handling import \ @@ -16,13 +16,13 @@ def test_new_node_catchup_plugin_ledger(txn_pool_node_set_post_creation, looper, some_requests, - newNodeCaughtUp): + sdk_new_node_caught_up): """ A new node catches up the demo plugin's ledger too """ - assert len(newNodeCaughtUp.getLedger(AUCTION_LEDGER_ID)) > 0 + assert len(sdk_new_node_caught_up.getLedger(AUCTION_LEDGER_ID)) > 0 for node in txn_pool_node_set_post_creation[:-1]: - assert len(newNodeCaughtUp.getLedger(AUCTION_LEDGER_ID)) == \ + assert len(sdk_new_node_caught_up.getLedger(AUCTION_LEDGER_ID)) == \ len(node.getLedger(AUCTION_LEDGER_ID)) @@ -55,12 +55,12 @@ def test_disconnected_node_catchup_plugin_ledger_txns(looper, txnPoolNodeSet, sdk_wallet_client, sdk_pool_handle, - newNodeCaughtUp): + sdk_new_node_caught_up): """ A node gets disconnected, a few config ledger txns happen, the disconnected node comes back up and catches up the config ledger """ - new_node = newNodeCaughtUp + new_node = sdk_new_node_caught_up disconnect_node_and_ensure_disconnected( looper, txnPoolNodeSet, new_node, stopNode=False) diff --git a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py index 1bbb43bf5b..bd38955ea6 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py @@ -92,6 +92,7 @@ def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle) +# TODO: probably delete @pytest.fixture(scope="module") def some_requests(txn_pool_node_set_post_creation, looper, stewardWallet, steward1, diff --git a/plenum/test/primary_election/test_primary_election_case6.py b/plenum/test/primary_election/test_primary_election_case6.py index ce25a60d1e..d1f9f800e8 100644 --- a/plenum/test/primary_election/test_primary_election_case6.py +++ b/plenum/test/primary_election/test_primary_election_case6.py @@ -2,7 +2,7 @@ from plenum.common.messages.node_messages import Nomination, Reelection, Primary from plenum.test.delayers import delay -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.test_node import checkNodesConnected, \ checkProtocolInstanceSetup from stp_core.loop.eventually import eventually diff --git a/plenum/test/primary_selection/conftest.py b/plenum/test/primary_selection/conftest.py index 957dbd6975..2b512f051f 100644 --- a/plenum/test/primary_selection/conftest.py +++ b/plenum/test/primary_selection/conftest.py @@ -5,7 +5,8 @@ from plenum.test.primary_selection.helper import check_newly_added_nodes, \ getPrimaryNodesIdxs from plenum.test.pool_transactions.helper import buildPoolClientAndWallet -from plenum.test.pool_transactions.conftest import nodeThetaAdded +from plenum.test.pool_transactions.conftest import nodeThetaAdded, \ + sdk_node_theta_added @pytest.fixture(scope="module") diff --git a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py index 03e255abd7..854bb9c644 100644 --- a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py +++ b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py @@ -2,11 +2,11 @@ from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.util import randomString -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.view_change.helper import ensure_several_view_change -from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected -from plenum.test.node_catchup.helper import ensureClientConnectedToNodesAndPoolLedgerSame, waitNodeDataEquality +from plenum.test.node_catchup.helper import waitNodeDataEquality def _get_ppseqno(nodes): diff --git a/plenum/test/primary_selection/test_catchup_after_view_change.py b/plenum/test/primary_selection/test_catchup_after_view_change.py index 4bd740d83b..180db2a06f 100644 --- a/plenum/test/primary_selection/test_catchup_after_view_change.py +++ b/plenum/test/primary_selection/test_catchup_after_view_change.py @@ -1,6 +1,6 @@ import pytest -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data, \ waitNodeDataInequality from plenum.test.delayers import cr_delay, ppDelay, pDelay, \ diff --git a/plenum/test/primary_selection/test_catchup_multiple_rounds.py b/plenum/test/primary_selection/test_catchup_multiple_rounds.py index df34f8b4c3..c4a05a05bf 100644 --- a/plenum/test/primary_selection/test_catchup_multiple_rounds.py +++ b/plenum/test/primary_selection/test_catchup_multiple_rounds.py @@ -2,9 +2,8 @@ from plenum.common.constants import DOMAIN_LEDGER_ID from plenum.test.delayers import delay_3pc_messages, icDelay -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \ - sendRandomRequests, waitForSufficientRepliesForRequests, checkViewNoForNodes, sdk_send_random_and_check, \ - sdk_send_random_requests, sdk_get_replies, sdk_get_and_check_replies +from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check, \ + sdk_send_random_requests, sdk_get_and_check_replies from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.batching_3pc.conftest import tconf diff --git a/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py b/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py index f78d924a44..839f04386c 100644 --- a/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py +++ b/plenum/test/primary_selection/test_new_node_accepts_chosen_primary.py @@ -2,7 +2,7 @@ from stp_core.common.log import getlogger -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.test_node import TestNode, TestViewChanger from plenum.test.view_change.helper import ensure_view_change_complete diff --git a/plenum/test/primary_selection/test_primary_selection.py b/plenum/test/primary_selection/test_primary_selection.py index 3f153a3e61..7febf864ea 100644 --- a/plenum/test/primary_selection/test_primary_selection.py +++ b/plenum/test/primary_selection/test_primary_selection.py @@ -2,7 +2,7 @@ import pytest -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.primary_selection.helper import \ check_rank_consistent_across_each_node from plenum.test.view_change.helper import ensure_view_change diff --git a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py index 435077e605..ffe7302247 100644 --- a/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py +++ b/plenum/test/primary_selection/test_primary_selection_after_primary_demotion_and_pool_restart.py @@ -2,13 +2,12 @@ from stp_core.common.log import getlogger -from plenum.common.constants import ALIAS, SERVICES, VALIDATOR -from plenum.test.pool_transactions.helper import updateNodeData, sdk_send_update_node +from plenum.test.pool_transactions.helper import sdk_send_update_node from plenum.test.test_node import TestNode, checkNodesConnected, \ ensureElectionsDone from plenum.test.helper import checkViewNoForNodes, \ - sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check + sdk_send_random_and_check from plenum.test.primary_selection.helper import getPrimaryNodesIdxs from plenum.common.config_helper import PNodeConfigHelper @@ -41,8 +40,7 @@ def test_primary_selection_after_primary_demotion_and_pool_restart(looper, ensureElectionsDone(looper, restNodes) # ensure pool is working properly - sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_steward, 3) + logger.info("2. restart pool") # Stopping existing nodes diff --git a/plenum/test/primary_selection/test_primary_selection_pool_txn.py b/plenum/test/primary_selection/test_primary_selection_pool_txn.py index a3bc89bbcf..6176ef489b 100644 --- a/plenum/test/primary_selection/test_primary_selection_pool_txn.py +++ b/plenum/test/primary_selection/test_primary_selection_pool_txn.py @@ -1,19 +1,10 @@ import pytest +from plenum.test.node_request.helper import sdk_ensure_pool_functional -from plenum.test.helper import send_reqs_batches_and_get_suff_replies -from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.pool_transactions.helper import add_2_nodes +from plenum.test.pool_transactions.helper import sdk_add_2_nodes from plenum.test.primary_selection.helper import check_newly_added_nodes -# TODO: replace this func with sdk_ensure_pool_functional -def ensure_pool_functional(looper, nodes, wallet, client, num_reqs=10, - num_batches=2): - send_reqs_batches_and_get_suff_replies(looper, wallet, client, num_reqs, - num_batches) - ensure_all_nodes_have_same_data(looper, nodes) - - def check_accepted_view_change_sent(node, nodes): for other_node in nodes: if node == other_node: @@ -23,20 +14,23 @@ def check_accepted_view_change_sent(node, nodes): node.view_changer._accepted_view_change_done_message -def test_primary_selection_non_genesis_node(one_node_added, looper, - txnPoolNodeSet, stewardWallet, - steward1): - ensure_pool_functional(looper, txnPoolNodeSet, stewardWallet, steward1) +def test_primary_selection_non_genesis_node(sdk_one_node_added, looper, + txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward): + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_steward) @pytest.fixture(scope='module') -def two_more_nodes_added(one_node_added, looper, txnPoolNodeSet, - stewardWallet, steward1, +def two_more_nodes_added(sdk_one_node_added, looper, txnPoolNodeSet, + sdk_pool_handle, sdk_wallet_steward, tdir, client_tdir, tconf, allPluginsPath): # check_accepted_view_change_sent(one_node_added, txnPoolNodeSet) - new_nodes = add_2_nodes(looper, txnPoolNodeSet, steward1, stewardWallet, - tdir, client_tdir, tconf, allPluginsPath) + new_nodes = sdk_add_2_nodes(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, + tdir, tconf, allPluginsPath) check_newly_added_nodes(looper, txnPoolNodeSet, new_nodes) return new_nodes @@ -46,11 +40,11 @@ def test_primary_selection_increase_f( two_more_nodes_added, looper, txnPoolNodeSet, - stewardWallet, - steward1): + sdk_wallet_steward, + sdk_pool_handle): # for n in two_more_nodes_added: # check_accepted_view_change_sent(n, txnPoolNodeSet) - ensure_pool_functional(looper, txnPoolNodeSet, stewardWallet, steward1) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) # TODO: Add more tests to make one next primary crashed, malicious, ensure primary # selection happens after catchup diff --git a/plenum/test/primary_selection/test_recover_more_than_f_failure.py b/plenum/test/primary_selection/test_recover_more_than_f_failure.py index 9304214bc9..ae520699cc 100644 --- a/plenum/test/primary_selection/test_recover_more_than_f_failure.py +++ b/plenum/test/primary_selection/test_recover_more_than_f_failure.py @@ -1,9 +1,7 @@ -import pytest - from stp_core.common.log import getlogger -from plenum.test.helper import stopNodes, waitForViewChange, \ - sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import waitForViewChange, \ + sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected diff --git a/plenum/test/primary_selection/test_recover_primary_no_view_change.py b/plenum/test/primary_selection/test_recover_primary_no_view_change.py index 7d329e34ea..bdc33689b2 100644 --- a/plenum/test/primary_selection/test_recover_primary_no_view_change.py +++ b/plenum/test/primary_selection/test_recover_primary_no_view_change.py @@ -4,7 +4,7 @@ from plenum.test.conftest import getValueFromModule from plenum.test.helper import waitForViewChange, \ - sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check + sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.test_node import ensureElectionsDone from plenum.test.view_change.helper import start_stopped_node diff --git a/plenum/test/script/helper.py b/plenum/test/script/helper.py index 0ef9fb3b7a..d777802052 100644 --- a/plenum/test/script/helper.py +++ b/plenum/test/script/helper.py @@ -1,14 +1,12 @@ import pytest +from plenum.common.constants import VALIDATOR -from plenum.client.wallet import Wallet +from plenum.test.pool_transactions.helper import sdk_send_update_node, sdk_pool_refresh from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger -from plenum.common.script_helper import changeHA -from plenum.common.signer_did import DidSigner -from plenum.common.util import getMaxFailures +from plenum.common.util import hexToFriendly from plenum.test import waits -from plenum.test.helper import waitForSufficientRepliesForRequests, \ - sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.test_client import genTestClient from plenum.test.test_node import TestNode, checkNodesConnected, \ ensureElectionsDone @@ -24,32 +22,31 @@ def looper(txnPoolNodesLooper): def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, - poolTxnData, poolTxnStewardNames, tconf, shouldBePrimary, tdir): + tconf, shouldBePrimary, tdir, + sdk_pool_handle, sdk_wallet_stewards): # prepare new ha for node and client stack subjectedNode = None - stewardName = None - stewardsSeed = None + node_index = None for nodeIndex, n in enumerate(txnPoolNodeSet): if shouldBePrimary == n.has_master_primary: subjectedNode = n - stewardName = poolTxnStewardNames[nodeIndex] - stewardsSeed = poolTxnData["seeds"][stewardName].encode() + node_index = nodeIndex break nodeStackNewHA, clientStackNewHA = genHa(2) logger.debug("change HA for node: {} to {}".format( subjectedNode.name, (nodeStackNewHA, clientStackNewHA))) - nodeSeed = poolTxnData["seeds"][subjectedNode.name].encode() - # change HA - stewardClient, req = changeHA(looper, tconf, subjectedNode.name, nodeSeed, - nodeStackNewHA, stewardName, stewardsSeed, - basedir=tdirWithClientPoolTxns) - - waitForSufficientRepliesForRequests(looper, stewardClient, - requests=[req]) + sdk_wallet_steward = sdk_wallet_stewards[node_index] + node_dest = hexToFriendly(subjectedNode.nodestack.verhex) + sdk_send_update_node(looper, sdk_wallet_steward, + sdk_pool_handle, + node_dest, subjectedNode.name, + nodeStackNewHA[0], nodeStackNewHA[1], + clientStackNewHA[0], clientStackNewHA[1], + services=[VALIDATOR]) # stop node for which HA will be changed subjectedNode.stop() @@ -64,6 +61,7 @@ def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, looper.add(restartedNode) txnPoolNodeSet[nodeIndex] = restartedNode looper.run(checkNodesConnected(txnPoolNodeSet, customTimeout=70)) + sdk_pool_refresh(looper, sdk_pool_handle) electionTimeout = waits.expectedPoolElectionTimeout( nodeCount=len(txnPoolNodeSet), @@ -78,7 +76,7 @@ def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, usePoolLedger=True) looper.add(anotherClient) looper.run(eventually(anotherClient.ensureConnectedToNodes)) - stewardWallet = Wallet(stewardName) - stewardWallet.addIdentifier(signer=DidSigner(seed=stewardsSeed)) - sendReqsToNodesAndVerifySuffReplies( - looper, stewardWallet, stewardClient, 8) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_steward, + 8) diff --git a/plenum/test/script/test_change_non_primary_node_ha.py b/plenum/test/script/test_change_non_primary_node_ha.py index da52b4cdf5..152ad1b518 100644 --- a/plenum/test/script/test_change_non_primary_node_ha.py +++ b/plenum/test/script/test_change_non_primary_node_ha.py @@ -1,6 +1,5 @@ import pytest -from plenum.test.script.helper import looper from stp_core.common.log import getlogger from plenum.test.script.helper import changeNodeHa @@ -14,12 +13,13 @@ @pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330') def testChangeNodeHaForNonPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns, - poolTxnData, poolTxnStewardNames, tconf, tdir): + poolTxnData, poolTxnStewardNames, tconf, tdir, + sdk_pool_handle, sdk_wallet_stewards): changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, - poolTxnData, - poolTxnStewardNames, tconf, shouldBePrimary=False, - tdir=tdir) + tdir=tdir, + sdk_pool_handle=sdk_pool_handle, + sdk_wallet_stewards=sdk_wallet_stewards) diff --git a/plenum/test/script/test_change_primary_node_ha.py b/plenum/test/script/test_change_primary_node_ha.py index e7289b5e87..c766ad61a0 100644 --- a/plenum/test/script/test_change_primary_node_ha.py +++ b/plenum/test/script/test_change_primary_node_ha.py @@ -14,12 +14,13 @@ @pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330') def testChangeNodeHaForPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns, - poolTxnData, poolTxnStewardNames, tconf, tdir): + poolTxnData, poolTxnStewardNames, tconf, tdir, + sdk_pool_handle, sdk_wallet_stewards): changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, - poolTxnData, - poolTxnStewardNames, tconf, shouldBePrimary=True, - tdir=tdir) + tdir=tdir, + sdk_pool_handle=sdk_pool_handle, + sdk_wallet_stewards=sdk_wallet_stewards) diff --git a/plenum/test/test_current_state_propagation.py b/plenum/test/test_current_state_propagation.py index 01aa43246a..7d92dff176 100644 --- a/plenum/test/test_current_state_propagation.py +++ b/plenum/test/test_current_state_propagation.py @@ -1,24 +1,22 @@ from plenum.test.node_catchup.conftest import whitelist, \ - looper, nodeCreatedAfterSomeTxns, nodeSetWithNodeAddedAfterSomeTxns, \ - newNodeCaughtUp + looper, sdk_node_set_with_node_added_after_some_txns, \ + sdk_new_node_caught_up, sdk_node_created_after_some_txns from plenum.test.pool_transactions.helper import \ - disconnect_node_and_ensure_disconnected, reconnect_node_and_ensure_connected -from plenum.test.node_catchup.helper import waitNodeDataEquality, \ - waitNodeDataInequality, checkNodeDataForEquality + disconnect_node_and_ensure_disconnected from plenum.test.test_node import checkNodesConnected from plenum.test.view_change.helper import start_stopped_node -def test_current_state_propagation(newNodeCaughtUp, +def test_current_state_propagation(sdk_new_node_caught_up, txnPoolNodeSet, - nodeSetWithNodeAddedAfterSomeTxns, + sdk_node_set_with_node_added_after_some_txns, tconf, tdir, allPluginsPath): """ Checks that nodes send CurrentState to lagged nodes. """ # 1. Start pool - looper, new_node, client, wallet, _, _ = nodeSetWithNodeAddedAfterSomeTxns + looper, new_node, _, _ = sdk_node_set_with_node_added_after_some_txns # 2. Stop one node disconnect_node_and_ensure_disconnected(looper, diff --git a/plenum/test/test_dirty_read.py b/plenum/test/test_dirty_read.py deleted file mode 100644 index 8084cfb85d..0000000000 --- a/plenum/test/test_dirty_read.py +++ /dev/null @@ -1,22 +0,0 @@ -from plenum.common.types import f -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, \ - getRepliesFromClientInbox, send_signed_requests, \ - waitForSufficientRepliesForRequests -from plenum.common.constants import GET_TXN, DATA, TXN_TYPE, DOMAIN_LEDGER_ID -from plenum.common.messages.node_messages import Ordered -from stp_core.common.log import getlogger - -logger = getlogger() - - -def make_node_slow(node): - old = node.serviceReplicas - - async def serviceReplicas(limit): - for replica in node.replicas: - for index, message in enumerate(list(replica.outBox)): - if isinstance(message, Ordered): - del replica.outBox[index] - return await old(limit) - - node.serviceReplicas = serviceReplicas diff --git a/plenum/test/test_memory_consumpion.py b/plenum/test/test_memory_consumpion.py index 77ba471f9d..fd72c2e2c4 100644 --- a/plenum/test/test_memory_consumpion.py +++ b/plenum/test/test_memory_consumpion.py @@ -2,7 +2,7 @@ from stp_core.common.log import getlogger from plenum.common.perf_util import get_size -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import \ ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.pool_transactions.helper import buildPoolClientAndWallet @@ -33,11 +33,13 @@ def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames, fVal = 1 for (client, wallet) in clients: logger.debug("{} sending {} requests".format(client, numRequests)) - sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, - wallet, - client, - numRequests, - fVal) + # sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, + # wallet, + # client, + # numRequests, + # fVal) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_steward, 3) logger.debug("{} sent {} requests".format(client, numRequests)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}". diff --git a/plenum/test/test_performance.py b/plenum/test/test_performance.py index 086b6f04b1..c2da8e3499 100644 --- a/plenum/test/test_performance.py +++ b/plenum/test/test_performance.py @@ -9,22 +9,20 @@ from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState from plenum.common.perf_util import get_memory_usage, timeit from plenum.test.delayers import cr_delay -from plenum.test.test_client import TestClient from stp_core.loop.eventually import eventually from plenum.common.types import HA from stp_core.common.log import getlogger, Logger -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, \ check_ledger_state from plenum.test.pool_transactions.helper import \ - disconnect_node_and_ensure_disconnected, buildPoolClientAndWallet + disconnect_node_and_ensure_disconnected from plenum.test.test_node import checkNodesConnected, TestNode from plenum.test import waits # noinspection PyUnresolvedReferences -from plenum.test.node_catchup.conftest import whitelist, \ - nodeCreatedAfterSomeTxns, nodeSetWithNodeAddedAfterSomeTxns, newNodeCaughtUp +from plenum.test.node_catchup.conftest import whitelist @pytest.fixture @@ -65,22 +63,16 @@ def change_checkpoint_freq(tconf): @skipper -def test_node_load(looper, txnPoolNodeSet, tconf, - tdirWithPoolTxns, allPluginsPath, - poolTxnStewardData, capsys): - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, - tdirWithPoolTxns, - clientClass=TestClient) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) - +def test_node_load(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + capsys): client_batches = 150 txns_per_batch = 25 for i in range(client_batches): s = perf_counter() - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, - txns_per_batch, - override_timeout_limit=True) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, txns_per_batch) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s)) @@ -89,14 +81,9 @@ def test_node_load(looper, txnPoolNodeSet, tconf, @skipper def test_node_load_consistent_time(tconf, change_checkpoint_freq, disable_node_monitor_config, looper, - txnPoolNodeSet, tdirWithPoolTxns, - allPluginsPath, poolTxnStewardData, capsys): + txnPoolNodeSet, capsys, + sdk_pool_handle, sdk_wallet_client): # One of the reason memory grows is because spylog grows - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, - tdirWithPoolTxns, - clientClass=TestClient) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) client_batches = 300 txns_per_batch = 25 @@ -122,9 +109,8 @@ def test_node_load_consistent_time(tconf, change_checkpoint_freq, for i in range(client_batches): s = perf_counter() - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, - txns_per_batch, - override_timeout_limit=True) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, txns_per_batch) t = perf_counter() - s with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. @@ -161,23 +147,18 @@ def test_node_load_consistent_time(tconf, change_checkpoint_freq, sd = tolerance_factor * pstdev(time_log) assert m > t or abs(t - m) <= sd, '{} {}'.format(abs(t - m), sd) time_log.append(t) - # Since client checks inbox for sufficient replies, clear inbox so that - # it takes constant time to check replies for each batch - client.inBox.clear() - client.txnLog.reset() @skipper -def test_node_load_after_add(newNodeCaughtUp, txnPoolNodeSet, tconf, - tdirWithPoolTxns, allPluginsPath, - poolTxnStewardData, looper, client1, wallet1, - capsys): +def test_node_load_after_add(sdk_new_node_caught_up, txnPoolNodeSet, + looper, sdk_pool_handle, + sdk_wallet_client, capsys): """ A node that restarts after some transactions should eventually get the transactions which happened while it was down :return: """ - new_node = newNodeCaughtUp + new_node = sdk_new_node_caught_up logger.debug("Sending requests") # Here's where we apply some load @@ -185,29 +166,30 @@ def test_node_load_after_add(newNodeCaughtUp, txnPoolNodeSet, tconf, txns_per_batch = 25 for i in range(client_batches): s = perf_counter() - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, - txns_per_batch, - override_timeout_limit=True) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, txns_per_batch) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s)) logger.debug("Starting the stopped node, {}".format(new_node)) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 5) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4]) @skipper -def test_node_load_after_add_then_disconnect(newNodeCaughtUp, txnPoolNodeSet, - tconf, looper, client1, wallet1, +def test_node_load_after_add_then_disconnect(sdk_new_node_caught_up, txnPoolNodeSet, + tconf, looper, sdk_pool_handle, + sdk_wallet_client, tdirWithPoolTxns, allPluginsPath, - poolTxnStewardData, capsys): + capsys): """ A node that restarts after some transactions should eventually get the transactions which happened while it was down :return: """ - new_node = newNodeCaughtUp + new_node = sdk_new_node_caught_up with capsys.disabled(): print("Stopping node {} with pool ledger size {}". format(new_node, new_node.poolManager.txnSeqNo)) @@ -218,9 +200,8 @@ def test_node_load_after_add_then_disconnect(newNodeCaughtUp, txnPoolNodeSet, txns_per_batch = 10 for i in range(client_batches): s = perf_counter() - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, - txns_per_batch, - override_timeout_limit=True) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, txns_per_batch) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s)) @@ -254,7 +235,8 @@ def test_node_load_after_add_then_disconnect(newNodeCaughtUp, txnPoolNodeSet, waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4], customTimeout=timeout) - sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, 5) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:4]) @@ -269,14 +251,11 @@ def test_nodestack_contexts_are_discrete(txnPoolNodeSet): @skipper def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf, - tdirWithPoolTxns, allPluginsPath, - poolTxnStewardData, capsys): - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, - tdirWithPoolTxns, - clientClass=TestClient) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) - + allPluginsPath, + tdirWithPoolTxns, + sdk_pool_handle, + sdk_wallet_client, + capsys): nodes = txnPoolNodeSet x = nodes[-1] @@ -291,9 +270,8 @@ def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf, txns_per_batch = 10 for i in range(client_batches): s = perf_counter() - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, - txns_per_batch, - override_timeout_limit=True) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, txns_per_batch) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s)) @@ -310,17 +288,9 @@ def test_node_load_after_disconnect(looper, txnPoolNodeSet, tconf, def test_node_load_after_one_node_drops_all_msgs( looper, txnPoolNodeSet, - tconf, - tdirWithPoolTxns, - allPluginsPath, - poolTxnStewardData, + sdk_pool_handle, + sdk_wallet_client, capsys): - client, wallet = buildPoolClientAndWallet(poolTxnStewardData, - tdirWithPoolTxns, - clientClass=TestClient) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) - nodes = txnPoolNodeSet x = nodes[-1] @@ -337,9 +307,8 @@ def handleOneNodeMsg(self, wrappedMsg): txns_per_batch = 25 for i in range(client_batches): s = perf_counter() - sendReqsToNodesAndVerifySuffReplies(looper, wallet, client, - txns_per_batch, - override_timeout_limit=True) + sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, txns_per_batch) with capsys.disabled(): print('{} executed {} client txns in {:.2f} seconds'. format(i + 1, txns_per_batch, perf_counter() - s)) diff --git a/plenum/test/test_state_regenerated_from_ledger.py b/plenum/test/test_state_regenerated_from_ledger.py index 6bb014c49d..d6aeb0437d 100644 --- a/plenum/test/test_state_regenerated_from_ledger.py +++ b/plenum/test/test_state_regenerated_from_ledger.py @@ -15,8 +15,8 @@ def test_state_regenerated_from_ledger( looper, txnPoolNodeSet, - client1, - wallet1, + sdk_pool_handle, + sdk_wallet_client, tdir, tconf, allPluginsPath): @@ -24,8 +24,11 @@ def test_state_regenerated_from_ledger( Node loses its state database but recreates it from ledger after start """ sent_batches = 10 - send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, - 5 * sent_batches, sent_batches) + send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 5 * sent_batches, + sent_batches) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) node_to_stop = txnPoolNodeSet[-1] node_state = node_to_stop.states[DOMAIN_LEDGER_ID] diff --git a/plenum/test/view_change/helper.py b/plenum/test/view_change/helper.py index a537e35674..29d58da2f5 100644 --- a/plenum/test/view_change/helper.py +++ b/plenum/test/view_change/helper.py @@ -5,8 +5,7 @@ from plenum.test.delayers import delayNonPrimaries, delay_3pc_messages, \ reset_delays_and_process_delayeds from plenum.test.helper import checkViewNoForNodes, \ - sendReqsToNodesAndVerifySuffReplies, sdk_send_random_requests, \ - sdk_send_random_and_check + sdk_send_random_requests, sdk_send_random_and_check from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data diff --git a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py index dc93cf60ce..d0cc2635fb 100644 --- a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py +++ b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py @@ -1,10 +1,9 @@ import pytest -import types from plenum.test.view_change.helper import ensure_all_nodes_have_same_data, \ - ensure_view_change_by_primary_restart, start_stopped_node + ensure_view_change_by_primary_restart from plenum.common.constants import DOMAIN_LEDGER_ID, LedgerState, POOL_LEDGER_ID -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually @@ -12,7 +11,7 @@ waitNodeDataEquality from plenum.common.util import randomString from plenum.test.test_node import checkNodesConnected -from plenum.test.pool_transactions.helper import addNewStewardAndNode, sdk_add_new_steward_and_node +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node from plenum.test import waits from plenum.common.startable import Mode diff --git a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py index f447aa135b..c4def36f7b 100644 --- a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py +++ b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py @@ -17,8 +17,9 @@ @pytest.mark.skip(reason='Pending complete implementation') def test_node_detecting_lag_from_view_change_done_messages(txnPoolNodeSet, - looper, wallet1, - client1, + looper, + sdk_pool_handle, + sdk_wallet_client, tconf): """ A node is slow and after view change starts, it marks it's `last_prepared` @@ -29,7 +30,11 @@ def test_node_detecting_lag_from_view_change_done_messages(txnPoolNodeSet, before view change it has different last_prepared from others. Also delay processing of COMMITs and INSTANCE_CHANGEs by other nodes """ - send_reqs_batches_and_get_suff_replies(looper, wallet1, client1, 2 * 3, 3) + send_reqs_batches_and_get_suff_replies(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 2 * 3, + 3) ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[-1].node diff --git a/plenum/test/view_change/test_view_change.py b/plenum/test/view_change/test_view_change.py index d296c7efaf..24768207de 100644 --- a/plenum/test/view_change/test_view_change.py +++ b/plenum/test/view_change/test_view_change.py @@ -1,4 +1,4 @@ -from plenum.test.helper import sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.spy_helpers import get_count from plenum.test.test_node import ensureElectionsDone diff --git a/plenum/test/view_change/test_view_change_min_cathup_timeout.py b/plenum/test/view_change/test_view_change_min_cathup_timeout.py index 550917fe33..cfeea99227 100644 --- a/plenum/test/view_change/test_view_change_min_cathup_timeout.py +++ b/plenum/test/view_change/test_view_change_min_cathup_timeout.py @@ -3,10 +3,8 @@ import pytest from plenum.test.node_request.helper import sdk_ensure_pool_functional -from plenum.test.helper import waitForViewChange, sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check +from plenum.test.helper import waitForViewChange, sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.primary_selection.test_primary_selection_pool_txn import \ - ensure_pool_functional from plenum.test.test_node import ensureElectionsDone from stp_core.loop.exceptions import EventuallyTimeoutException diff --git a/plenum/test/view_change/test_view_change_on_master_degraded.py b/plenum/test/view_change/test_view_change_on_master_degraded.py index 4213a5e175..b4cd866de9 100644 --- a/plenum/test/view_change/test_view_change_on_master_degraded.py +++ b/plenum/test/view_change/test_view_change_on_master_degraded.py @@ -1,11 +1,9 @@ import types -import pytest - from plenum.server.view_change.view_changer import ViewChanger from plenum.test.delayers import delayNonPrimaries from plenum.test.helper import waitForViewChange, \ - sendReqsToNodesAndVerifySuffReplies, sdk_send_random_and_check + sdk_send_random_and_check from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.test_node import get_master_primary_node, getPrimaryReplica, \ ensureElectionsDone diff --git a/plenum/test/view_change/test_view_change_timeout.py b/plenum/test/view_change/test_view_change_timeout.py index cd69d952ad..c2197d0802 100644 --- a/plenum/test/view_change/test_view_change_timeout.py +++ b/plenum/test/view_change/test_view_change_timeout.py @@ -1,9 +1,9 @@ import pytest +from plenum.test.node_request.helper import sdk_ensure_pool_functional + from plenum.test.delayers import reset_delays_and_process_delayeds, vcd_delay from plenum.test.helper import waitForViewChange, stopNodes from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.primary_selection.test_primary_selection_pool_txn import \ - ensure_pool_functional from plenum.test.spy_helpers import get_count, getAllReturnVals from plenum.test.test_node import get_master_primary_node, \ ensureElectionsDone @@ -32,7 +32,7 @@ def setup(txnPoolNodeSet, looper): def test_view_change_retry_by_timeout( - txnPoolNodeSet, looper, setup, wallet1, client1): + txnPoolNodeSet, looper, setup, sdk_pool_handle, sdk_wallet_client): """ Verifies that a view change is restarted if it is not completed in time """ @@ -71,11 +71,14 @@ def test_view_change_retry_by_timeout( for node in txnPoolNodeSet: assert node.viewNo - initial_view_no == 2 - ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + sdk_wallet_client, + sdk_pool_handle) def test_multiple_view_change_retries_by_timeouts( - txnPoolNodeSet, looper, setup, wallet1, client1): + txnPoolNodeSet, looper, setup, + sdk_pool_handle, sdk_wallet_client): """ Verifies that a view change is restarted each time when the previous one is timed out @@ -117,11 +120,13 @@ def check_timeout_callback_called(times): for node in txnPoolNodeSet: assert node.viewNo - initial_view_no == 4 - ensure_pool_functional(looper, txnPoolNodeSet, wallet1, client1) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + sdk_wallet_client, + sdk_pool_handle) def test_view_change_restarted_by_timeout_if_next_primary_disconnected( - txnPoolNodeSet, looper, setup, wallet1, client1): + txnPoolNodeSet, looper, setup): """ Verifies that a view change is restarted by timeout if the next primary has been disconnected From dae8722a3f651d5f84244a4eaea7a194175ff14e Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 28 Mar 2018 08:06:46 +0300 Subject: [PATCH 45/73] little fix Signed-off-by: ArtObr --- plenum/test/test_current_state_propagation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plenum/test/test_current_state_propagation.py b/plenum/test/test_current_state_propagation.py index 7d92dff176..b9c730470a 100644 --- a/plenum/test/test_current_state_propagation.py +++ b/plenum/test/test_current_state_propagation.py @@ -1,5 +1,5 @@ from plenum.test.node_catchup.conftest import whitelist, \ - looper, sdk_node_set_with_node_added_after_some_txns, \ + sdk_node_set_with_node_added_after_some_txns, \ sdk_new_node_caught_up, sdk_node_created_after_some_txns from plenum.test.pool_transactions.helper import \ disconnect_node_and_ensure_disconnected From 7adb4392e0199ec5c8c1c57e7fe49b1b67b2e494 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 28 Mar 2018 08:46:32 +0300 Subject: [PATCH 46/73] little fix Signed-off-by: ArtObr --- .../test/primary_selection/test_primary_selection_pool_txn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plenum/test/primary_selection/test_primary_selection_pool_txn.py b/plenum/test/primary_selection/test_primary_selection_pool_txn.py index 6176ef489b..5bd4c42205 100644 --- a/plenum/test/primary_selection/test_primary_selection_pool_txn.py +++ b/plenum/test/primary_selection/test_primary_selection_pool_txn.py @@ -18,8 +18,8 @@ def test_primary_selection_non_genesis_node(sdk_one_node_added, looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward): sdk_ensure_pool_functional(looper, txnPoolNodeSet, - sdk_pool_handle, - sdk_wallet_steward) + sdk_wallet_steward, + sdk_pool_handle) @pytest.fixture(scope='module') From 4804c50aa9ca585f703bcdabcbd9132fcd00c964 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 28 Mar 2018 11:48:42 +0300 Subject: [PATCH 47/73] little fix Signed-off-by: ArtObr --- ..._node_reject_invalid_txn_during_catchup.py | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py b/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py index 0329d64c5f..855249b58f 100644 --- a/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py +++ b/plenum/test/node_catchup/test_node_reject_invalid_txn_during_catchup.py @@ -1,7 +1,12 @@ import pytest import types +from plenum.common.util import randomString +from plenum.test.helper import sdk_send_random_and_check + from plenum.common.ledger import Ledger +from plenum.test.conftest import getValueFromModule +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node, sdk_pool_refresh from stp_core.common.log import getlogger from plenum.common.constants import TXN_TYPE, DOMAIN_LEDGER_ID from plenum.common.messages.node_messages import CatchupReq, CatchupRep @@ -18,8 +23,11 @@ txnCount = 10 -def testNodeRejectingInvalidTxns(tconf, txnPoolNodeSet, patched_node, - sdk_node_created_after_some_txns): +def testNodeRejectingInvalidTxns(looper, sdk_pool_handle, sdk_wallet_client, + tconf, tdir, txnPoolNodeSet, patched_node, + request, + sdk_wallet_steward, testNodeClass, allPluginsPath, + do_post_node_creation): """ A newly joined node is catching up and sends catchup requests to other nodes but one of the nodes replies with incorrect transactions. The newly @@ -27,7 +35,20 @@ def testNodeRejectingInvalidTxns(tconf, txnPoolNodeSet, patched_node, the node. Ii thus cannot complete the process till the timeout and then requests the missing transactions. """ - looper, new_node, _, _ = sdk_node_created_after_some_txns + txnCount = getValueFromModule(request, "txnCount", 5) + sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + txnCount) + new_steward_name = randomString() + new_node_name = "Epsilon" + new_steward_wallet_handle, new_node = sdk_add_new_steward_and_node( + looper, sdk_pool_handle, sdk_wallet_steward, + new_steward_name, new_node_name, tdir, tconf, nodeClass=testNodeClass, + allPluginsPath=allPluginsPath, autoStart=True, + do_post_node_creation=do_post_node_creation) + sdk_pool_refresh(looper, sdk_pool_handle) + bad_node = patched_node do_not_tell_clients_about_newly_joined_node(txnPoolNodeSet) From 0bc3e5e0917d33951aef8cea9acd3f5e8092a969 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 28 Mar 2018 22:35:59 +0300 Subject: [PATCH 48/73] Review changes Signed-off-by: ArtObr --- ...test_catchup_while_new_request_incoming.py | 3 +- ...plus_2_nodes_but_not_primary_off_and_on.py | 2 + plenum/test/pool_transactions/helper.py | 1 + plenum/test/test_dirty_read.py | 49 +++++++++++++++++++ plenum/test/test_memory_consumpion.py | 7 +-- ...in_after_view_change_by_primary_restart.py | 3 +- 6 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 plenum/test/test_dirty_read.py diff --git a/plenum/test/node_catchup/test_catchup_while_new_request_incoming.py b/plenum/test/node_catchup/test_catchup_while_new_request_incoming.py index 49e6ce9224..0c3dbfe4d6 100644 --- a/plenum/test/node_catchup/test_catchup_while_new_request_incoming.py +++ b/plenum/test/node_catchup/test_catchup_while_new_request_incoming.py @@ -8,7 +8,7 @@ from plenum.test.helper import sdk_send_random_and_check, \ sdk_send_random_requests from plenum.test.node_catchup.helper import checkNodeDataForEquality -from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node, sdk_pool_refresh from plenum.test.test_node import TestNode from stp_core.loop.eventually import eventually @@ -49,6 +49,7 @@ def chkAfterCall(self, req, frm): looper, sdk_pool_handle, sdk_wallet_steward, new_steward_name, new_node_name, tdir, tconf, nodeClass=testNodeClass, allPluginsPath=allPluginsPath, autoStart=True) + sdk_pool_refresh(looper, sdk_pool_handle) txnPoolNodeSet.append(new_node) looper.runFor(2) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 5) diff --git a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py index d6cfea9a81..cb1c4615be 100644 --- a/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py +++ b/plenum/test/node_request/test_quorum_f_plus_2_nodes_but_not_primary_off_and_on.py @@ -10,6 +10,8 @@ from plenum.test.test_node import ensureElectionsDone, getRequiredInstances from plenum.test.view_change.helper import start_stopped_node +# TODO: restore previous value (200) after sdk will +# have api call to change its timeout TestRunningTimeLimitSec = 400 nodeCount = 5 diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index bd2646f2a2..d2ae536851 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -447,6 +447,7 @@ def sdk_send_update_node(looper, sdk_submitter_wallet, # waitng for replies reply = sdk_get_and_check_replies(looper, [request_couple])[0][1] + sdk_pool_refresh(looper, sdk_pool_handle) return reply diff --git a/plenum/test/test_dirty_read.py b/plenum/test/test_dirty_read.py new file mode 100644 index 0000000000..681988cf2f --- /dev/null +++ b/plenum/test/test_dirty_read.py @@ -0,0 +1,49 @@ +from plenum.common.types import f +from plenum.test.helper import getRepliesFromClientInbox, \ + send_signed_requests, \ + waitForSufficientRepliesForRequests, sdk_send_random_and_check, sdk_get_and_check_replies +from plenum.common.constants import GET_TXN, DATA, TXN_TYPE, DOMAIN_LEDGER_ID +from plenum.common.messages.node_messages import Ordered +from plenum.test.pool_transactions.helper import sdk_build_get_txn_request, sdk_sign_and_send_prepared_request +from stp_core.common.log import getlogger + +logger = getlogger() + + +def make_node_slow(node): + old = node.serviceReplicas + + async def serviceReplicas(limit): + for replica in node.replicas: + for index, message in enumerate(list(replica.outBox)): + if isinstance(message, Ordered): + del replica.outBox[index] + return await old(limit) + + node.serviceReplicas = serviceReplicas + + +def test_dirty_read(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_client): + """ + Tests the case when read request comes before write request is + not executed on some nodes + """ + + slow_nodes = list(txnPoolNodeSet)[2:4] + for node in slow_nodes: + logger.debug("Making node {} slow".format(node)) + make_node_slow(node) + + received_replies = sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + 1) + seq_no = received_replies[0][1]["result"]["seqNo"] + _, did = sdk_wallet_client + req = sdk_build_get_txn_request(looper, did, seq_no) + request = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client, + sdk_pool_handle, req) + received_replies = sdk_get_and_check_replies(looper, [request]) + results = [str(reply[1]['result'][DATA]) for reply in received_replies] + + assert len(set(results)) == 1 diff --git a/plenum/test/test_memory_consumpion.py b/plenum/test/test_memory_consumpion.py index fd72c2e2c4..62cbc55a0c 100644 --- a/plenum/test/test_memory_consumpion.py +++ b/plenum/test/test_memory_consumpion.py @@ -33,13 +33,8 @@ def testRequestsSize(txnPoolNodesLooper, txnPoolNodeSet, poolTxnClientNames, fVal = 1 for (client, wallet) in clients: logger.debug("{} sending {} requests".format(client, numRequests)) - # sendReqsToNodesAndVerifySuffReplies(txnPoolNodesLooper, - # wallet, - # client, - # numRequests, - # fVal) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_steward, 3) + sdk_wallet_steward, numRequests) logger.debug("{} sent {} requests".format(client, numRequests)) for node in txnPoolNodeSet: logger.debug("{} has requests {} with size {}". diff --git a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py index d0cc2635fb..a55a8672da 100644 --- a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py +++ b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py @@ -11,7 +11,7 @@ waitNodeDataEquality from plenum.common.util import randomString from plenum.test.test_node import checkNodesConnected -from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node, sdk_pool_refresh from plenum.test import waits from plenum.common.startable import Mode @@ -53,6 +53,7 @@ def add_new_node(looper, nodes, sdk_pool_handle, sdk_wallet_steward, timeout = waits.expectedPoolCatchupTime(nodeCount=len(nodes)) waitNodeDataEquality(looper, new_node, *nodes[:-1], customTimeout=timeout) + sdk_pool_refresh(looper, sdk_pool_handle) return new_node From ec0912a2630de377278583f5bb818d3bcdbc9ede Mon Sep 17 00:00:00 2001 From: ArtObr Date: Thu, 29 Mar 2018 17:42:44 +0300 Subject: [PATCH 49/73] little fix for test passing Signed-off-by: ArtObr --- plenum/test/node_catchup/test_revert_during_catchup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plenum/test/node_catchup/test_revert_during_catchup.py b/plenum/test/node_catchup/test_revert_during_catchup.py index ef92b5fe17..d1cb43d588 100644 --- a/plenum/test/node_catchup/test_revert_during_catchup.py +++ b/plenum/test/node_catchup/test_revert_during_catchup.py @@ -42,7 +42,7 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, slow_master_replica = slow_node.master_replica commit_delay = 150 - catchup_rep_delay = 15 + catchup_rep_delay = 25 # Delay COMMITs to one node slow_node.nodeIbStasher.delay(cDelay(commit_delay, 0)) From 1f8bbe2a7b484b95804123839e2b8c5d7e0c92e3 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Thu, 29 Mar 2018 17:42:44 +0300 Subject: [PATCH 50/73] little fix for test passing Signed-off-by: ArtObr --- plenum/test/node_catchup/test_revert_during_catchup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plenum/test/node_catchup/test_revert_during_catchup.py b/plenum/test/node_catchup/test_revert_during_catchup.py index ef92b5fe17..d1cb43d588 100644 --- a/plenum/test/node_catchup/test_revert_during_catchup.py +++ b/plenum/test/node_catchup/test_revert_during_catchup.py @@ -42,7 +42,7 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, slow_master_replica = slow_node.master_replica commit_delay = 150 - catchup_rep_delay = 15 + catchup_rep_delay = 25 # Delay COMMITs to one node slow_node.nodeIbStasher.delay(cDelay(commit_delay, 0)) From dfba309fecd5a98d005b5d5e02eecb729724b833 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Thu, 29 Mar 2018 17:50:58 +0300 Subject: [PATCH 51/73] Review change Signed-off-by: ArtObr --- .../plugin/demo_plugin/test_plugin_request_handling.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py index bd38955ea6..1f9c28202c 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_request_handling.py @@ -92,10 +92,8 @@ def test_plugin_dynamic_validation(txn_pool_node_set_post_creation, looper, successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle) -# TODO: probably delete @pytest.fixture(scope="module") -def some_requests(txn_pool_node_set_post_creation, looper, stewardWallet, - steward1, +def some_requests(txn_pool_node_set_post_creation, looper, sdk_wallet_steward, sdk_pool_handle): op = { TXN_TYPE: AUCTION_START, @@ -109,10 +107,11 @@ def some_requests(txn_pool_node_set_post_creation, looper, stewardWallet, } successful_op(looper, op, sdk_wallet_steward, sdk_pool_handle) + _, did = sdk_wallet_steward for node in txn_pool_node_set_post_creation: auctions = node.get_req_handler(AUCTION_LEDGER_ID).auctions assert 'pqr' in auctions - assert auctions['pqr'][stewardWallet.defaultId] == 20 + assert auctions['pqr'][did] == 20 op = { TXN_TYPE: PLACE_BID, @@ -123,7 +122,7 @@ def some_requests(txn_pool_node_set_post_creation, looper, stewardWallet, for node in txn_pool_node_set_post_creation: auctions = node.get_req_handler(AUCTION_LEDGER_ID).auctions assert 'pqr' in auctions - assert auctions['pqr'][stewardWallet.defaultId] == 40 + assert auctions['pqr'][did] == 40 op = { TXN_TYPE: AUCTION_END, From cf1fdbd409ab600cdea1263c3ca7b2ea6ca81041 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Thu, 29 Mar 2018 21:04:12 +0300 Subject: [PATCH 52/73] deletion of send_signed_request Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 2 +- ...st_client_resends_not_confirmed_request.py | 50 ----------- ...st_client_sends_get_request_to_one_node.py | 82 ++++-------------- plenum/test/client/test_protocol_version.py | 85 +++++++++---------- plenum/test/helper.py | 6 -- .../test_order/test_request_ordering_2.py | 4 +- ...xecuted_once_and_without_failing_behind.py | 20 ++--- ...detecting_lag_from_view_change_messages.py | 2 +- scripts/load.py | 45 ---------- 9 files changed, 69 insertions(+), 227 deletions(-) delete mode 100644 plenum/test/client/test_client_resends_not_confirmed_request.py delete mode 100644 scripts/load.py diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index b7f20a9dbf..edceadd2ae 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -12,7 +12,7 @@ from plenum.common.keygen_utils import init_bls_keys from plenum.common.messages.node_messages import Commit, Prepare, PrePrepare from plenum.common.util import get_utc_epoch, randomString, random_from_alphabet, hexToFriendly -from plenum.test.helper import sendRandomRequests, waitForSufficientRepliesForRequests, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_catchup.helper import waitNodeDataEquality, ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.pool_transactions.helper import updateNodeData, sdk_send_update_node, \ diff --git a/plenum/test/client/test_client_resends_not_confirmed_request.py b/plenum/test/client/test_client_resends_not_confirmed_request.py deleted file mode 100644 index b5c09ac2f0..0000000000 --- a/plenum/test/client/test_client_resends_not_confirmed_request.py +++ /dev/null @@ -1,50 +0,0 @@ -import random - -from stp_core.common.log import getlogger -from plenum.test.client.conftest import passThroughReqAcked1 -from plenum.test.helper import send_signed_requests, \ - waitForSufficientRepliesForRequests -from plenum.test.malicious_behaviors_client import \ - genDoesntSendRequestToSomeNodes - -logger = getlogger() - -nodeCount = 4 -clientFault = genDoesntSendRequestToSomeNodes("AlphaC") -reqAcked1 = passThroughReqAcked1 -nodes_with_bls = 0 - - -def test_client_resends_not_confirmed_request(looper, - client1, - wallet1, - txnPoolNodeSet): - """ - Check that client resends request to all nodes if it was previously sent - to one node but reply cannot be verified - """ - client = client1 - wallet = wallet1 - - initial_submit_count = client.spylog.count(client.submitReqs) - initial_resent_count = client.spylog.count(client.resendRequests) - - def sign_and_send(op): - signed = wallet.signOp(op) - return send_signed_requests(client, [signed]) - - buy = {'type': 'buy', 'amount': random.randint(10, 100)} - requests = sign_and_send(buy) - waitForSufficientRepliesForRequests(looper, client, requests=requests) - - buy = {'type': 'get_buy'} - client._read_only_requests.add('get_buy') - requests = sign_and_send(buy) - waitForSufficientRepliesForRequests(looper, client, requests=requests) - - # submitReqs should be called twice: first for but and then got get_buy - assert initial_submit_count + 2 == \ - client.spylog.count(client.submitReqs) - - assert initial_resent_count + 1 == \ - client.spylog.count(client.resendRequests) diff --git a/plenum/test/client/test_client_sends_get_request_to_one_node.py b/plenum/test/client/test_client_sends_get_request_to_one_node.py index 0ea6a1d61e..0a946e86d8 100644 --- a/plenum/test/client/test_client_sends_get_request_to_one_node.py +++ b/plenum/test/client/test_client_sends_get_request_to_one_node.py @@ -1,91 +1,45 @@ import random +import pytest + +from plenum.test.pool_transactions.helper import sdk_build_get_txn_request, sdk_sign_and_send_prepared_request from plenum.test.spy_helpers import getAllArgs from stp_core.common.log import getlogger from stp_core.loop.eventually import eventually from plenum.test.client.conftest import passThroughReqAcked1 -from plenum.test.helper import stopNodes, send_signed_requests +from plenum.test.helper import stopNodes, send_signed_requests, sdk_send_random_and_check, sdk_get_replies, \ + sdk_get_and_check_replies from plenum.test.malicious_behaviors_client import \ genDoesntSendRequestToSomeNodes logger = getlogger() nodeCount = 4 -clientFault = genDoesntSendRequestToSomeNodes("AlphaC") -reqAcked1 = passThroughReqAcked1 - - -def test_client_sends_get_request_to_one_node(looper, - client1, - wallet1, - txnPoolNodeSet): - """ - Check that client sends read only request to one node only - """ - client = client1 - wallet = wallet1 - - def sign_and_send(op): - signed = wallet.signOp(op) - send_signed_requests(client, [signed]) - - logger.info("Send set request") - buy = {'type': 'buy', 'amount': random.randint(10, 100)} - sign_and_send(buy) - - send_args = getAllArgs(client, client.send) - rids = send_args[0]['rids'] - assert len(rids) > 1 - - logger.info("Send get request") - get_buy = {'type': 'get_buy'} - client._read_only_requests.add('get_buy') - sign_and_send(get_buy) - - send_args = getAllArgs(client, client.send) - rids = send_args[0]['rids'] - assert len(rids) == 1 +@pytest.mark.skip(reason='sdk client cannot do that yet') def test_client_can_send_get_request_to_one_node(looper, - client1, - wallet1, + sdk_pool_handle, + sdk_wallet_client, txnPoolNodeSet): """ Check that read only request can be sent without having connection to all nodes """ - client = client1 - wallet = wallet1 + logger.info("Send set request") + req = sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, 1)[0] logger.info("Stopping nodes") nodes_to_stop = list(txnPoolNodeSet)[1:] stopNodes(nodes_to_stop, looper) - def sign_and_send(op): - signed = wallet.signOp(op) - send_signed_requests(client, [signed]) - - def check_client_disconnected(): - assert not client.hasSufficientConnections - - logger.info("Ensure that client has disconnected from the stopped nodes") - # TODO non-default timeout - looper.run(eventually(check_client_disconnected, retryWait=1)) - - initial_submit_count = client.spylog.count(client.submitReqs) - initial_send_count = client.spylog.count(client.send) - - logger.info("Send set request") - buy = {'type': 'buy', 'amount': random.randint(10, 100)} - sign_and_send(buy) - assert initial_submit_count + 1 == client.spylog.count(client.submitReqs) - assert initial_send_count == client.spylog.count(client.send) - logger.info("Send get request") - get_buy = {'type': 'get_buy'} - client._read_only_requests.add('get_buy') - sign_and_send(get_buy) - assert initial_submit_count + 2 == client.spylog.count(client.submitReqs) - assert initial_send_count + 1 == client.spylog.count(client.send) + _, did = sdk_wallet_client + req = sdk_build_get_txn_request(looper, did, req[0]['reqId']) + request = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client, + sdk_pool_handle, req) + replies = sdk_get_and_check_replies(looper, [request]) + pass diff --git a/plenum/test/client/test_protocol_version.py b/plenum/test/client/test_protocol_version.py index da3b89426e..a5656c611f 100644 --- a/plenum/test/client/test_protocol_version.py +++ b/plenum/test/client/test_protocol_version.py @@ -1,9 +1,13 @@ +import json + import pytest from plenum.common.constants import CURRENT_PROTOCOL_VERSION +from plenum.common.exceptions import RequestNackedException from plenum.common.request import Request from plenum.test.helper import waitForSufficientRepliesForRequests, \ send_signed_requests, checkReqNackWithReason, random_request_objects, \ - sign_request_objects, random_requests + sign_request_objects, random_requests, sdk_signed_random_requests, sdk_send_signed_requests, \ + sdk_get_and_check_replies, sdk_random_request_objects, sdk_sign_request_objects from stp_core.loop.eventually import eventually @@ -13,58 +17,49 @@ def request_num(request): def test_request_no_protocol_version(looper, txnPoolNodeSet, - client1, - wallet1, + sdk_pool_handle, + sdk_wallet_client, request_num): - reqs = random_request_objects(request_num, protocol_version=None) - reqs = sign_request_objects(wallet1, reqs) - for req in reqs: - assert req.protocolVersion is None - - send_signed_requests(client1, reqs) - waitForSufficientRepliesForRequests(looper, client1, requests=reqs) - - -def test_version_not_set_by_default(looper, txnPoolNodeSet, - client1, - wallet1, - request_num): - req_dicts = random_requests(request_num) - reqs = [Request(operation=op) for op in req_dicts] - for req in reqs: - assert req.protocolVersion is None - reqs = sign_request_objects(wallet1, reqs) - for req in reqs: - assert req.protocolVersion is None + _, did = sdk_wallet_client + reqs_obj = sdk_random_request_objects(request_num, identifier=did, + protocol_version=None) + for req_obj in reqs_obj: + assert req_obj.protocolVersion == None - send_signed_requests(client1, reqs) - waitForSufficientRepliesForRequests(looper, client1, requests=reqs) + signed_reqs = sdk_sign_request_objects(looper, sdk_wallet_client, reqs_obj) + reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs) + sdk_get_and_check_replies(looper, reqs) def test_request_with_correct_version(looper, - txnPoolNodeSet, client1, - wallet1, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, request_num): - reqs = random_request_objects(request_num, protocol_version=CURRENT_PROTOCOL_VERSION) - reqs = sign_request_objects(wallet1, reqs) - for req in reqs: - assert req.protocolVersion == CURRENT_PROTOCOL_VERSION + _, did = sdk_wallet_client + reqs_obj = sdk_random_request_objects(request_num, identifier=did, + protocol_version=CURRENT_PROTOCOL_VERSION) + for req_obj in reqs_obj: + assert req_obj.protocolVersion == CURRENT_PROTOCOL_VERSION - send_signed_requests(client1, reqs) - waitForSufficientRepliesForRequests(looper, client1, requests=reqs) + signed_reqs = sdk_sign_request_objects(looper, sdk_wallet_client, reqs_obj) + reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs) + sdk_get_and_check_replies(looper, reqs) -def test_request_with_invalid_version(looper, txnPoolNodeSet, - client1, - wallet1, +def test_request_with_invalid_version(looper, + txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, request_num): - reqs = random_request_objects(request_num, protocol_version=-1) - reqs = sign_request_objects(wallet1, reqs) - for req in reqs: - assert req.protocolVersion == -1 + _, did = sdk_wallet_client + reqs_obj = sdk_random_request_objects(request_num, identifier=did, + protocol_version=-1) + for req_obj in reqs_obj: + assert req_obj.protocolVersion == -1 - send_signed_requests(client1, reqs) - for node in txnPoolNodeSet: - looper.run(eventually(checkReqNackWithReason, client1, - 'Unknown protocol version value -1', - node.clientstack.name, retryWait=1)) + signed_reqs = sdk_sign_request_objects(looper, sdk_wallet_client, reqs_obj) + reqs = sdk_send_signed_requests(sdk_pool_handle, signed_reqs) + with pytest.raises(RequestNackedException)as e: + sdk_get_and_check_replies(looper, reqs) + assert 'Unknown protocol version value -1' in e._excinfo[1].args[0] diff --git a/plenum/test/helper.py b/plenum/test/helper.py index c86652cc30..87d303158d 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -270,12 +270,6 @@ def send_signed_requests(client: Client, signed_reqs: Sequence): return client.submitReqs(*signed_reqs)[0] -def sendRandomRequests(wallet: Wallet, client: Client, count: int): - logger.debug('Sending {} random requests'.format(count)) - return send_signed_requests(client, - signed_random_requests(wallet, count)) - - def buildCompletedTxnFromReply(request, reply: Reply) -> Dict: txn = request.operation txn.update(reply) diff --git a/plenum/test/node_request/test_order/test_request_ordering_2.py b/plenum/test/node_request/test_order/test_request_ordering_2.py index 6790f016d2..7e9811b7a3 100644 --- a/plenum/test/node_request/test_order/test_request_ordering_2.py +++ b/plenum/test/node_request/test_order/test_request_ordering_2.py @@ -2,9 +2,7 @@ from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger from plenum.common.messages.node_messages import PrePrepare, Commit -from plenum.test.helper import sendRandomRequests, \ - waitForSufficientRepliesForRequests, checkLedgerEquality, checkAllLedgersEqual, sdk_send_random_and_check, \ - sdk_send_random_requests, sdk_get_replies, sdk_eval_timeout, sdk_get_and_check_replies +from plenum.test.helper import sdk_send_random_requests, sdk_get_and_check_replies from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits diff --git a/plenum/test/test_request_executed_once_and_without_failing_behind.py b/plenum/test/test_request_executed_once_and_without_failing_behind.py index 36c2533f13..6cacc1cd77 100644 --- a/plenum/test/test_request_executed_once_and_without_failing_behind.py +++ b/plenum/test/test_request_executed_once_and_without_failing_behind.py @@ -3,7 +3,7 @@ from plenum.test.helper import \ send_signed_requests, \ waitForSufficientRepliesForRequests, \ - random_requests + random_requests, sdk_send_random_and_check def set_checkpoint_faking(replica): @@ -20,8 +20,8 @@ def send(msg, stat=None): def test_request_executed_once_and_without_failing_behind(tconf, looper, txnPoolNodeSet, - client1, - wallet1): + sdk_pool_handle, + sdk_wallet_client): """ Checks that all requests executed only once and without failing behind in wrote transactions @@ -41,16 +41,12 @@ def test_request_executed_once_and_without_failing_behind(tconf, looper, for replica in node.replicas: set_checkpoint_faking(replica) - requests = [wallet1.signOp(req) - for req in random_requests(number_of_requests)] + replies = sdk_send_random_and_check(looper, txnPoolNodeSet, + sdk_pool_handle, + sdk_wallet_client, + number_of_requests) - for request in requests: - send_signed_requests(client1, [request]) - waitForSufficientRepliesForRequests(looper, - client1, - requests=[request]) - - expected = [request.reqId for request in requests] + expected = [request[0]['reqId'] for request in replies] for node in txnPoolNodeSet: real_ledger_state = [txn[1]['reqId'] for txn in node.getLedger(DOMAIN_LEDGER_ID).getAllTxn() if 'reqId' in txn[1]] diff --git a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py index c4def36f7b..d2734cfcd8 100644 --- a/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py +++ b/plenum/test/view_change/test_node_detecting_lag_from_view_change_messages.py @@ -5,7 +5,7 @@ from plenum.common.util import compare_3PC_keys from plenum.test.delayers import delay_3pc_messages, icDelay, cDelay from plenum.test.helper import send_reqs_batches_and_get_suff_replies, \ - sendRandomRequests, sdk_send_random_requests + sdk_send_random_requests from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.spy_helpers import get_count from plenum.test.test_node import getNonPrimaryReplicas diff --git a/scripts/load.py b/scripts/load.py deleted file mode 100644 index ab87114dc0..0000000000 --- a/scripts/load.py +++ /dev/null @@ -1,45 +0,0 @@ -from time import perf_counter - -from plenum.client.client import Client -from plenum.client.wallet import Wallet -from plenum.test.helper import sendRandomRequests, \ - waitForSufficientRepliesForRequests -from stp_core.network.port_dispenser import genHa -from stp_core.loop.looper import Looper -from plenum.common.signer_simple import SimpleSigner -from stp_core.types import HA -from plenum.common.config_util import getConfig - -numReqs = 10000 -splits = 5 - - -def load(): - port = genHa()[1] - ha = HA('0.0.0.0', port) - name = "hello" - wallet = Wallet(name) - wallet.addIdentifier( - signer=SimpleSigner(seed=b'000000000000000000000000Steward1')) - client = Client(name, ha=ha) - with Looper(debug=getConfig().LOOPER_DEBUG) as looper: - looper.add(client) - print('Will send {} reqs in all'.format(numReqs)) - requests = sendRandomRequests(wallet, client, numReqs) - start = perf_counter() - for i in range(0, numReqs, numReqs // splits): - print('Will wait for {} now'.format(numReqs // splits)) - s = perf_counter() - reqs = requests[i:i + numReqs // splits + 1] - waitForSufficientRepliesForRequests(looper, client, - requests=reqs, - customTimeoutPerReq=3) - print('>>> Got replies for {} requests << in {}'. - format(numReqs // splits, perf_counter() - s)) - end = perf_counter() - print('>>>{}<<<'.format(end - start)) - exit(0) - - -if __name__ == "__main__": - load() From 81dcecc9275e0417b64cd0f366bd8b083cabb541 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 30 Mar 2018 08:00:24 +0300 Subject: [PATCH 53/73] Deletion of send_signed_request Signed-off-by: ArtObr --- plenum/test/batching_3pc/helper.py | 8 -------- .../test_client_sends_get_request_to_one_node.py | 7 +------ plenum/test/client/test_protocol_version.py | 8 +------- plenum/test/helper.py | 4 ---- plenum/test/test_dirty_read.py | 11 +++++------ ...equest_executed_once_and_without_failing_behind.py | 5 +---- 6 files changed, 8 insertions(+), 35 deletions(-) diff --git a/plenum/test/batching_3pc/helper.py b/plenum/test/batching_3pc/helper.py index 4def1d28df..cd67489980 100644 --- a/plenum/test/batching_3pc/helper.py +++ b/plenum/test/batching_3pc/helper.py @@ -6,8 +6,6 @@ from plenum.common.txn_util import reqToTxn from plenum.common.messages.node_messages import ThreePhaseType from plenum.common.util import check_if_all_equal_in_list -from plenum.test.helper import waitForSufficientRepliesForRequests, \ - send_signed_requests def checkNodesHaveSameRoots(nodes, checkUnCommitted=True, @@ -58,12 +56,6 @@ def addRoot(root, collection): assert len(txnRoots) == 1 -def send_and_check(signed_reqs, looper, txnPoolNodeSet, client): - reqs = send_signed_requests(client, signed_reqs) - waitForSufficientRepliesForRequests(looper, client, requests=reqs) - checkNodesHaveSameRoots(txnPoolNodeSet) - - def add_txns_to_ledger_before_order(replica, reqs): added = False origMethod = replica.tryOrder diff --git a/plenum/test/client/test_client_sends_get_request_to_one_node.py b/plenum/test/client/test_client_sends_get_request_to_one_node.py index 0a946e86d8..c31b276d0c 100644 --- a/plenum/test/client/test_client_sends_get_request_to_one_node.py +++ b/plenum/test/client/test_client_sends_get_request_to_one_node.py @@ -3,15 +3,10 @@ import pytest from plenum.test.pool_transactions.helper import sdk_build_get_txn_request, sdk_sign_and_send_prepared_request -from plenum.test.spy_helpers import getAllArgs from stp_core.common.log import getlogger -from stp_core.loop.eventually import eventually -from plenum.test.client.conftest import passThroughReqAcked1 -from plenum.test.helper import stopNodes, send_signed_requests, sdk_send_random_and_check, sdk_get_replies, \ +from plenum.test.helper import stopNodes, sdk_send_random_and_check, \ sdk_get_and_check_replies -from plenum.test.malicious_behaviors_client import \ - genDoesntSendRequestToSomeNodes logger = getlogger() diff --git a/plenum/test/client/test_protocol_version.py b/plenum/test/client/test_protocol_version.py index a5656c611f..ee90d4910a 100644 --- a/plenum/test/client/test_protocol_version.py +++ b/plenum/test/client/test_protocol_version.py @@ -1,14 +1,8 @@ -import json - import pytest from plenum.common.constants import CURRENT_PROTOCOL_VERSION from plenum.common.exceptions import RequestNackedException -from plenum.common.request import Request -from plenum.test.helper import waitForSufficientRepliesForRequests, \ - send_signed_requests, checkReqNackWithReason, random_request_objects, \ - sign_request_objects, random_requests, sdk_signed_random_requests, sdk_send_signed_requests, \ +from plenum.test.helper import sdk_send_signed_requests, \ sdk_get_and_check_replies, sdk_random_request_objects, sdk_sign_request_objects -from stp_core.loop.eventually import eventually @pytest.yield_fixture(scope="function", params=['1', '2']) diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 87d303158d..8ffde7b0fc 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -266,10 +266,6 @@ def signed_random_requests(wallet, count): return sign_requests(wallet, reqs) -def send_signed_requests(client: Client, signed_reqs: Sequence): - return client.submitReqs(*signed_reqs)[0] - - def buildCompletedTxnFromReply(request, reply: Reply) -> Dict: txn = request.operation txn.update(reply) diff --git a/plenum/test/test_dirty_read.py b/plenum/test/test_dirty_read.py index 681988cf2f..72b5e09e50 100644 --- a/plenum/test/test_dirty_read.py +++ b/plenum/test/test_dirty_read.py @@ -1,10 +1,9 @@ -from plenum.common.types import f -from plenum.test.helper import getRepliesFromClientInbox, \ - send_signed_requests, \ - waitForSufficientRepliesForRequests, sdk_send_random_and_check, sdk_get_and_check_replies -from plenum.common.constants import GET_TXN, DATA, TXN_TYPE, DOMAIN_LEDGER_ID +from plenum.test.helper import sdk_send_random_and_check, \ + sdk_get_and_check_replies +from plenum.common.constants import DATA from plenum.common.messages.node_messages import Ordered -from plenum.test.pool_transactions.helper import sdk_build_get_txn_request, sdk_sign_and_send_prepared_request +from plenum.test.pool_transactions.helper import sdk_build_get_txn_request, \ + sdk_sign_and_send_prepared_request from stp_core.common.log import getlogger logger = getlogger() diff --git a/plenum/test/test_request_executed_once_and_without_failing_behind.py b/plenum/test/test_request_executed_once_and_without_failing_behind.py index 6cacc1cd77..8ff934d3d4 100644 --- a/plenum/test/test_request_executed_once_and_without_failing_behind.py +++ b/plenum/test/test_request_executed_once_and_without_failing_behind.py @@ -1,9 +1,6 @@ from plenum.common.constants import DOMAIN_LEDGER_ID from plenum.common.messages.node_messages import Checkpoint -from plenum.test.helper import \ - send_signed_requests, \ - waitForSufficientRepliesForRequests, \ - random_requests, sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check def set_checkpoint_faking(replica): From 0c1603b653293b9daa9fce78f7c5387b6c60177e Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 30 Mar 2018 12:55:29 +0300 Subject: [PATCH 54/73] deletion of waitForSufficientReplies. Part 1. Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 44 ++-- .../bls/test_bls_key_registry_pool_manager.py | 17 +- plenum/test/client/test_client.py | 2 + .../test_faulty_client_with_faulty_node.py | 1 + plenum/test/conftest.py | 56 ++--- plenum/test/helper.py | 79 +------ .../test_msgs_from_slow_instances.py | 3 - plenum/test/instances/test_multiple_commit.py | 3 - .../instances/test_multiple_pre_prepare.py | 3 - .../test/instances/test_multiple_prepare.py | 3 - .../test/instances/test_pre_prepare_digest.py | 3 - plenum/test/instances/test_prepare_digest.py | 3 - .../test/node_catchup/test_discard_view_no.py | 5 +- .../node_catchup/test_req_id_key_error.py | 19 +- plenum/test/node_request/conftest.py | 96 -------- plenum/test/pool_transactions/conftest.py | 41 +--- plenum/test/pool_transactions/helper.py | 217 +++--------------- .../pool_transactions/test_suspend_node.py | 51 ++-- .../test_txn_pool_manager.py | 15 +- plenum/test/primary_selection/conftest.py | 13 +- .../test_add_node_with_f_changed.py | 51 ++-- .../test_recover_after_demoted.py | 35 +-- .../test_new_node_joins_after_view_change.py | 8 +- ...no_instance_change_before_node_is_ready.py | 9 +- 24 files changed, 168 insertions(+), 609 deletions(-) delete mode 100644 plenum/test/node_request/conftest.py diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index edceadd2ae..f1d03f926d 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -7,16 +7,16 @@ from crypto.bls.bls_multi_signature import MultiSignatureValue from state.pruning_state import PruningState from common.serializers.serialization import state_roots_serializer, proof_nodes_serializer -from plenum.common.constants import DOMAIN_LEDGER_ID, ALIAS, BLS_KEY, STATE_PROOF, TXN_TYPE, MULTI_SIGNATURE, \ +from plenum.common.constants import DOMAIN_LEDGER_ID, STATE_PROOF, TXN_TYPE, MULTI_SIGNATURE, \ MULTI_SIGNATURE_PARTICIPANTS, MULTI_SIGNATURE_SIGNATURE, MULTI_SIGNATURE_VALUE from plenum.common.keygen_utils import init_bls_keys from plenum.common.messages.node_messages import Commit, Prepare, PrePrepare -from plenum.common.util import get_utc_epoch, randomString, random_from_alphabet, hexToFriendly +from plenum.common.util import get_utc_epoch, randomString, hexToFriendly from plenum.test.helper import sdk_send_random_and_check -from plenum.test.node_catchup.helper import waitNodeDataEquality, ensureClientConnectedToNodesAndPoolLedgerSame +from plenum.test.node_catchup.helper import waitNodeDataEquality, ensure_all_nodes_have_same_data from plenum.test.node_request.helper import sdk_ensure_pool_functional -from plenum.test.pool_transactions.helper import updateNodeData, sdk_send_update_node, \ - sdk_pool_refresh +from plenum.test.pool_transactions.helper import sdk_send_update_node, \ + sdk_pool_refresh, sdk_add_new_nym from stp_core.common.log import getlogger logger = getlogger() @@ -203,35 +203,13 @@ def create_prepare(req_key, state_root): return Prepare(*params) -def change_bls_key(looper, txnPoolNodeSet, - node, - steward_client, steward_wallet, - add_wrong=False): - new_blspk = init_bls_keys(node.keys_dir, node.name) - - key_in_txn = \ - new_blspk \ - if not add_wrong \ - else ''.join(random_from_alphabet(32, base58.alphabet)) - - node_data = { - ALIAS: node.name, - BLS_KEY: key_in_txn - } - - updateNodeData(looper, steward_client, steward_wallet, node, node_data) - waitNodeDataEquality(looper, node, *txnPoolNodeSet[:-1]) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward_client, - *txnPoolNodeSet) - return new_blspk - - def sdk_change_bls_key(looper, txnPoolNodeSet, node, sdk_pool_handle, sdk_wallet_steward, add_wrong=False, - new_bls=None): + new_bls=None, + use_in_plenum=True): new_blspk = init_bls_keys(node.keys_dir, node.name) key_in_txn = new_bls or new_blspk \ if not add_wrong \ @@ -248,7 +226,13 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, poolSetExceptOne.remove(node) waitNodeDataEquality(looper, node, *poolSetExceptOne) sdk_pool_refresh(looper, sdk_pool_handle) - sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) + if (use_in_plenum): + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + sdk_wallet_steward, sdk_pool_handle) + else: + sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, + alias=randomString(5)) + ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) return new_blspk diff --git a/plenum/test/bls/test_bls_key_registry_pool_manager.py b/plenum/test/bls/test_bls_key_registry_pool_manager.py index d305ccfc81..ae1fb0f83f 100644 --- a/plenum/test/bls/test_bls_key_registry_pool_manager.py +++ b/plenum/test/bls/test_bls_key_registry_pool_manager.py @@ -1,10 +1,9 @@ import base58 import pytest from plenum.bls.bls_key_register_pool_manager import BlsKeyRegisterPoolManager -from plenum.common.constants import NODE, TXN_TYPE, BLS_KEY, DATA, ALIAS +from plenum.common.constants import NODE, TXN_TYPE, BLS_KEY, DATA from plenum.common.util import randomString from plenum.test.bls.helper import sdk_change_bls_key -from plenum.test.pool_transactions.helper import updateNodeData nodeCount = 4 @@ -83,17 +82,3 @@ def test_get_key_for_old_root_keys_changed(bls_key_register_ledger, new_pool_root_hash) assert bls_key assert bls_key == new_bls_key - - -def change_bls_keys(new_bls_key, node, - looper, client, wallet): - node_data = { - ALIAS: node.name, - BLS_KEY: new_bls_key - } - updateNodeData(looper, - client, - wallet, - node, - node_data) - return diff --git a/plenum/test/client/test_client.py b/plenum/test/client/test_client.py index 55e30c6257..e0c17fcfb2 100644 --- a/plenum/test/client/test_client.py +++ b/plenum/test/client/test_client.py @@ -111,6 +111,7 @@ def testRequestFullRoundTrip(replied1, client1): # noinspection PyIncorrectDocstring +pytest.mark.skip(reason='sdk integration') def testEveryNodeRepliesWithNoFaultyNodes(looper, client1, replied1): """ Every node will send a reply to the client when there are no faulty nodes in @@ -127,6 +128,7 @@ def chk(): # noinspection PyIncorrectDocstring +pytest.mark.skip(reason='sdk integration') def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, client1, sent1): """ When a request has already been executed the previously executed reply diff --git a/plenum/test/client/test_faulty_client_with_faulty_node.py b/plenum/test/client/test_faulty_client_with_faulty_node.py index c0cc8e766c..82fc6b8c76 100644 --- a/plenum/test/client/test_faulty_client_with_faulty_node.py +++ b/plenum/test/client/test_faulty_client_with_faulty_node.py @@ -28,6 +28,7 @@ def nodeChangesRequest(txnPoolNodeSet): # noinspection PyIncorrectDocstring,PyUnusedLocal,PyShadowingNames +pytest.mark.skip(reason='sdk integration') def testReplyUnaffectedByFaultyNode(looper, txnPoolNodeSet, nodeChangesRequest, fClient, replied1): """ diff --git a/plenum/test/conftest.py b/plenum/test/conftest.py index d8821244a1..aa487c3e19 100644 --- a/plenum/test/conftest.py +++ b/plenum/test/conftest.py @@ -55,7 +55,8 @@ checkReqAck, checkLastClientReqForNode, waitForSufficientRepliesForRequests, \ waitForViewChange, requestReturnedToNode, randomText, \ mockGetInstalledDistributions, mockImportModule, chk_all_funcs, \ - create_new_test_node + create_new_test_node, sdk_json_to_request_object, sdk_send_random_requests, sdk_get_and_check_replies, \ + sdk_request_object_to_json from plenum.test.node_request.node_request_helper import checkPrePrepared, \ checkPropagated, checkPrepared, checkCommitted from plenum.test.plugin.helper import getPluginPath @@ -469,46 +470,26 @@ def wallet1(clientAndWallet1): @pytest.fixture(scope="module") -def request1(wallet1): - op = randomOperation() - req = wallet1.signOp(op) - return req +def sent1(looper, sdk_pool_handle, + sdk_wallet_client): + request_couple_json = sdk_send_random_requests( + looper, sdk_pool_handle, sdk_wallet_client, 1) + return request_couple_json @pytest.fixture(scope="module") -def sent1(client1, request1): - return client1.submitReqs(request1)[0][0] - - -@pytest.fixture(scope="module") -def reqAcked1(looper, txnPoolNodeSet, client1, sent1, faultyNodes): +def reqAcked1(looper, txnPoolNodeSet, sent1, faultyNodes): numerOfNodes = len(txnPoolNodeSet) + request = sdk_json_to_request_object(sent1[0][0]) + # Wait until request received by all nodes propTimeout = waits.expectedClientToPoolRequestDeliveryTime(numerOfNodes) - coros = [partial(checkLastClientReqForNode, node, sent1) + coros = [partial(checkLastClientReqForNode, node, request) for node in txnPoolNodeSet] - # looper.run(eventuallyAll(*coros, - # totalTimeout=propTimeout, - # acceptableFails=faultyNodes)) chk_all_funcs(looper, coros, acceptable_fails=faultyNodes, timeout=propTimeout) - - # Wait until sufficient number of acks received - coros2 = [ - partial( - checkReqAck, - client1, - node, - sent1.identifier, - sent1.reqId) for node in txnPoolNodeSet] - ackTimeout = waits.expectedReqAckQuorumTime() - # looper.run(eventuallyAll(*coros2, - # totalTimeout=ackTimeout, - # acceptableFails=faultyNodes)) - chk_all_funcs(looper, coros2, acceptable_fails=faultyNodes, - timeout=ackTimeout) - return sent1 + return request @pytest.fixture(scope="module") @@ -551,7 +532,7 @@ def preprepared1(looper, txnPoolNodeSet, propagated1, faultyNodes): @pytest.fixture(scope="module") -def prepared1(looper, txnPoolNodeSet, client1, preprepared1, faultyNodes): +def prepared1(looper, txnPoolNodeSet, preprepared1, faultyNodes): checkPrepared(looper, txnPoolNodeSet, preprepared1, @@ -561,7 +542,7 @@ def prepared1(looper, txnPoolNodeSet, client1, preprepared1, faultyNodes): @pytest.fixture(scope="module") -def committed1(looper, txnPoolNodeSet, client1, prepared1, faultyNodes): +def committed1(looper, txnPoolNodeSet, prepared1, faultyNodes): checkCommitted(looper, txnPoolNodeSet, prepared1, @@ -571,14 +552,17 @@ def committed1(looper, txnPoolNodeSet, client1, prepared1, faultyNodes): @pytest.fixture(scope="module") -def replied1(looper, txnPoolNodeSet, client1, committed1, wallet1, faultyNodes): +def replied1(looper, txnPoolNodeSet, sdk_wallet_client, + committed1, faultyNodes, sent1): numOfNodes = len(txnPoolNodeSet) numOfInstances = getNoInstances(numOfNodes) quorum = numOfInstances * (numOfNodes - faultyNodes) + _, did = sdk_wallet_client + def checkOrderedCount(): resp = [requestReturnedToNode(node, - wallet1.defaultId, + did, committed1.reqId, instId) for node in txnPoolNodeSet for instId in range(numOfInstances)] @@ -589,7 +573,7 @@ def checkOrderedCount(): retryWait=1, timeout=orderingTimeout)) - waitForSufficientRepliesForRequests(looper, client1, requests=[committed1]) + sdk_get_and_check_replies(looper, sent1) return committed1 diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 8ffde7b0fc..2ea3dea417 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -188,55 +188,6 @@ def assertEquality(observed: Any, expected: Any, details=None): "was {}, details: {}".format(observed, expected, details) -def setupNodesAndClient( - looper: Looper, - nodes: Sequence[TestNode], - nodeReg=None, - tmpdir=None): - looper.run(checkNodesConnected(nodes)) - ensureElectionsDone(looper=looper, nodes=nodes) - return setupClient(looper, nodes, nodeReg=nodeReg, tmpdir=tmpdir) - - -def setupClient(looper: Looper, - nodes: Sequence[TestNode] = None, - nodeReg=None, - tmpdir=None, - identifier=None, - verkey=None): - client1, wallet = genTestClient(nodes=nodes, - nodeReg=nodeReg, - tmpdir=tmpdir, - identifier=identifier, - verkey=verkey) - looper.add(client1) - looper.run(client1.ensureConnectedToNodes()) - return client1, wallet - - -def setupClients(count: int, - looper: Looper, - nodes: Sequence[TestNode] = None, - nodeReg=None, - tmpdir=None): - wallets = {} - clients = {} - for i in range(count): - name = "test-wallet-{}".format(i) - wallet = Wallet(name) - idr, _ = wallet.addIdentifier() - verkey = wallet.getVerkey(idr) - client, _ = setupClient(looper, - nodes, - nodeReg, - tmpdir, - identifier=idr, - verkey=verkey) - clients[client.name] = client - wallets[client.name] = wallet - return clients, wallets - - def randomOperation(): return { "type": "buy", @@ -253,19 +204,6 @@ def random_request_objects(count, protocol_version): return [Request(operation=op, protocolVersion=protocol_version) for op in req_dicts] -def sign_request_objects(wallet, reqs: Sequence): - return [wallet.signRequest(req) for req in reqs] - - -def sign_requests(wallet, reqs: Sequence): - return [wallet.signOp(req) for req in reqs] - - -def signed_random_requests(wallet, count): - reqs = random_requests(count) - return sign_requests(wallet, reqs) - - def buildCompletedTxnFromReply(request, reply: Reply) -> Dict: txn = request.operation txn.update(reply) @@ -424,12 +362,6 @@ def check_request_is_not_returned_to_nodes(txnPoolNodeSet, request): inst_id) -def verify_request_not_replied_and_not_ordered(request, looper, client, nodes): - with pytest.raises(AssertionError): - waitForSufficientRepliesForRequests(looper, client, requests=[request]) - check_request_is_not_returned_to_nodes(nodes, request) - - def checkPrePrepareReqSent(replica: TestReplica, req: Request): prePreparesSent = getAllArgs(replica, replica.sendPrePrepare) expectedDigest = TestReplica.batchDigest([req]) @@ -1087,3 +1019,14 @@ def sdk_json_couples_to_request_list(json_couples): for json_couple in json_couples: req_list.append(sdk_json_to_request_object(json_couple[0])) return req_list + + +def sdk_request_object_to_json(request: Request): + json_req = {} + json_req['identifier'] = request.identifier + json_req['reqId'] = request.reqId + json_req['operation'] = request.operation + if request.signature: json_req['signature'] = request.signature + if request.protocolVersion: json_req['protocolVersion'] = \ + request.protocolVersion + return json_req \ No newline at end of file diff --git a/plenum/test/instances/test_msgs_from_slow_instances.py b/plenum/test/instances/test_msgs_from_slow_instances.py index 9cdb67fde0..5f762ed0b9 100644 --- a/plenum/test/instances/test_msgs_from_slow_instances.py +++ b/plenum/test/instances/test_msgs_from_slow_instances.py @@ -7,9 +7,6 @@ from plenum.test.delayers import delayerMsgTuple from plenum.test.test_node import TestNode from plenum.test import waits -from plenum.test.node_request.conftest import committed1, \ - prepared1, preprepared1, propagated1, reqAcked1, \ - sent1, noRetryReq, faultyNodes nodeCount = 4 diff --git a/plenum/test/instances/test_multiple_commit.py b/plenum/test/instances/test_multiple_commit.py index feac6ea9e2..c00d2a9bdd 100644 --- a/plenum/test/instances/test_multiple_commit.py +++ b/plenum/test/instances/test_multiple_commit.py @@ -12,9 +12,6 @@ sendDuplicate3PhaseMsg from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.node_request.conftest import committed1, \ - prepared1, preprepared1, propagated1, reqAcked1, \ - sent1, noRetryReq, faultyNodes @pytest.fixture("module") diff --git a/plenum/test/instances/test_multiple_pre_prepare.py b/plenum/test/instances/test_multiple_pre_prepare.py index b326c054ce..7f0b040573 100644 --- a/plenum/test/instances/test_multiple_pre_prepare.py +++ b/plenum/test/instances/test_multiple_pre_prepare.py @@ -12,9 +12,6 @@ sendDuplicate3PhaseMsg from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.node_request.conftest import committed1, \ - prepared1, preprepared1, propagated1, reqAcked1, \ - sent1, noRetryReq, faultyNodes @pytest.fixture("module") diff --git a/plenum/test/instances/test_multiple_prepare.py b/plenum/test/instances/test_multiple_prepare.py index ae34618a77..f0aba95d84 100644 --- a/plenum/test/instances/test_multiple_prepare.py +++ b/plenum/test/instances/test_multiple_prepare.py @@ -11,9 +11,6 @@ sendDuplicate3PhaseMsg from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.node_request.conftest import committed1, \ - prepared1, preprepared1, propagated1, reqAcked1, \ - sent1, noRetryReq, faultyNodes @pytest.fixture("module") diff --git a/plenum/test/instances/test_pre_prepare_digest.py b/plenum/test/instances/test_pre_prepare_digest.py index cc528504b7..285c9c28e9 100644 --- a/plenum/test/instances/test_pre_prepare_digest.py +++ b/plenum/test/instances/test_pre_prepare_digest.py @@ -12,9 +12,6 @@ send3PhaseMsgWithIncorrectDigest from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.node_request.conftest import committed1, \ - prepared1, preprepared1, propagated1, reqAcked1, \ - sent1, noRetryReq, faultyNodes @pytest.fixture("module") diff --git a/plenum/test/instances/test_prepare_digest.py b/plenum/test/instances/test_prepare_digest.py index b9692a8629..1afc2a6ebc 100644 --- a/plenum/test/instances/test_prepare_digest.py +++ b/plenum/test/instances/test_prepare_digest.py @@ -11,9 +11,6 @@ send3PhaseMsgWithIncorrectDigest from plenum.test.test_node import getNonPrimaryReplicas, getPrimaryReplica from plenum.test import waits -from plenum.test.node_request.conftest import committed1, \ - prepared1, preprepared1, propagated1, reqAcked1, \ - sent1, noRetryReq, faultyNodes @pytest.fixture("module") diff --git a/plenum/test/node_catchup/test_discard_view_no.py b/plenum/test/node_catchup/test_discard_view_no.py index d7625f96e3..03bddeaad2 100644 --- a/plenum/test/node_catchup/test_discard_view_no.py +++ b/plenum/test/node_catchup/test_discard_view_no.py @@ -6,9 +6,8 @@ from plenum.test.helper import checkDiscardMsg from plenum.test.view_change.helper import ensure_view_change from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.pool_transactions.helper import addNewStewardAndNode -from plenum.test.test_node import checkNodesConnected, \ - checkProtocolInstanceSetup, getPrimaryReplica +from plenum.test.test_node import checkProtocolInstanceSetup, \ + getPrimaryReplica from plenum.test import waits whitelist = ['found legacy entry'] # warnings diff --git a/plenum/test/node_catchup/test_req_id_key_error.py b/plenum/test/node_catchup/test_req_id_key_error.py index b1651ae759..1fe2f821c2 100644 --- a/plenum/test/node_catchup/test_req_id_key_error.py +++ b/plenum/test/node_catchup/test_req_id_key_error.py @@ -1,22 +1,25 @@ +import json + from plenum.common.types import f -from plenum.common.txn_util import reqToTxn -from plenum.test.helper import signed_random_requests +from plenum.test.helper import sdk_signed_random_requests from stp_core.common.log import getlogger logger = getlogger() -def test_req_id_key_error(testNode, wallet1): +def test_req_id_key_error(testNode, looper, sdk_wallet_client): # create random transactions count_of_txn = 3 - reqs = signed_random_requests(wallet1, count_of_txn) + reqs = sdk_signed_random_requests(looper, sdk_wallet_client, count_of_txn) txns = [] + # prepare transactions and remove reqId from for i, req in enumerate(reqs): - txnreq = reqToTxn(req) - txnreq[f.SEQ_NO.nm] = i - txnreq.pop(f.REQ_ID.nm) - txns.append(txnreq) + j_req = json.loads(req) + j_req[f.SEQ_NO.nm] = i + j_req.pop(f.REQ_ID.nm) + txns.append(j_req) + node = testNode logger.debug(txns) node.updateSeqNoMap(txns) diff --git a/plenum/test/node_request/conftest.py b/plenum/test/node_request/conftest.py deleted file mode 100644 index 333adc5220..0000000000 --- a/plenum/test/node_request/conftest.py +++ /dev/null @@ -1,96 +0,0 @@ -import pytest - -from functools import partial - -from plenum.test.conftest import getValueFromModule - -from plenum.test import waits -from plenum.test.helper import checkLastClientReqForNode, \ - chk_all_funcs, sdk_json_to_request_object, sdk_send_random_requests -from plenum.test.node_request.node_request_helper import checkPrePrepared, \ - checkPropagated, checkPrepared -from plenum.test.node_request.node_request_helper import checkCommitted - -from plenum.common.util import getNoInstances - - -@pytest.fixture(scope="module") -def committed1(looper, txnPoolNodeSet, prepared1, faultyNodes): - checkCommitted(looper, - txnPoolNodeSet, - prepared1, - range(getNoInstances(len(txnPoolNodeSet))), - faultyNodes) - return prepared1 - - -@pytest.fixture(scope="module") -def prepared1(looper, txnPoolNodeSet, preprepared1, faultyNodes): - checkPrepared(looper, - txnPoolNodeSet, - preprepared1, - range(getNoInstances(len(txnPoolNodeSet))), - faultyNodes) - return preprepared1 - - -@pytest.fixture(scope="module") -def preprepared1(looper, txnPoolNodeSet, propagated1, faultyNodes): - checkPrePrepared(looper, - txnPoolNodeSet, - propagated1, - range(getNoInstances(len(txnPoolNodeSet))), - faultyNodes) - return propagated1 - - -@pytest.fixture(scope="module") -def propagated1(looper, - txnPoolNodeSet, - reqAcked1, - faultyNodes): - checkPropagated(looper, txnPoolNodeSet, reqAcked1, faultyNodes) - return reqAcked1 - - -@pytest.fixture(scope="module") -def reqAcked1(looper, txnPoolNodeSet, sent1, faultyNodes): - numerOfNodes = len(txnPoolNodeSet) - - request = sdk_json_to_request_object(sent1[0][0]) - - # Wait until request received by all nodes - propTimeout = waits.expectedClientToPoolRequestDeliveryTime(numerOfNodes) - coros = [partial(checkLastClientReqForNode, node, request) - for node in txnPoolNodeSet] - chk_all_funcs(looper, coros, acceptable_fails=faultyNodes, - timeout=propTimeout) - return request - - -@pytest.fixture(scope="module") -def sent1(looper, sdk_pool_handle, - sdk_wallet_client): - request_couple_json = sdk_send_random_requests( - looper, sdk_pool_handle, sdk_wallet_client, 1) - return request_couple_json - - -@pytest.fixture(scope="module") -def noRetryReq(tconf, request): - oldRetryAck = tconf.CLIENT_MAX_RETRY_ACK - oldRetryReply = tconf.CLIENT_MAX_RETRY_REPLY - tconf.CLIENT_MAX_RETRY_ACK = 0 - tconf.CLIENT_MAX_RETRY_REPLY = 0 - - def reset(): - tconf.CLIENT_MAX_RETRY_ACK = oldRetryAck - tconf.CLIENT_MAX_RETRY_REPLY = oldRetryReply - - request.addfinalizer(reset) - return tconf - - -@pytest.fixture(scope="module") -def faultyNodes(request): - return getValueFromModule(request, "faultyNodes", 0) diff --git a/plenum/test/pool_transactions/conftest.py b/plenum/test/pool_transactions/conftest.py index 94930f90e5..c1ad33d254 100644 --- a/plenum/test/pool_transactions/conftest.py +++ b/plenum/test/pool_transactions/conftest.py @@ -2,10 +2,8 @@ from plenum.common.util import randomString from plenum.test.test_node import checkNodesConnected -from plenum.test.node_catchup.helper import \ - ensureClientConnectedToNodesAndPoolLedgerSame -from plenum.test.pool_transactions.helper import addNewStewardAndNode, \ - buildPoolClientAndWallet, addNewSteward, sdk_add_new_steward_and_node, sdk_pool_refresh +from plenum.test.pool_transactions.helper import \ + sdk_add_new_steward_and_node, sdk_pool_refresh @pytest.fixture(scope="module") @@ -18,32 +16,6 @@ def tconf(tconf, request): return tconf -@pytest.fixture("module") -def nodeThetaAdded(looper, txnPoolNodeSet, tdir, client_tdir, - tconf, steward1, stewardWallet, allPluginsPath, testNodeClass=None, - testClientClass=None, name=None): - newStewardName = "testClientSteward" + randomString(3) - newNodeName = name or "Theta" - newSteward, newStewardWallet, newNode = addNewStewardAndNode(looper, - steward1, - stewardWallet, - newStewardName, - newNodeName, - tdir, - client_tdir, - tconf, - allPluginsPath, - nodeClass=testNodeClass, - clientClass=testClientClass) - txnPoolNodeSet.append(newNode) - looper.run(checkNodesConnected(txnPoolNodeSet)) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, - *txnPoolNodeSet) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, - *txnPoolNodeSet) - return newSteward, newStewardWallet, newNode - - @pytest.fixture(scope='module') def sdk_node_theta_added(looper, txnPoolNodeSet, @@ -69,12 +41,3 @@ def sdk_node_theta_added(looper, looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) return new_steward_wallet, new_node - - -@pytest.fixture(scope="function") -def newAdHocSteward(looper, client_tdir, steward1, stewardWallet): - newStewardName = "testClientSteward" + randomString(3) - newSteward, newStewardWallet = addNewSteward(looper, client_tdir, steward1, - stewardWallet, - newStewardName) - return newSteward, newStewardWallet diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index d2ae536851..5f234bc2ac 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -1,5 +1,7 @@ import json +from plenum.test.node_request.helper import sdk_ensure_pool_functional + from indy.did import create_and_store_my_did from indy.ledger import build_node_request, build_nym_request, build_get_txn_request from indy.pool import refresh_pool_ledger @@ -53,15 +55,6 @@ def sendAddNewClient(role, name, creatorClient, creatorWallet): return req, wallet -def addNewClient(role, looper, creatorClient: Client, creatorWallet: Wallet, - name: str): - req, wallet = sendAddNewClient(role, name, creatorClient, creatorWallet) - waitForSufficientRepliesForRequests(looper, creatorClient, - requests=[req]) - - return wallet - - def sendAddNewNode(tdir, tconf, newNodeName, stewardClient, stewardWallet, transformOpFunc=None): sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort = \ @@ -113,23 +106,6 @@ def send_new_node_txn(sigseed, sigseed -def addNewNode(looper, stewardClient, stewardWallet, newNodeName, tdir, tconf, - allPluginsPath=None, autoStart=True, nodeClass=TestNode, - transformOpFunc=None, do_post_node_creation: Callable = None): - nodeClass = nodeClass or TestNode - req, nodeIp, nodePort, clientIp, clientPort, sigseed \ - = sendAddNewNode(tdir, tconf, newNodeName, stewardClient, stewardWallet, - transformOpFunc) - waitForSufficientRepliesForRequests(looper, stewardClient, - requests=[req]) - - return create_and_start_new_node(looper, newNodeName, tdir, sigseed, - (nodeIp, nodePort), (clientIp, clientPort), - tconf, autoStart, allPluginsPath, - nodeClass, - do_post_node_creation=do_post_node_creation) - - def start_not_added_node(looper, tdir, tconf, allPluginsPath, newNodeName): @@ -152,32 +128,33 @@ def add_started_node(looper, node_ha, client_ha, txnPoolNodeSet, - client_tdir, - stewardClient, stewardWallet, - sigseed, + sdk_pool_handle, + sdk_wallet_steward, bls_key): ''' Adds already created node to the pool, that is sends NODE txn. Makes sure that node is actually added and connected to all otehr nodes. ''' - newSteward, newStewardWallet = addNewSteward(looper, client_tdir, - stewardClient, stewardWallet, - "Steward" + new_node.name, - clientClass=TestClient) + new_steward_wallet_handle = sdk_add_new_nym(looper, sdk_pool_handle, + sdk_wallet_steward, + "Steward" + new_node.name, + role=STEWARD_STRING) node_name = new_node.name - send_new_node_txn(sigseed, - node_ha[0], - node_ha[1], - client_ha[0], - client_ha[1], - bls_key, - node_name, - newSteward, newStewardWallet) + node_dest = hexToFriendly(new_node.nodestack.verhex) + sdk_send_update_node(looper, new_steward_wallet_handle, + sdk_pool_handle, node_dest, node_name, + node_ha[0], node_ha[1], + client_ha[0], client_ha[1], + services=[VALIDATOR], + bls_key=bls_key) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, *txnPoolNodeSet) + sdk_pool_refresh(looper, sdk_pool_handle) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, + sdk_wallet_steward, + sdk_pool_handle) waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) @@ -225,45 +202,6 @@ def new_node( return node -def addNewSteward(looper, client_tdir, - creatorClient, creatorWallet, stewardName, - clientClass=TestClient): - clientClass = clientClass or TestClient - newStewardWallet = addNewClient(STEWARD, looper, creatorClient, - creatorWallet, stewardName) - newSteward = clientClass(name=stewardName, - nodeReg=None, ha=genHa(), - basedirpath=client_tdir) - - looper.add(newSteward) - looper.run(newSteward.ensureConnectedToNodes()) - return newSteward, newStewardWallet - - -def addNewStewardAndNode(looper, creatorClient, creatorWallet, stewardName, - newNodeName, tdir, client_tdir, tconf, allPluginsPath=None, - autoStart=True, nodeClass=TestNode, - clientClass=TestClient, transformNodeOpFunc=None, - do_post_node_creation: Callable = None): - newSteward, newStewardWallet = addNewSteward(looper, client_tdir, creatorClient, - creatorWallet, stewardName, - clientClass=clientClass) - - newNode = addNewNode( - looper, - newSteward, - newStewardWallet, - newNodeName, - tdir, - tconf, - allPluginsPath, - autoStart=autoStart, - nodeClass=nodeClass, - transformOpFunc=transformNodeOpFunc, - do_post_node_creation=do_post_node_creation) - return newSteward, newStewardWallet, newNode - - def sdk_add_new_steward_and_node(looper, sdk_pool_handle, sdk_wallet_steward, @@ -282,7 +220,7 @@ def sdk_add_new_steward_and_node(looper, sdk_wallet_steward, alias=new_steward_name, role=STEWARD_STRING) - newNode = sdk_add_new_node( + new_node = sdk_add_new_node( looper, sdk_pool_handle, new_steward_wallet_handle, @@ -295,7 +233,7 @@ def sdk_add_new_steward_and_node(looper, transformOpFunc=transformNodeOpFunc, do_post_node_creation=do_post_node_creation, services=services) - return new_steward_wallet_handle, newNode + return new_steward_wallet_handle, new_node def sdk_add_new_nym(looper, sdk_pool_handle, creators_wallet, @@ -408,19 +346,6 @@ def sdk_sign_and_send_prepared_request(looper, sdk_wallet, sdk_pool_handle, stri return request_couple -def sendUpdateNode(stewardClient, stewardWallet, node, node_data): - nodeNym = hexToFriendly(node.nodestack.verhex) - op = { - TXN_TYPE: NODE, - TARGET_NYM: nodeNym, - DATA: node_data, - } - - req = stewardWallet.signOp(op) - stewardClient.submitReqs(req) - return req - - def sdk_send_update_node(looper, sdk_submitter_wallet, sdk_pool_handle, destination, alias, @@ -451,12 +376,6 @@ def sdk_send_update_node(looper, sdk_submitter_wallet, return reply -def updateNodeData(looper, stewardClient, stewardWallet, node, node_data): - req = sendUpdateNode(stewardClient, stewardWallet, node, node_data) - waitForSufficientRepliesForRequests(looper, stewardClient, - requests=[req]) - - def sdk_pool_refresh(looper, sdk_pool_handle): looper.loop.run_until_complete( refresh_pool_ledger(sdk_pool_handle)) @@ -508,31 +427,6 @@ def update_node_data_and_reconnect(looper, txnPoolNodeSet, return restartedNode -def changeNodeKeys(looper, stewardClient, stewardWallet, node, verkey): - nodeNym = hexToFriendly(node.nodestack.verhex) - - op = { - TXN_TYPE: NODE, - TARGET_NYM: nodeNym, - VERKEY: verkey, - DATA: { - ALIAS: node.name - } - } - req = stewardWallet.signOp(op) - stewardClient.submitReqs(req) - - waitForSufficientRepliesForRequests(looper, stewardClient, - requests=[req]) - - node.nodestack.clearLocalRoleKeep() - node.nodestack.clearRemoteRoleKeeps() - node.nodestack.clearAllDir() - node.clientstack.clearLocalRoleKeep() - node.clientstack.clearRemoteRoleKeeps() - node.clientstack.clearAllDir() - - def sdk_change_node_keys(looper, node, sdk_wallet_steward, sdk_pool_handle, verkey): _, steward_did = sdk_wallet_steward node_dest = hexToFriendly(node.nodestack.verhex) @@ -557,37 +451,24 @@ def sdk_change_node_keys(looper, node, sdk_wallet_steward, sdk_pool_handle, verk node.clientstack.clearAllDir() -def suspendNode(looper, stewardClient, stewardWallet, nodeNym, nodeName): - op = { - TXN_TYPE: NODE, - TARGET_NYM: nodeNym, - DATA: { - SERVICES: [], - ALIAS: nodeName - } - } - req = stewardWallet.signOp(op) - stewardClient.submitReqs(req) - - waitForSufficientRepliesForRequests(looper, stewardClient, - requests=[req]) - +def demote_node(looper, steward_wallet, sdk_pool_handle, + node): + node_nym = hexToFriendly(node.nodestack.verhex) + sdk_send_update_node(looper, steward_wallet, + sdk_pool_handle, node_nym, node.name, + None, None, + None, None, + services=[]) -def cancelNodeSuspension(looper, stewardClient, stewardWallet, nodeNym, - nodeName): - op = { - TXN_TYPE: NODE, - TARGET_NYM: nodeNym, - DATA: { - SERVICES: [VALIDATOR], - ALIAS: nodeName - } - } - req = stewardWallet.signOp(op) - stewardClient.submitReqs(req) - waitForSufficientRepliesForRequests(looper, stewardClient, - requests=[req]) +def promote_node(looper, steward_wallet, sdk_pool_handle, + node): + node_nym = hexToFriendly(node.nodestack.verhex) + sdk_send_update_node(looper, steward_wallet, + sdk_pool_handle, node_nym, node.name, + None, None, + None, None, + services=[VALIDATOR]) def buildPoolClientAndWallet(clientData, tempDir, clientClass=None, walletClass=None): @@ -695,30 +576,6 @@ def reconnect_node_and_ensure_connected(looper: Looper, looper.run(checkNodesConnected(poolNodes, customTimeout=timeout)) -def add_2_nodes(looper, existing_nodes, steward, steward_wallet, - tdir, client_tdir, tconf, all_plugins_path, names=None): - assert names is None or (isinstance(names, list) and len(names) == 2) - names = names or ("Zeta", "Eta") - new_nodes = [] - for node_name in names: - new_steward_name = "testClientSteward" + randomString(3) - new_steward, new_steward_wallet, new_node = addNewStewardAndNode(looper, - steward, - steward_wallet, - new_steward_name, - node_name, - tdir, - client_tdir, - tconf, - all_plugins_path) - existing_nodes.append(new_node) - looper.run(checkNodesConnected(existing_nodes)) - waitNodeDataEquality(looper, new_node, *existing_nodes[:-1]) - new_nodes.append(new_node) - - return new_nodes - - def sdk_add_2_nodes(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, tdir, tconf, allPluginsPath): diff --git a/plenum/test/pool_transactions/test_suspend_node.py b/plenum/test/pool_transactions/test_suspend_node.py index 46e76cfe99..dd6e3e77c7 100644 --- a/plenum/test/pool_transactions/test_suspend_node.py +++ b/plenum/test/pool_transactions/test_suspend_node.py @@ -7,19 +7,15 @@ from plenum.test.helper import waitForSufficientRepliesForRequests from plenum.test.node_catchup.helper import \ ensureClientConnectedToNodesAndPoolLedgerSame -from plenum.test.pool_transactions.helper import suspendNode, \ - buildPoolClientAndWallet, cancelNodeSuspension +from plenum.test.pool_transactions.helper import demote_node, \ + buildPoolClientAndWallet, promote_node from plenum.test.test_node import TestNode, checkNodesConnected -def checkNodeNotInNodeReg(nodeOrClient, nodeName): - if isinstance(nodeOrClient, Node): - assert nodeName not in nodeOrClient.nodeReg - assert nodeName not in nodeOrClient.nodestack.connecteds - elif isinstance(nodeOrClient, Client): - clientStackName = nodeName + CLIENT_STACK_SUFFIX - assert clientStackName not in nodeOrClient.nodeReg - assert clientStackName not in nodeOrClient.nodestack.connecteds +def checkNodeNotInNodeReg(node, nodeName): + if isinstance(node, Node): + assert nodeName not in node.nodeReg + assert nodeName not in node.nodestack.connecteds else: raise ValueError("pass a node or client object as first argument") @@ -27,26 +23,25 @@ def checkNodeNotInNodeReg(nodeOrClient, nodeName): @pytest.mark.skip(reason="SOV-383") def testStewardSuspendsNode(looper, txnPoolNodeSet, tdirWithPoolTxns, tconf, - steward1, stewardWallet, - nodeThetaAdded, + sdk_pool_handle, + sdk_wallet_steward, + sdk_node_theta_added, poolTxnStewardData, allPluginsPath): - newSteward, newStewardWallet, newNode = nodeThetaAdded - newNodeNym = hexToFriendly(newNode.nodestack.verhex) - suspendNode(looper, newSteward, newStewardWallet, newNodeNym, newNode.name) + new_steward_wallet, new_node = sdk_node_theta_added + newNodeNym = hexToFriendly(new_node.nodestack.verhex) + demote_node(looper, newSteward, newStewardWallet, newNodeNym, new_node.name) # Check suspended node does not exist in any nodeReg or remotes of # nodes or clients txnPoolNodeSet = txnPoolNodeSet[:-1] for node in txnPoolNodeSet: - looper.run(eventually(checkNodeNotInNodeReg, node, newNode.name)) - for client in (steward1, newSteward): - looper.run(eventually(checkNodeNotInNodeReg, client, newNode.name)) + looper.run(eventually(checkNodeNotInNodeReg, node, new_node.name)) # Check a client can send request and receive replies # req = sendRandomRequest(newStewardWallet, newSteward) - waitForSufficientRepliesForRequests(looper, newSteward, - requests=[req]) + # waitForSufficientRepliesForRequests(looper, newSteward, + # requests=[req]) # Check that a restarted client or node does not connect to the suspended # node @@ -57,10 +52,10 @@ def testStewardSuspendsNode(looper, txnPoolNodeSet, looper.add(steward1) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) - looper.run(eventually(checkNodeNotInNodeReg, steward1, newNode.name)) + looper.run(eventually(checkNodeNotInNodeReg, steward1, new_node.name)) - newNode.stop() - looper.removeProdable(newNode) + new_node.stop() + looper.removeProdable(new_node) # TODO: There is a bug that if a primary node is turned off, it sends # duplicate Pre-Prepare and gets blacklisted. Here is the gist @@ -74,15 +69,15 @@ def testStewardSuspendsNode(looper, txnPoolNodeSet, looper.add(oldNode) txnPoolNodeSet[oldNodeIdx] = oldNode looper.run(checkNodesConnected(txnPoolNodeSet)) - looper.run(eventually(checkNodeNotInNodeReg, oldNode, newNode.name)) + looper.run(eventually(checkNodeNotInNodeReg, oldNode, new_node.name)) # Check that a node whose suspension is revoked can reconnect to other # nodes and clients can also connect to that node - cancelNodeSuspension(looper, newSteward, newStewardWallet, newNodeNym, - newNode.name) - nodeTheta = TestNode(newNode.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, + promote_node(looper, newSteward, newStewardWallet, newNodeNym, + new_node.name) + nodeTheta = TestNode(new_node.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, config=tconf, pluginPaths=allPluginsPath, - ha=newNode.nodestack.ha, cliha=newNode.clientstack.ha) + ha=new_node.nodestack.ha, cliha=new_node.clientstack.ha) looper.add(nodeTheta) txnPoolNodeSet.append(nodeTheta) looper.run(checkNodesConnected(txnPoolNodeSet)) diff --git a/plenum/test/pool_transactions/test_txn_pool_manager.py b/plenum/test/pool_transactions/test_txn_pool_manager.py index a0cba2d1ef..af5b40da63 100644 --- a/plenum/test/pool_transactions/test_txn_pool_manager.py +++ b/plenum/test/pool_transactions/test_txn_pool_manager.py @@ -1,8 +1,9 @@ import pytest + from plenum.common.util import hexToFriendly from plenum.common.constants import TARGET_NYM, TXN_TYPE, NODE, CLIENT_STACK_SUFFIX -from plenum.test.pool_transactions.helper import sdk_send_update_node +from plenum.test.pool_transactions.helper import sdk_send_update_node, demote_node nodeCount = 7 nodes_wth_bls = 0 @@ -34,19 +35,11 @@ def test_get_nym_by_name_demoted(txnPoolNodeSet, pool_node_txns, looper, sdk_wallet_steward, sdk_pool_handle): # sdk_wallet_steward fixture is a steward for [0] node, # so we can do things below: - demote_node(txnPoolNodeSet[0], looper, sdk_wallet_steward, sdk_pool_handle) + demote_node(looper, sdk_wallet_steward, sdk_pool_handle, + txnPoolNodeSet[0]) check_get_nym_by_name(txnPoolNodeSet, pool_node_txns) -def demote_node(node, looper, sdk_steward_wallet, sdk_pool_handle): - node_dest = hexToFriendly(node.nodestack.verhex) - sdk_send_update_node(looper, sdk_steward_wallet, sdk_pool_handle, - node_dest, node.name, - None, None, - None, None, - services=[]) - - def check_get_nym_by_name(txnPoolNodeSet, pool_node_txns): for i in range(nodeCount): node = txnPoolNodeSet[i] diff --git a/plenum/test/primary_selection/conftest.py b/plenum/test/primary_selection/conftest.py index 2b512f051f..2b9e6e4a8f 100644 --- a/plenum/test/primary_selection/conftest.py +++ b/plenum/test/primary_selection/conftest.py @@ -5,18 +5,7 @@ from plenum.test.primary_selection.helper import check_newly_added_nodes, \ getPrimaryNodesIdxs from plenum.test.pool_transactions.helper import buildPoolClientAndWallet -from plenum.test.pool_transactions.conftest import nodeThetaAdded, \ - sdk_node_theta_added - - -@pytest.fixture(scope="module") -def one_node_added(looper, txnPoolNodeSet, nodeThetaAdded): - # New node knows primary same primary as others and has rank greater - # than others - _, _, new_node = nodeThetaAdded - waitNodeDataEquality(looper, new_node, *txnPoolNodeSet[:-1]) - check_newly_added_nodes(looper, txnPoolNodeSet, [new_node]) - return new_node +from plenum.test.pool_transactions.conftest import sdk_node_theta_added @pytest.fixture(scope='module') diff --git a/plenum/test/primary_selection/test_add_node_with_f_changed.py b/plenum/test/primary_selection/test_add_node_with_f_changed.py index 8ebe438813..ac0d4c8013 100644 --- a/plenum/test/primary_selection/test_add_node_with_f_changed.py +++ b/plenum/test/primary_selection/test_add_node_with_f_changed.py @@ -3,7 +3,7 @@ from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.common.util import randomString from plenum.test.test_node import checkNodesConnected -from plenum.test.pool_transactions.helper import addNewStewardAndNode +from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node from plenum.test import waits logger = getlogger() @@ -26,19 +26,19 @@ def tconf(tconf): tconf.RETRY_TIMEOUT_NOT_RESTRICTED = old_timeout_not_restricted -def add_new_node(looper, nodes, steward, steward_wallet, - tdir, client_tdir, tconf, all_plugins_path, name=None): +def add_new_node(looper, nodes, sdk_pool_handle, sdk_wallet_steward, + tdir, tconf, all_plugins_path, name=None): node_name = name or randomString(5) new_steward_name = "testClientSteward" + randomString(3) - new_steward, new_steward_wallet, new_node = addNewStewardAndNode(looper, - steward, - steward_wallet, - new_steward_name, - node_name, - tdir, - client_tdir, - tconf, - all_plugins_path) + new_steward_wallet_handle, new_node = \ + sdk_add_new_steward_and_node(looper, + sdk_pool_handle, + sdk_wallet_steward, + new_steward_name, + node_name, + tdir, + tconf, + all_plugins_path) nodes.append(new_node) looper.run(checkNodesConnected(nodes, customTimeout=60)) timeout = waits.expectedPoolCatchupTime(nodeCount=len(nodes)) @@ -48,46 +48,41 @@ def add_new_node(looper, nodes, steward, steward_wallet, def test_add_node_with_f_changed(looper, txnPoolNodeSet, tdir, tconf, - allPluginsPath, steward1, stewardWallet, - client_tdir, limitTestRunningTime): - + allPluginsPath, sdk_pool_handle, + sdk_wallet_steward, limitTestRunningTime): nodes = txnPoolNodeSet add_new_node(looper, nodes, - steward1, - stewardWallet, + sdk_pool_handle, + sdk_wallet_steward, tdir, - client_tdir, tconf, allPluginsPath, name="Node5") add_new_node(looper, nodes, - steward1, - stewardWallet, + sdk_pool_handle, + sdk_wallet_steward, tdir, - client_tdir, tconf, allPluginsPath, name="Node6") add_new_node(looper, nodes, - steward1, - stewardWallet, + sdk_pool_handle, + sdk_wallet_steward, tdir, - client_tdir, tconf, allPluginsPath, name="Node7") add_new_node(looper, nodes, - steward1, - stewardWallet, + sdk_pool_handle, + sdk_wallet_steward, tdir, - client_tdir, tconf, allPluginsPath, name="Node8") # check that all nodes have equal number of replica assert len(set([n.replicas.num_replicas for n in txnPoolNodeSet])) == 1 - assert txnPoolNodeSet[-1].replicas.num_replicas == txnPoolNodeSet[-1].requiredNumberOfInstances \ No newline at end of file + assert txnPoolNodeSet[-1].replicas.num_replicas == txnPoolNodeSet[-1].requiredNumberOfInstances diff --git a/plenum/test/primary_selection/test_recover_after_demoted.py b/plenum/test/primary_selection/test_recover_after_demoted.py index 3386b99031..9838f60de1 100644 --- a/plenum/test/primary_selection/test_recover_after_demoted.py +++ b/plenum/test/primary_selection/test_recover_after_demoted.py @@ -1,9 +1,6 @@ -from plenum.common.constants import ALIAS, SERVICES - from plenum.test.helper import sdk_send_random_and_check, waitForViewChange from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data -from plenum.test.pool_transactions.helper import updateNodeData, \ - buildPoolClientAndWallet +from plenum.test.pool_transactions.helper import demote_node from plenum.test.view_change.helper import ensure_view_change_by_primary_restart from stp_core.common.log import getlogger @@ -13,25 +10,12 @@ def demote_primary_node(looper, initial_pool_of_nodes, pool_of_nodes, - poolTxnStewardNames, - poolTxnData, - tdirWithClientPoolTxns): + sdk_pool_handle, + sdk_wallet_stewards): demoted_node = [node for node in pool_of_nodes if node.has_master_primary][0] indx = initial_pool_of_nodes.index(demoted_node) - steward_name = poolTxnStewardNames[indx] - stewards_seed = poolTxnData["seeds"][steward_name].encode() - - stewardClient, stewardWallet = buildPoolClientAndWallet( - (steward_name, stewards_seed), tdirWithClientPoolTxns) - looper.add(stewardClient) - looper.run(stewardClient.ensureConnectedToNodes()) - - node_data = { - ALIAS: demoted_node.name, - SERVICES: [] - } - updateNodeData(looper, stewardClient, - stewardWallet, demoted_node, node_data) + demote_node(looper, sdk_wallet_stewards[indx], + sdk_pool_handle, demoted_node) pool_of_nodes = list(set(pool_of_nodes) - {demoted_node}) return pool_of_nodes @@ -41,9 +25,7 @@ def test_restart_primaries_then_demote( looper, txnPoolNodeSet, tconf, tdir, allPluginsPath, sdk_pool_handle, sdk_wallet_steward, - poolTxnStewardNames, - poolTxnData, - tdirWithClientPoolTxns): + sdk_wallet_stewards): """ """ logger.info("1. Restart Node1") @@ -75,9 +57,8 @@ def test_restart_primaries_then_demote( pool_of_nodes = demote_primary_node(looper, txnPoolNodeSet, pool_of_nodes, - poolTxnStewardNames, - poolTxnData, - tdirWithClientPoolTxns) + sdk_pool_handle, + sdk_wallet_stewards) # make sure view changed waitForViewChange(looper, pool_of_nodes, expectedViewNo=3) diff --git a/plenum/test/view_change/test_new_node_joins_after_view_change.py b/plenum/test/view_change/test_new_node_joins_after_view_change.py index 8b8c6f3dbc..2b197980d7 100644 --- a/plenum/test/view_change/test_new_node_joins_after_view_change.py +++ b/plenum/test/view_change/test_new_node_joins_after_view_change.py @@ -10,8 +10,8 @@ from stp_core.loop.eventually import eventually from plenum.test.helper import checkViewNoForNodes, sdk_send_random_and_check -from plenum.test.pool_transactions.conftest import nodeThetaAdded -from plenum.test.primary_selection.conftest import one_node_added +from plenum.test.pool_transactions.conftest import sdk_node_theta_added +from plenum.test.primary_selection.conftest import sdk_one_node_added from stp_core.common.log import getlogger @@ -33,8 +33,8 @@ def all_nodes_view_change( @pytest.fixture(scope='module') def new_node_in_correct_view(all_nodes_view_change, looper, txnPoolNodeSet, - one_node_added, sdk_pool_handle, sdk_wallet_client): - new_node = one_node_added + sdk_one_node_added, sdk_pool_handle, sdk_wallet_client): + new_node = sdk_one_node_added looper.run(eventually(checkViewNoForNodes, txnPoolNodeSet, retryWait=1, timeout=10)) assert len(getAllReturnVals(new_node.view_changer, diff --git a/plenum/test/view_change/test_no_instance_change_before_node_is_ready.py b/plenum/test/view_change/test_no_instance_change_before_node_is_ready.py index b532f6917f..c637dfa7ea 100644 --- a/plenum/test/view_change/test_no_instance_change_before_node_is_ready.py +++ b/plenum/test/view_change/test_no_instance_change_before_node_is_ready.py @@ -18,8 +18,7 @@ def tconf(tconf): def test_no_instance_change_on_primary_disconnection_for_not_ready_node( looper, txnPoolNodeSet, tdir, tconf, - allPluginsPath, steward1, stewardWallet, - client_tdir): + allPluginsPath, sdk_pool_handle, sdk_wallet_steward): """ Test steps: 1. create a new node, but don't add it to the pool (so not send NODE txn), so that the node is not ready. @@ -48,9 +47,9 @@ def test_no_instance_change_on_primary_disconnection_for_not_ready_node( node_ha, client_ha, txnPoolNodeSet, - client_tdir, - steward1, stewardWallet, - sigseed, bls_key) + sdk_pool_handle, + sdk_wallet_steward, + bls_key) # 5. wait for more than VIEW_CHANGE_TIMEOUT (a timeout for initial check for disconnected primary) looper.runFor(tconf.VIEW_CHANGE_TIMEOUT + 2) From db1dee960a87ba751cda05fa8488182f01a7646f Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 2 Apr 2018 07:54:35 +0300 Subject: [PATCH 55/73] little fixes Signed-off-by: ArtObr --- plenum/test/client/test_client.py | 4 ++-- plenum/test/client/test_client_sends_to_less_nodes.py | 3 +++ .../test/propagate/test_propagate_recvd_after_request.py | 2 ++ plenum/test/signing/test_signing.py | 2 ++ plenum/test/storage/test_client_req_rep_store.py | 7 ------- 5 files changed, 9 insertions(+), 9 deletions(-) delete mode 100644 plenum/test/storage/test_client_req_rep_store.py diff --git a/plenum/test/client/test_client.py b/plenum/test/client/test_client.py index e0c17fcfb2..236db2984a 100644 --- a/plenum/test/client/test_client.py +++ b/plenum/test/client/test_client.py @@ -111,7 +111,7 @@ def testRequestFullRoundTrip(replied1, client1): # noinspection PyIncorrectDocstring -pytest.mark.skip(reason='sdk integration') +@pytest.mark.skip(reason='sdk integration') def testEveryNodeRepliesWithNoFaultyNodes(looper, client1, replied1): """ Every node will send a reply to the client when there are no faulty nodes in @@ -128,7 +128,7 @@ def chk(): # noinspection PyIncorrectDocstring -pytest.mark.skip(reason='sdk integration') +@pytest.mark.skip(reason='sdk integration') def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, client1, sent1): """ When a request has already been executed the previously executed reply diff --git a/plenum/test/client/test_client_sends_to_less_nodes.py b/plenum/test/client/test_client_sends_to_less_nodes.py index 5facd46649..43c8a94a51 100644 --- a/plenum/test/client/test_client_sends_to_less_nodes.py +++ b/plenum/test/client/test_client_sends_to_less_nodes.py @@ -1,3 +1,5 @@ +import pytest + from plenum.test.client.conftest import passThroughReqAcked1 from plenum.client.client import Client @@ -10,6 +12,7 @@ # noinspection PyIncorrectDocstring +@pytest.mark.skip(reason='sdk integration') def testReplyWhenRequestSentToLessThanFPlusOneNodes(looper, txnPoolNodeSet, fClient: Client, replied1): """ diff --git a/plenum/test/propagate/test_propagate_recvd_after_request.py b/plenum/test/propagate/test_propagate_recvd_after_request.py index 88559c8c2b..f9a5a56af6 100644 --- a/plenum/test/propagate/test_propagate_recvd_after_request.py +++ b/plenum/test/propagate/test_propagate_recvd_after_request.py @@ -1,6 +1,7 @@ import pytest from plenum.common.constants import PROPAGATE +from plenum.test.helper import sdk_json_to_request_object from stp_core.loop.eventually import eventually from plenum.common.messages.node_messages import Propagate from plenum.test.delayers import delay, msg_rep_delay @@ -23,6 +24,7 @@ def setup(txnPoolNodeSet): def testPropagateRecvdAfterRequest(setup, looper, txnPoolNodeSet, sent1): A, B, C, D = txnPoolNodeSet # type: TestNode + sent1 = sdk_json_to_request_object(sent1[0][0]) def x(): # A should have received a request from the client diff --git a/plenum/test/signing/test_signing.py b/plenum/test/signing/test_signing.py index b06814c321..05f10a2306 100644 --- a/plenum/test/signing/test_signing.py +++ b/plenum/test/signing/test_signing.py @@ -1,5 +1,6 @@ import pytest +from plenum.test.helper import sdk_json_to_request_object from stp_core.loop.eventually import eventually from plenum.common.exceptions import InsufficientCorrectSignatures from stp_core.common.log import getlogger @@ -38,6 +39,7 @@ def testOneNodeAltersAClientRequest(looper, setup, evilAlpha, sent1): + sent1 = sdk_json_to_request_object(sent1[0][0]) checkPropagated(looper, txnPoolNodeSet, sent1, faultyNodes) goodNodes = setup.goodNodes diff --git a/plenum/test/storage/test_client_req_rep_store.py b/plenum/test/storage/test_client_req_rep_store.py deleted file mode 100644 index 3676e9c351..0000000000 --- a/plenum/test/storage/test_client_req_rep_store.py +++ /dev/null @@ -1,7 +0,0 @@ -def testReqAcks(replied1, client1): - reqId = replied1.reqId - identifier = replied1.identifier - assert len(client1.nodeReg) == len(client1.reqRepStore.getAcks(identifier, - reqId)) - assert set(client1.nodeReg.keys()) == \ - set(client1.reqRepStore.getAcks(identifier, reqId)) From 63bb62a535b5a52ca9c77a24afd04c352e88675c Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 2 Apr 2018 16:29:22 +0300 Subject: [PATCH 56/73] more tests Signed-off-by: ArtObr --- plenum/test/cli/helper.py | 48 ----------------- plenum/test/cli/test_client_request.py | 9 ---- plenum/test/cli/test_log_filtering.py | 30 ----------- .../test/storage/test_client_req_rep_store.py | 11 ++++ .../validator_info/test_validator_info.py | 52 +++++++------------ 5 files changed, 29 insertions(+), 121 deletions(-) delete mode 100644 plenum/test/cli/test_client_request.py delete mode 100644 plenum/test/cli/test_log_filtering.py create mode 100644 plenum/test/storage/test_client_req_rep_store.py diff --git a/plenum/test/cli/helper.py b/plenum/test/cli/helper.py index aa94017932..dc363436e9 100644 --- a/plenum/test/cli/helper.py +++ b/plenum/test/cli/helper.py @@ -255,52 +255,6 @@ def createClientAndConnect(cli, nodeNames, clientName): waitClientConnected(cli, nodeNames, clientName) -def checkRequest(cli, operation): - cName = "Joe" - cli.enterCmd("new client {}".format(cName)) - # Let client connect to the nodes - - nodeNames = list(cli.nodes.keys()) - waitClientConnected(cli, nodeNames, cName) - - # Send request to all nodes - - createNewKeyring(cName, cli) - - cli.enterCmd("new key {}".format("testkey1")) - assert 'Key created in wallet {}'.format(cName) in cli.lastCmdOutput - - cli.enterCmd('client {} send {}'.format(cName, operation)) - client = cli.clients[cName] - wallet = cli.wallets[cName] # type: Wallet - # Ensure client gets back the replies - lastReqId = client.reqRepStore.lastReqId - - request = Request(identifier=wallet.defaultId, - reqId=lastReqId, - protocolVersion=CURRENT_PROTOCOL_VERSION) - - waitForSufficientRepliesForRequests(cli.looper, client, - requests=[request]) - - txn, status = client.getReply(wallet.defaultId, lastReqId) - - # Ensure the cli shows appropriate output - cli.enterCmd('client {} show {}'.format(cName, lastReqId)) - printeds = cli.printeds - printedReply = printeds[1] - printedStatus = printeds[0] - # txnTimePattern = "'txnTime', \d+\.*\d*" - # txnIdPattern = "'txnId', '" + txn['txnId'] + "'" - txnTimePattern = "\'txnTime\': \d+\.*\d*" - # DEPR - # txnIdPattern = "\'txnId\': '" + txn['txnId'] + "'" - # assert re.search(txnIdPattern, printedReply['msg']) - assert re.search(txnTimePattern, printedReply['msg']) - assert printedStatus['msg'] == "Status: {}".format(status) - return client, wallet - - def newCLI(looper, basedir, ledger_base_dir, cliClass=TestCli, nodeClass=TestNode, @@ -681,7 +635,6 @@ def restartCliAndAssert(cli, do, expectedRestoredWalletName, assert len(cli._activeWallet.identifiers) == expectedIdentifiers - def _newStewardsAddedByName(cli): cli.enterCmd( "add genesis transaction {nym} for 59d9225473451efffe6b36dbcaefdbf7b1895de62084509a7f5b58bf01d06418 role={role}".format( @@ -705,7 +658,6 @@ def _newStewardsAddedByValue(cli): nym=NYM, role=Roles.STEWARD.name)) - def _newNodesAddedByName(cli): cli.enterCmd( 'add genesis transaction {node} for 59d9225473451efffe6b36dbcaefdbf7b1895de62084509a7f5b58bf01d06418 by 59d9225473451efffe6b36dbcaefdbf7b1895de62084509a7f5b58bf01d06418 with data ' diff --git a/plenum/test/cli/test_client_request.py b/plenum/test/cli/test_client_request.py deleted file mode 100644 index 775cb4a22f..0000000000 --- a/plenum/test/cli/test_client_request.py +++ /dev/null @@ -1,9 +0,0 @@ -from plenum.test.cli.helper import checkRequest - - -def testClientRequest(cli, validNodeNames, createAllNodes): - """ - Test client sending request and checking reply and status - """ - operation = '{"amount": 12, "type": "buy"}' - checkRequest(cli, operation) diff --git a/plenum/test/cli/test_log_filtering.py b/plenum/test/cli/test_log_filtering.py deleted file mode 100644 index 60bcda3762..0000000000 --- a/plenum/test/cli/test_log_filtering.py +++ /dev/null @@ -1,30 +0,0 @@ -import types - -from plenum.common.constants import CURRENT_PROTOCOL_VERSION -from plenum.test.cli.helper import checkRequest -from plenum.test.helper import waitForSufficientRepliesForRequests -from plenum.common.request import Request - - -def testLogFiltering(cli, validNodeNames, createAllNodes): - msg = '{"amount": 20, "type": "buy"}' - client, wallet = checkRequest(cli, msg) - - x = client.handleOneNodeMsg - - def handleOneNodeMsg(self, wrappedMsg, excludeFromCli=None): - return x(wrappedMsg, excludeFromCli=True) - - client.handleOneNodeMsg = types.MethodType(handleOneNodeMsg, client) - client.nodestack.msgHandler = client.handleOneNodeMsg - msg = '{"amount": 30, "type": "buy"}' - cli.enterCmd('client {} send {}'.format(client.name, msg)) - - lastRequestId = client.reqRepStore.lastReqId - request = Request(identifier=wallet.defaultId, - reqId=lastRequestId, - protocolVersion=CURRENT_PROTOCOL_VERSION) - waitForSufficientRepliesForRequests(cli.looper, client, - requests=[request]) - - assert "got msg from node" not in cli.lastCmdOutput diff --git a/plenum/test/storage/test_client_req_rep_store.py b/plenum/test/storage/test_client_req_rep_store.py new file mode 100644 index 0000000000..07ee174606 --- /dev/null +++ b/plenum/test/storage/test_client_req_rep_store.py @@ -0,0 +1,11 @@ +import pytest + + +@pytest.mark.skip(reason='sdk integration') +def testReqAcks(replied1, client1): + reqId = replied1.reqId + identifier = replied1.identifier + assert len(client1.nodeReg) == len(client1.reqRepStore.getAcks(identifier, + reqId)) + assert set(client1.nodeReg.keys()) == \ + set(client1.reqRepStore.getAcks(identifier, reqId)) \ No newline at end of file diff --git a/plenum/test/validator_info/test_validator_info.py b/plenum/test/validator_info/test_validator_info.py index 020d5bc245..34c6142218 100644 --- a/plenum/test/validator_info/test_validator_info.py +++ b/plenum/test/validator_info/test_validator_info.py @@ -5,21 +5,16 @@ import pytest import re -from plenum.common.constants import TXN_TYPE, GET_TXN, DATA, NODE, \ - CURRENT_PROTOCOL_VERSION, DOMAIN_LEDGER_ID -from plenum.common.request import Request -from plenum.common.types import f -from plenum.common.util import getTimeBasedId +from plenum.common.constants import TXN_TYPE, NODE from plenum.server.validator_info_tool import ValidatorNodeInfoTool -from plenum.test import waits -from plenum.test.helper import check_sufficient_replies_received, \ - sdk_send_random_and_check + +from plenum.test.helper import sdk_send_random_and_check # noinspection PyUnresolvedReferences from plenum.test.node_catchup.helper import ensureClientConnectedToNodesAndPoolLedgerSame -from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected +from plenum.test.pool_transactions.helper import disconnect_node_and_ensure_disconnected, sdk_build_get_txn_request, \ + sdk_sign_and_send_prepared_request from plenum.test.test_client import genTestClient from stp_core.common.constants import ZMQ_NETWORK_PROTOCOL -from stp_core.loop.eventually import eventually TEST_NODE_NAME = 'Alpha' INFO_FILENAME = '{}_info.json'.format(TEST_NODE_NAME.lower()) @@ -113,7 +108,7 @@ def test_validator_info_file_metrics_avg_read_field_valid(info, read_txn_and_get_latest_info ): assert info['metrics']['average-per-second']['read-transactions'] == 0 - latest_info = read_txn_and_get_latest_info(GET_TXN) + latest_info = read_txn_and_get_latest_info() assert latest_info['metrics']['average-per-second']['read-transactions'] > 0 @@ -216,29 +211,18 @@ def node(txnPoolNodeSet): @pytest.fixture -def read_txn_and_get_latest_info(txnPoolNodesLooper, patched_dump_info_period, - client_and_wallet, info_path): - client, wallet = client_and_wallet - - def read_wrapped(txn_type): - op = { - TXN_TYPE: txn_type, - f.LEDGER_ID.nm: DOMAIN_LEDGER_ID, - DATA: 1 - } - req = Request(identifier=wallet.defaultId, - operation=op, reqId=getTimeBasedId(), - protocolVersion=CURRENT_PROTOCOL_VERSION) - client.submitReqs(req) - - timeout = waits.expectedTransactionExecutionTime( - len(client.inBox)) - - txnPoolNodesLooper.run( - eventually(check_sufficient_replies_received, - client, req.identifier, req.reqId, - retryWait=1, timeout=timeout)) - txnPoolNodesLooper.runFor(patched_dump_info_period) +def read_txn_and_get_latest_info(looper, sdk_wallet_steward, + sdk_pool_handle, info_path, + patched_dump_info_period): + def read_wrapped(): + _, steward_did = sdk_wallet_steward + request = sdk_build_get_txn_request(looper, steward_did, 1) + sdk_sign_and_send_prepared_request(looper, + sdk_wallet_steward, + sdk_pool_handle, + request) + + looper.runFor(patched_dump_info_period) return load_info(info_path) return read_wrapped From 37d679c97963414dd38bd461a86e8c97cd14cd12 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 2 Apr 2018 19:21:46 +0300 Subject: [PATCH 57/73] more fixes Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 3 +- .../test_faulty_client_with_faulty_node.py | 2 +- plenum/test/conftest.py | 7 +- plenum/test/helper.py | 90 ++++--------------- ...come_active_with_less_than_four_servers.py | 3 +- plenum/test/node_catchup/helper.py | 4 - .../node_catchup/test_catchup_scenarios.py | 3 +- .../test_no_catchup_if_got_from_3pc.py | 4 +- .../test_revert_during_catchup.py | 2 +- plenum/test/node_request/helper.py | 17 ---- ...est_different_ledger_request_interleave.py | 4 +- .../test_node_lacks_finalised_requests.py | 3 +- .../plugin/demo_plugin/test_plugin_basic.py | 31 +++---- plenum/test/pool_transactions/helper.py | 83 ++++------------- ...t_change_ha_persists_post_nodes_restart.py | 2 +- .../test_client_with_pool_txns.py | 4 +- .../test_nodes_data_changed.py | 2 +- .../test_nodes_with_pool_txns.py | 3 +- .../test_start_many_nodes.py | 3 +- .../pool_transactions/test_suspend_node.py | 3 +- .../test_z_node_key_changed.py | 2 +- plenum/test/primary_selection/conftest.py | 16 ---- ...est_add_node_to_pool_with_large_ppseqno.py | 3 +- .../test_primary_selection_pool_txn.py | 2 +- .../test/restart/test_restart_node_4_all.py | 2 +- plenum/test/restart/test_restart_nodes.py | 2 +- .../restart/test_restart_nodes_4_all_wp.py | 2 +- .../test/restart/test_restart_nodes_4_np.py | 2 +- plenum/test/restart/test_restart_nodes_6.py | 2 +- .../test/restart/test_restart_nodes_6_all.py | 2 +- .../restart/test_restart_nodes_6_all_wp.py | 2 +- .../test/restart/test_restart_nodes_6_np.py | 2 +- plenum/test/restart/test_restart_nodes_7.py | 2 +- .../test/restart/test_restart_nodes_7_all.py | 2 +- .../test_queueing_req_from_future_view.py | 1 - .../test_view_change_min_cathup_timeout.py | 3 +- .../view_change/test_view_change_timeout.py | 3 +- 37 files changed, 82 insertions(+), 241 deletions(-) diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index f1d03f926d..273565eb7c 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -12,9 +12,8 @@ from plenum.common.keygen_utils import init_bls_keys from plenum.common.messages.node_messages import Commit, Prepare, PrePrepare from plenum.common.util import get_utc_epoch, randomString, hexToFriendly -from plenum.test.helper import sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check, sdk_ensure_pool_functional from plenum.test.node_catchup.helper import waitNodeDataEquality, ensure_all_nodes_have_same_data -from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.pool_transactions.helper import sdk_send_update_node, \ sdk_pool_refresh, sdk_add_new_nym from stp_core.common.log import getlogger diff --git a/plenum/test/client/test_faulty_client_with_faulty_node.py b/plenum/test/client/test_faulty_client_with_faulty_node.py index 82fc6b8c76..7faf5c88a9 100644 --- a/plenum/test/client/test_faulty_client_with_faulty_node.py +++ b/plenum/test/client/test_faulty_client_with_faulty_node.py @@ -28,7 +28,7 @@ def nodeChangesRequest(txnPoolNodeSet): # noinspection PyIncorrectDocstring,PyUnusedLocal,PyShadowingNames -pytest.mark.skip(reason='sdk integration') +@pytest.mark.skip(reason='sdk integration') def testReplyUnaffectedByFaultyNode(looper, txnPoolNodeSet, nodeChangesRequest, fClient, replied1): """ diff --git a/plenum/test/conftest.py b/plenum/test/conftest.py index aa487c3e19..adbf6186f0 100644 --- a/plenum/test/conftest.py +++ b/plenum/test/conftest.py @@ -51,12 +51,11 @@ from plenum.common.types import PLUGIN_TYPE_STATS_CONSUMER, f from plenum.common.util import getNoInstances from plenum.server.notifier_plugin_manager import PluginManager -from plenum.test.helper import randomOperation, \ - checkReqAck, checkLastClientReqForNode, waitForSufficientRepliesForRequests, \ +from plenum.test.helper import checkLastClientReqForNode, \ waitForViewChange, requestReturnedToNode, randomText, \ mockGetInstalledDistributions, mockImportModule, chk_all_funcs, \ - create_new_test_node, sdk_json_to_request_object, sdk_send_random_requests, sdk_get_and_check_replies, \ - sdk_request_object_to_json + create_new_test_node, sdk_json_to_request_object, sdk_send_random_requests, \ + sdk_get_and_check_replies from plenum.test.node_request.node_request_helper import checkPrePrepared, \ checkPropagated, checkPrepared, checkCommitted from plenum.test.plugin.helper import getPluginPath diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 2ea3dea417..ac1f8fdae2 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -10,7 +10,6 @@ from time import sleep from typing import Tuple, Iterable, Dict, Optional, List, Any, Sequence, Union -import pytest from psutil import Popen import json import asyncio @@ -20,7 +19,6 @@ from ledger.genesis_txn.genesis_txn_file_util import genesis_txn_file from plenum.client.client import Client -from plenum.client.wallet import Wallet from plenum.common.constants import DOMAIN_LEDGER_ID, OP_FIELD_NAME, REPLY, REQACK, REQNACK, REJECT, \ CURRENT_PROTOCOL_VERSION from plenum.common.exceptions import RequestNackedException, RequestRejectedException, CommonSdkIOException, \ @@ -33,11 +31,11 @@ from plenum.server.node import Node from plenum.test import waits from plenum.test.msgs import randomMsg +from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.spy_helpers import getLastClientReqReceivedForNode, getAllArgs, getAllReturnVals, \ getAllMsgReceivedForNode -from plenum.test.test_client import TestClient, genTestClient -from plenum.test.test_node import TestNode, TestReplica, TestNodeSet, \ - checkNodesConnected, ensureElectionsDone, NodeRef, getPrimaryReplica +from plenum.test.test_node import TestNode, TestReplica, \ + getPrimaryReplica from stp_core.common.log import getlogger from stp_core.loop.eventually import eventuallyAll, eventually from stp_core.loop.looper import Looper @@ -75,7 +73,7 @@ def check_sufficient_replies_received(client: Client, "quorum achieved for request {}" .format(full_request_id)) - +# TODO: delete after removal from node def waitForSufficientRepliesForRequests(looper, client, *, # To force usage of names @@ -405,64 +403,6 @@ def checkSufficientCommitReqRecvd(replicas: Iterable[TestReplica], viewNo: int, assert received > minimum -def checkReqAck(client, node, idr, reqId, update: Dict[str, str] = None): - rec = {OP_FIELD_NAME: REQACK, f.REQ_ID.nm: reqId, f.IDENTIFIER.nm: idr} - if update: - rec.update(update) - expected = (rec, node.clientstack.name) - # More than one matching message could be present in the client's inBox - # since client on not receiving request under timeout might have retried - # the request - assert client.inBox.count(expected) > 0 - - -def checkReqNackWithReason(client, reason: str, sender: str): - found = False - for msg, sdr in client.inBox: - if msg[OP_FIELD_NAME] == REQNACK and reason in msg.get( - f.REASON.nm, "") and sdr == sender: - found = True - break - assert found, "there is no Nack with reason: {}".format(reason) - - -def wait_negative_resp(looper, client, reason, sender, timeout, chk_method): - return looper.run(eventually(chk_method, - client, - reason, - sender, - timeout=timeout)) - - -def waitReqNackWithReason(looper, client, reason: str, sender: str): - timeout = waits.expectedReqNAckQuorumTime() - return wait_negative_resp(looper, client, reason, sender, timeout, - checkReqNackWithReason) - - -def checkRejectWithReason(client, reason: str, sender: str): - found = False - for msg, sdr in client.inBox: - if msg[OP_FIELD_NAME] == REJECT and reason in msg.get( - f.REASON.nm, "") and sdr == sender: - found = True - break - assert found - - -def waitRejectWithReason(looper, client, reason: str, sender: str): - timeout = waits.expectedReqRejectQuorumTime() - return wait_negative_resp(looper, client, reason, sender, timeout, - checkRejectWithReason) - - -def ensureRejectsRecvd(looper, nodes, client, reason, timeout=5): - for node in nodes: - looper.run(eventually(checkRejectWithReason, client, reason, - node.clientstack.name, retryWait=1, - timeout=timeout)) - - def checkViewNoForNodes(nodes: Iterable[TestNode], expectedViewNo: int = None): """ Checks if all the given nodes have the expected view no @@ -1021,12 +961,16 @@ def sdk_json_couples_to_request_list(json_couples): return req_list -def sdk_request_object_to_json(request: Request): - json_req = {} - json_req['identifier'] = request.identifier - json_req['reqId'] = request.reqId - json_req['operation'] = request.operation - if request.signature: json_req['signature'] = request.signature - if request.protocolVersion: json_req['protocolVersion'] = \ - request.protocolVersion - return json_req \ No newline at end of file +def sdk_ensure_pool_functional(looper, nodes, sdk_wallet, sdk_pool, + num_reqs=10, num_batches=2): + sdk_send_batches_of_random_and_check(looper, + nodes, + sdk_pool, + sdk_wallet, + num_reqs, + num_batches) + ensure_all_nodes_have_same_data(looper, nodes) + + +def get_node_by_name(txnPoolNodeSet, name): + return next(node for node in txnPoolNodeSet if node.name == name) \ No newline at end of file diff --git a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py index c521c924c0..ce44160994 100644 --- a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py +++ b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py @@ -2,12 +2,11 @@ import pytest -from plenum.test.node_request.helper import get_node_by_name from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger from plenum.common.startable import Status from plenum.test.greek import genNodeNames -from plenum.test.helper import addNodeBack, ordinal +from plenum.test.helper import addNodeBack, ordinal, get_node_by_name from plenum.test.test_node import checkNodesConnected, \ checkNodeRemotes from plenum.test.test_stack import CONNECTED, JOINED_NOT_ALLOWED diff --git a/plenum/test/node_catchup/helper.py b/plenum/test/node_catchup/helper.py index 1ed9b2cf13..c2b1c304b6 100644 --- a/plenum/test/node_catchup/helper.py +++ b/plenum/test/node_catchup/helper.py @@ -3,10 +3,6 @@ from plenum.common.util import check_if_all_equal_in_list from stp_core.common.log import getlogger -from typing import Iterable - -from plenum.common.constants import POOL_LEDGER_ID, DOMAIN_LEDGER_ID, \ - CONFIG_LEDGER_ID from stp_core.loop.eventually import eventually from stp_core.types import HA from plenum.test.helper import checkLedgerEquality, checkStateEquality, \ diff --git a/plenum/test/node_catchup/test_catchup_scenarios.py b/plenum/test/node_catchup/test_catchup_scenarios.py index 761103beab..36bd9e3cbc 100644 --- a/plenum/test/node_catchup/test_catchup_scenarios.py +++ b/plenum/test/node_catchup/test_catchup_scenarios.py @@ -1,11 +1,10 @@ import pytest -from plenum.test.node_request.helper import sdk_ensure_pool_functional from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger from plenum.common.startable import Mode from plenum.test.delayers import cqDelay -from plenum.test.helper import sdk_send_random_requests +from plenum.test.helper import sdk_send_random_requests, sdk_ensure_pool_functional from plenum.test.test_node import checkNodesConnected from plenum.test import waits diff --git a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py index 7cd38b51b3..d47a4c3768 100644 --- a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py +++ b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py @@ -1,10 +1,8 @@ -from plenum.test.node_request.helper import sdk_ensure_pool_functional - from plenum.common.constants import DOMAIN_LEDGER_ID from plenum.common.messages.node_messages import Commit, ConsistencyProof from plenum.test.delayers import cpDelay, cDelay -from plenum.test.helper import send_reqs_batches_and_get_suff_replies +from plenum.test.helper import send_reqs_batches_and_get_suff_replies, sdk_ensure_pool_functional from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data, \ waitNodeDataInequality, waitNodeDataEquality from plenum.test.spy_helpers import getAllReturnVals diff --git a/plenum/test/node_catchup/test_revert_during_catchup.py b/plenum/test/node_catchup/test_revert_during_catchup.py index d1cb43d588..5129d9be51 100644 --- a/plenum/test/node_catchup/test_revert_during_catchup.py +++ b/plenum/test/node_catchup/test_revert_during_catchup.py @@ -42,7 +42,7 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, slow_master_replica = slow_node.master_replica commit_delay = 150 - catchup_rep_delay = 25 + catchup_rep_delay = 35 # Delay COMMITs to one node slow_node.nodeIbStasher.delay(cDelay(commit_delay, 0)) diff --git a/plenum/test/node_request/helper.py b/plenum/test/node_request/helper.py index 997296e2f4..b0e1cfcff2 100644 --- a/plenum/test/node_request/helper.py +++ b/plenum/test/node_request/helper.py @@ -1,24 +1,7 @@ import operator -from plenum.test.helper import sdk_send_batches_of_random_and_check -from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data - - -def get_node_by_name(txnPoolNodeSet, name): - return next(node for node in txnPoolNodeSet if node.name == name) def nodes_by_rank(txnPoolNodeSet): return [t[1] for t in sorted([(node.rank, node) for node in txnPoolNodeSet], key=operator.itemgetter(0))] - - -def sdk_ensure_pool_functional(looper, nodes, sdk_wallet, sdk_pool, - num_reqs=10, num_batches=2): - sdk_send_batches_of_random_and_check(looper, - nodes, - sdk_pool, - sdk_wallet, - num_reqs, - num_batches) - ensure_all_nodes_have_same_data(looper, nodes) diff --git a/plenum/test/node_request/test_different_ledger_request_interleave.py b/plenum/test/node_request/test_different_ledger_request_interleave.py index 4102e26208..a14aee0f74 100644 --- a/plenum/test/node_request/test_different_ledger_request_interleave.py +++ b/plenum/test/node_request/test_different_ledger_request_interleave.py @@ -1,7 +1,5 @@ -from plenum.test.node_request.helper import sdk_ensure_pool_functional - from plenum.test.helper import sdk_send_random_and_check, sdk_send_random_requests, \ - sdk_eval_timeout, sdk_get_and_check_replies + sdk_eval_timeout, sdk_get_and_check_replies, sdk_ensure_pool_functional from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.pool_transactions.helper import sdk_add_new_nym, \ prepare_new_node_data, prepare_node_request, sdk_sign_and_send_prepared_request diff --git a/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py b/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py index 1e9f4263b0..4fc1c876bd 100644 --- a/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py +++ b/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py @@ -2,8 +2,7 @@ from plenum.test.delayers import ppgDelay, req_delay from plenum.test.spy_helpers import get_count, getAllReturnVals from plenum.test.test_node import getNonPrimaryReplicas -from plenum.test.helper import sdk_send_random_and_check -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_send_random_and_check, sdk_ensure_pool_functional @pytest.fixture(scope='function', params=['client_requests', diff --git a/plenum/test/plugin/demo_plugin/test_plugin_basic.py b/plenum/test/plugin/demo_plugin/test_plugin_basic.py index aa36471a65..4585764647 100644 --- a/plenum/test/plugin/demo_plugin/test_plugin_basic.py +++ b/plenum/test/plugin/demo_plugin/test_plugin_basic.py @@ -1,11 +1,14 @@ -from plenum.test.helper import checkReqNackWithReason, sdk_gen_request, \ - sdk_sign_and_submit_req_obj, sdk_get_reply -from plenum.common.constants import CURRENT_PROTOCOL_VERSION, TXN_TYPE, DATA +import pytest + +from plenum.common.exceptions import RequestNackedException +from plenum.test.helper import sdk_gen_request, \ + sdk_sign_and_submit_req_obj, sdk_get_reply, sdk_sign_request_objects, sdk_send_signed_requests, \ + sdk_get_and_check_replies +from plenum.common.constants import TXN_TYPE, DATA from plenum.common.request import Request from plenum.common.util import randomString from plenum.test.plugin.demo_plugin import AUCTION_LEDGER_ID, dummy_field_length from plenum.test.plugin.demo_plugin.constants import GET_BAL -from stp_core.loop.eventually import eventually def test_plugin_setup(txn_pool_node_set_post_creation): @@ -19,8 +22,7 @@ def test_plugin_setup(txn_pool_node_set_post_creation): assert AUCTION_LEDGER_ID in node.states -def test_plugin_client_req_fields(txn_pool_node_set_post_creation, looper, stewardWallet, - steward1, +def test_plugin_client_req_fields(txn_pool_node_set_post_creation, looper, sdk_wallet_steward, sdk_pool_handle): """ Test that plugin's addition of request fields and their validation is @@ -39,12 +41,11 @@ def test_plugin_client_req_fields(txn_pool_node_set_post_creation, looper, stewa sdk_get_reply(looper, req) # Invalid field value results in proper failure - req = Request(operation=op, reqId=Request.gen_req_id(), - protocolVersion=CURRENT_PROTOCOL_VERSION, - identifier=stewardWallet.defaultId, - fix_length_dummy=randomString(dummy_field_length + 1)) - steward1.submitReqs(req) - for node in txn_pool_node_set_post_creation: - looper.run(eventually(checkReqNackWithReason, steward1, - 'should have length', - node.clientstack.name, retryWait=1)) + _, did = sdk_wallet_steward + req = sdk_gen_request(op, identifier=did, fix_length_dummy=randomString(dummy_field_length + 1)) + reqs = sdk_sign_request_objects(looper, sdk_wallet_steward, [req]) + reqs = sdk_send_signed_requests(sdk_pool_handle, reqs) + + with pytest.raises(RequestNackedException) as e: + sdk_get_and_check_replies(looper, reqs) + assert 'should have length' in e._excinfo[1].args[0] diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index 5f234bc2ac..d76af9198f 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -1,7 +1,5 @@ import json -from plenum.test.node_request.helper import sdk_ensure_pool_functional - from indy.did import create_and_store_my_did from indy.ledger import build_node_request, build_nym_request, build_get_txn_request from indy.pool import refresh_pool_ledger @@ -11,22 +9,20 @@ from stp_core.types import HA from typing import Iterable, Union, Callable -from plenum.client.client import Client from plenum.client.wallet import Wallet -from plenum.common.constants import STEWARD, TXN_TYPE, NYM, ROLE, TARGET_NYM, ALIAS, \ +from plenum.common.constants import TXN_TYPE, NYM, ROLE, TARGET_NYM, ALIAS, \ NODE_PORT, CLIENT_IP, NODE_IP, DATA, NODE, CLIENT_PORT, VERKEY, SERVICES, \ - VALIDATOR, BLS_KEY, CLIENT_STACK_SUFFIX, STEWARD_STRING + VALIDATOR, BLS_KEY, STEWARD_STRING from plenum.common.keygen_utils import initNodeKeysForBothStacks from plenum.common.signer_simple import SimpleSigner from plenum.common.signer_did import DidSigner from plenum.common.util import randomString, hexToFriendly -from plenum.test.helper import waitForSufficientRepliesForRequests, \ - sdk_sign_request_objects, sdk_send_signed_requests, \ - sdk_json_to_request_object, sdk_get_and_check_replies +from plenum.test.helper import sdk_sign_request_objects, \ + sdk_send_signed_requests, sdk_json_to_request_object, \ + sdk_get_and_check_replies, sdk_ensure_pool_functional from plenum.test.test_client import TestClient, genTestClient from plenum.test.test_node import TestNode, \ ensure_node_disconnected, checkNodesConnected -from stp_core.loop.eventually import eventually from stp_core.network.port_dispenser import genHa from plenum.common.config_helper import PNodeConfigHelper @@ -49,23 +45,6 @@ def new_client_request(role, name, creatorWallet): return creatorWallet.signOp(op), wallet -def sendAddNewClient(role, name, creatorClient, creatorWallet): - req, wallet = new_client_request(role, name, creatorWallet) - creatorClient.submitReqs(req) - return req, wallet - - -def sendAddNewNode(tdir, tconf, newNodeName, stewardClient, stewardWallet, - transformOpFunc=None): - sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort = \ - prepare_new_node_data(tconf, tdir, newNodeName) - return send_new_node_txn(sigseed, - nodeIp, nodePort, clientIp, clientPort, - bls_key, - newNodeName, stewardClient, stewardWallet, - transformOpFunc) - - def prepare_new_node_data(tconf, tdir, newNodeName): sigseed = randomString(32).encode() @@ -76,36 +55,6 @@ def prepare_new_node_data(tconf, tdir, return sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort -def send_new_node_txn(sigseed, - nodeIp, nodePort, clientIp, clientPort, - bls_key, - newNodeName, stewardClient, stewardWallet, - transformOpFunc=None): - nodeSigner = SimpleSigner(seed=sigseed) - op = { - TXN_TYPE: NODE, - TARGET_NYM: nodeSigner.identifier, - DATA: { - NODE_IP: nodeIp, - NODE_PORT: nodePort, - CLIENT_IP: clientIp, - CLIENT_PORT: clientPort, - ALIAS: newNodeName, - SERVICES: [VALIDATOR, ], - BLS_KEY: bls_key - } - } - if transformOpFunc is not None: - transformOpFunc(op) - - req = stewardWallet.signOp(op) - stewardClient.submitReqs(req) - return req, \ - op[DATA].get(NODE_IP), op[DATA].get(NODE_PORT), \ - op[DATA].get(CLIENT_IP), op[DATA].get(CLIENT_PORT), \ - sigseed - - def start_not_added_node(looper, tdir, tconf, allPluginsPath, newNodeName): @@ -295,6 +244,18 @@ def sdk_add_new_node(looper, do_post_node_creation=do_post_node_creation) +async def prepare_schema_request(wallet, named_seed, alias, role): + wh, submitter_did = wallet + (named_did, named_verkey) = await create_and_store_my_did(wh, + json.dumps({ + 'seed': named_seed, + 'cid': True}) + ) + nym_request = await build_nym_request(submitter_did, named_did, named_verkey, + alias, role) + return nym_request, named_did + + async def prepare_nym_request(wallet, named_seed, alias, role): wh, submitter_did = wallet (named_did, named_verkey) = await create_and_store_my_did(wh, @@ -483,16 +444,6 @@ def buildPoolClientAndWallet(clientData, tempDir, clientClass=None, walletClass= return client, w -def new_client(looper, poolTxnClientData, txnPoolNodeSet, client_tdir): - client, wallet = buildPoolClientAndWallet(poolTxnClientData, - client_tdir) - looper.add(client) - looper.run(client.ensureConnectedToNodes()) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, client, - *txnPoolNodeSet) - return client, wallet - - def disconnectPoolNode(poolNodes: Iterable, disconnect: Union[str, TestNode], stopNode=True): diff --git a/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py b/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py index 1edd3cfe81..09c1c41f0e 100644 --- a/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py +++ b/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py @@ -1,7 +1,7 @@ from plenum.common.util import hexToFriendly, randomString from stp_core.common.log import getlogger from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.test.pool_transactions.helper import sdk_send_update_node, sdk_pool_refresh, \ sdk_add_new_steward_and_node from plenum.test.test_node import TestNode, checkNodesConnected diff --git a/plenum/test/pool_transactions/test_client_with_pool_txns.py b/plenum/test/pool_transactions/test_client_with_pool_txns.py index 241e8fc526..bb99c06a29 100644 --- a/plenum/test/pool_transactions/test_client_with_pool_txns.py +++ b/plenum/test/pool_transactions/test_client_with_pool_txns.py @@ -1,10 +1,8 @@ -from plenum.test.node_request.helper import sdk_ensure_pool_functional - from plenum.test.pool_transactions.helper import sdk_pool_refresh from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger from plenum.test import waits -from plenum.test.helper import sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check, sdk_ensure_pool_functional from plenum.test.test_node import checkNodesConnected, TestNode, \ ensureElectionsDone from plenum.common.config_helper import PNodeConfigHelper diff --git a/plenum/test/pool_transactions/test_nodes_data_changed.py b/plenum/test/pool_transactions/test_nodes_data_changed.py index 9687e96ab4..be6e696ee9 100644 --- a/plenum/test/pool_transactions/test_nodes_data_changed.py +++ b/plenum/test/pool_transactions/test_nodes_data_changed.py @@ -1,7 +1,7 @@ import pytest from plenum.common.exceptions import RequestRejectedException -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.constants import CLIENT_STACK_SUFFIX from plenum.common.util import randomString, hexToFriendly diff --git a/plenum/test/pool_transactions/test_nodes_with_pool_txns.py b/plenum/test/pool_transactions/test_nodes_with_pool_txns.py index d58afac793..c89d7b2695 100644 --- a/plenum/test/pool_transactions/test_nodes_with_pool_txns.py +++ b/plenum/test/pool_transactions/test_nodes_with_pool_txns.py @@ -6,7 +6,6 @@ from plenum.common.exceptions import RequestRejectedException, \ RequestNackedException -from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.constants import DATA, TARGET_NYM, \ NODE_IP, NODE_PORT, CLIENT_IP, CLIENT_PORT, STEWARD_STRING @@ -14,7 +13,7 @@ from plenum.common.util import getMaxFailures, randomString from plenum.test import waits from plenum.test.helper import sdk_send_random_and_check, \ - sdk_get_and_check_replies + sdk_get_and_check_replies, sdk_ensure_pool_functional from plenum.test.pool_transactions.helper import sdk_add_new_node, \ sdk_add_2_nodes, sdk_pool_refresh, sdk_add_new_nym, prepare_new_node_data, \ prepare_node_request, sdk_sign_and_send_prepared_request diff --git a/plenum/test/pool_transactions/test_start_many_nodes.py b/plenum/test/pool_transactions/test_start_many_nodes.py index 2286bffd50..ecbfa76168 100644 --- a/plenum/test/pool_transactions/test_start_many_nodes.py +++ b/plenum/test/pool_transactions/test_start_many_nodes.py @@ -1,9 +1,8 @@ # Start with 8 nodes -from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.constants import NYM, ROLE, STEWARD from plenum.common.constants import TXN_TYPE -from plenum.test.helper import assertEquality +from plenum.test.helper import assertEquality, sdk_ensure_pool_functional nodeCount = 8 diff --git a/plenum/test/pool_transactions/test_suspend_node.py b/plenum/test/pool_transactions/test_suspend_node.py index dd6e3e77c7..bdef22aebc 100644 --- a/plenum/test/pool_transactions/test_suspend_node.py +++ b/plenum/test/pool_transactions/test_suspend_node.py @@ -47,8 +47,7 @@ def testStewardSuspendsNode(looper, txnPoolNodeSet, # node steward1.stop() looper.removeProdable(steward1) - steward1, stewardWallet = buildPoolClientAndWallet(poolTxnStewardData, - tdirWithPoolTxns) + steward1, stewardWallet looper.add(steward1) ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, *txnPoolNodeSet) diff --git a/plenum/test/pool_transactions/test_z_node_key_changed.py b/plenum/test/pool_transactions/test_z_node_key_changed.py index d95ca1c779..827cab52f4 100644 --- a/plenum/test/pool_transactions/test_z_node_key_changed.py +++ b/plenum/test/pool_transactions/test_z_node_key_changed.py @@ -2,7 +2,7 @@ import base58 import types -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common import stack_manager from plenum.common.keygen_utils import initNodeKeysForBothStacks, \ diff --git a/plenum/test/primary_selection/conftest.py b/plenum/test/primary_selection/conftest.py index 2b9e6e4a8f..fb2d2efa66 100644 --- a/plenum/test/primary_selection/conftest.py +++ b/plenum/test/primary_selection/conftest.py @@ -24,22 +24,6 @@ def txnPoolMasterNodes(txnPoolNodeSet): return txnPoolNodeSet[primariesIdxs[0]], txnPoolNodeSet[primariesIdxs[1]] -@pytest.fixture(scope="module") -def stewardAndWalletForMasterNode(looper, poolTxnData, poolTxnStewardNames, - tdirWithClientPoolTxns, txnPoolNodeSet, txnPoolMasterNodes): - primariesIdxs = getPrimaryNodesIdxs(txnPoolNodeSet) - master_node = txnPoolMasterNodes[0] - stewardName = poolTxnStewardNames[primariesIdxs[0]] - stewardsSeed = poolTxnData["seeds"][stewardName].encode() - - stewardClient, stewardWallet = buildPoolClientAndWallet( - (stewardName, stewardsSeed), tdirWithClientPoolTxns) - looper.add(stewardClient) - looper.run(stewardClient.ensureConnectedToNodes()) - - return stewardClient, stewardWallet - - @pytest.fixture(scope="module") def checkpoint_size(tconf, request): oldChkFreq = tconf.CHK_FREQ diff --git a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py index 854bb9c644..b0d16a04dc 100644 --- a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py +++ b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py @@ -1,8 +1,7 @@ import pytest -from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.util import randomString -from plenum.test.helper import sdk_send_random_and_check +from plenum.test.helper import sdk_send_random_and_check, sdk_ensure_pool_functional from plenum.test.view_change.helper import ensure_several_view_change from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected diff --git a/plenum/test/primary_selection/test_primary_selection_pool_txn.py b/plenum/test/primary_selection/test_primary_selection_pool_txn.py index 5bd4c42205..74f97e9a58 100644 --- a/plenum/test/primary_selection/test_primary_selection_pool_txn.py +++ b/plenum/test/primary_selection/test_primary_selection_pool_txn.py @@ -1,5 +1,5 @@ import pytest -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.test.pool_transactions.helper import sdk_add_2_nodes from plenum.test.primary_selection.helper import check_newly_added_nodes diff --git a/plenum/test/restart/test_restart_node_4_all.py b/plenum/test/restart/test_restart_node_4_all.py index 6f39800dad..fb1af739ca 100644 --- a/plenum/test/restart/test_restart_node_4_all.py +++ b/plenum/test/restart/test_restart_node_4_all.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes.py b/plenum/test/restart/test_restart_nodes.py index 8816bb3255..cac91a004a 100644 --- a/plenum/test/restart/test_restart_nodes.py +++ b/plenum/test/restart/test_restart_nodes.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode diff --git a/plenum/test/restart/test_restart_nodes_4_all_wp.py b/plenum/test/restart/test_restart_nodes_4_all_wp.py index 8576f90e95..3880c9101f 100644 --- a/plenum/test/restart/test_restart_nodes_4_all_wp.py +++ b/plenum/test/restart/test_restart_nodes_4_all_wp.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_4_np.py b/plenum/test/restart/test_restart_nodes_4_np.py index b0a6ab713f..9c980dd743 100644 --- a/plenum/test/restart/test_restart_nodes_4_np.py +++ b/plenum/test/restart/test_restart_nodes_4_np.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_6.py b/plenum/test/restart/test_restart_nodes_6.py index dd21658233..9193d6a942 100644 --- a/plenum/test/restart/test_restart_nodes_6.py +++ b/plenum/test/restart/test_restart_nodes_6.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_6_all.py b/plenum/test/restart/test_restart_nodes_6_all.py index 5496e7c9b4..cb798b7ccd 100644 --- a/plenum/test/restart/test_restart_nodes_6_all.py +++ b/plenum/test/restart/test_restart_nodes_6_all.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_6_all_wp.py b/plenum/test/restart/test_restart_nodes_6_all_wp.py index 5aebce9826..58355850b2 100644 --- a/plenum/test/restart/test_restart_nodes_6_all_wp.py +++ b/plenum/test/restart/test_restart_nodes_6_all_wp.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_6_np.py b/plenum/test/restart/test_restart_nodes_6_np.py index 873ed9ef1a..213ac392ec 100644 --- a/plenum/test/restart/test_restart_nodes_6_np.py +++ b/plenum/test/restart/test_restart_nodes_6_np.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_7.py b/plenum/test/restart/test_restart_nodes_7.py index 35b9ef35a8..9547ea4f6b 100644 --- a/plenum/test/restart/test_restart_nodes_7.py +++ b/plenum/test/restart/test_restart_nodes_7.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_7_all.py b/plenum/test/restart/test_restart_nodes_7_all.py index d11968e92a..f8176871f8 100644 --- a/plenum/test/restart/test_restart_nodes_7_all.py +++ b/plenum/test/restart/test_restart_nodes_7_all.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/view_change/test_queueing_req_from_future_view.py b/plenum/test/view_change/test_queueing_req_from_future_view.py index 33b042992e..b879264766 100644 --- a/plenum/test/view_change/test_queueing_req_from_future_view.py +++ b/plenum/test/view_change/test_queueing_req_from_future_view.py @@ -50,7 +50,6 @@ def chk_fut_view(view_no, is_empty): [lagging_node]) # send more requests that will be queued for the lagged node - # sendReqsToNodesAndVerifySuffReplies(looper, wallet1, client1, 3) reqs = sdk_send_random_requests(looper, sdk_pool_handle, sdk_wallet_client, 5) l = looper.run(eventually(chk_fut_view, old_view_no + 1, False, diff --git a/plenum/test/view_change/test_view_change_min_cathup_timeout.py b/plenum/test/view_change/test_view_change_min_cathup_timeout.py index cfeea99227..d7bf1ae700 100644 --- a/plenum/test/view_change/test_view_change_min_cathup_timeout.py +++ b/plenum/test/view_change/test_view_change_min_cathup_timeout.py @@ -1,9 +1,8 @@ import types import pytest -from plenum.test.node_request.helper import sdk_ensure_pool_functional -from plenum.test.helper import waitForViewChange, sdk_send_random_and_check +from plenum.test.helper import waitForViewChange, sdk_send_random_and_check, sdk_ensure_pool_functional from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.test_node import ensureElectionsDone from stp_core.loop.exceptions import EventuallyTimeoutException diff --git a/plenum/test/view_change/test_view_change_timeout.py b/plenum/test/view_change/test_view_change_timeout.py index c2197d0802..a0bcd0b494 100644 --- a/plenum/test/view_change/test_view_change_timeout.py +++ b/plenum/test/view_change/test_view_change_timeout.py @@ -1,8 +1,7 @@ import pytest -from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.delayers import reset_delays_and_process_delayeds, vcd_delay -from plenum.test.helper import waitForViewChange, stopNodes +from plenum.test.helper import waitForViewChange, stopNodes, sdk_ensure_pool_functional from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.spy_helpers import get_count, getAllReturnVals from plenum.test.test_node import get_master_primary_node, \ From e9c70d8d7a2815a1f3f6609de209f4311614d695 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 2 Apr 2018 19:41:10 +0300 Subject: [PATCH 58/73] little fix Signed-off-by: ArtObr --- plenum/test/bls/helper.py | 3 ++- plenum/test/helper.py | 19 ++----------------- ...come_active_with_less_than_four_servers.py | 3 ++- .../node_catchup/test_catchup_scenarios.py | 3 ++- .../test_no_catchup_if_got_from_3pc.py | 3 ++- plenum/test/node_request/helper.py | 18 ++++++++++++++++++ ...est_different_ledger_request_interleave.py | 3 ++- .../test_node_lacks_finalised_requests.py | 3 ++- plenum/test/pool_transactions/helper.py | 3 ++- ...t_change_ha_persists_post_nodes_restart.py | 2 +- .../test_client_with_pool_txns.py | 3 ++- .../test_nodes_data_changed.py | 2 +- .../test_nodes_with_pool_txns.py | 3 ++- .../test_start_many_nodes.py | 3 ++- .../test_z_node_key_changed.py | 2 +- ...est_add_node_to_pool_with_large_ppseqno.py | 3 ++- .../test_primary_selection_pool_txn.py | 2 +- .../test/restart/test_restart_node_4_all.py | 2 +- plenum/test/restart/test_restart_nodes.py | 2 +- .../restart/test_restart_nodes_4_all_wp.py | 2 +- .../test/restart/test_restart_nodes_4_np.py | 2 +- plenum/test/restart/test_restart_nodes_6.py | 2 +- .../test/restart/test_restart_nodes_6_all.py | 2 +- .../restart/test_restart_nodes_6_all_wp.py | 2 +- .../test/restart/test_restart_nodes_6_np.py | 2 +- plenum/test/restart/test_restart_nodes_7.py | 2 +- .../test/restart/test_restart_nodes_7_all.py | 2 +- .../test_view_change_min_cathup_timeout.py | 3 ++- .../view_change/test_view_change_timeout.py | 3 ++- 29 files changed, 60 insertions(+), 44 deletions(-) diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 273565eb7c..1d91e356dc 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -12,7 +12,8 @@ from plenum.common.keygen_utils import init_bls_keys from plenum.common.messages.node_messages import Commit, Prepare, PrePrepare from plenum.common.util import get_utc_epoch, randomString, hexToFriendly -from plenum.test.helper import sdk_send_random_and_check, sdk_ensure_pool_functional +from plenum.test.helper import sdk_send_random_and_check +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.node_catchup.helper import waitNodeDataEquality, ensure_all_nodes_have_same_data from plenum.test.pool_transactions.helper import sdk_send_update_node, \ sdk_pool_refresh, sdk_add_new_nym diff --git a/plenum/test/helper.py b/plenum/test/helper.py index ac1f8fdae2..204cb11fbe 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -19,7 +19,7 @@ from ledger.genesis_txn.genesis_txn_file_util import genesis_txn_file from plenum.client.client import Client -from plenum.common.constants import DOMAIN_LEDGER_ID, OP_FIELD_NAME, REPLY, REQACK, REQNACK, REJECT, \ +from plenum.common.constants import DOMAIN_LEDGER_ID, OP_FIELD_NAME, REPLY, REQNACK, REJECT, \ CURRENT_PROTOCOL_VERSION from plenum.common.exceptions import RequestNackedException, RequestRejectedException, CommonSdkIOException, \ PoolLedgerTimeoutException @@ -31,7 +31,6 @@ from plenum.server.node import Node from plenum.test import waits from plenum.test.msgs import randomMsg -from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.spy_helpers import getLastClientReqReceivedForNode, getAllArgs, getAllReturnVals, \ getAllMsgReceivedForNode from plenum.test.test_node import TestNode, TestReplica, \ @@ -73,6 +72,7 @@ def check_sufficient_replies_received(client: Client, "quorum achieved for request {}" .format(full_request_id)) + # TODO: delete after removal from node def waitForSufficientRepliesForRequests(looper, client, @@ -959,18 +959,3 @@ def sdk_json_couples_to_request_list(json_couples): for json_couple in json_couples: req_list.append(sdk_json_to_request_object(json_couple[0])) return req_list - - -def sdk_ensure_pool_functional(looper, nodes, sdk_wallet, sdk_pool, - num_reqs=10, num_batches=2): - sdk_send_batches_of_random_and_check(looper, - nodes, - sdk_pool, - sdk_wallet, - num_reqs, - num_batches) - ensure_all_nodes_have_same_data(looper, nodes) - - -def get_node_by_name(txnPoolNodeSet, name): - return next(node for node in txnPoolNodeSet if node.name == name) \ No newline at end of file diff --git a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py index ce44160994..d2a3127895 100644 --- a/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py +++ b/plenum/test/instances/test_instance_cannot_become_active_with_less_than_four_servers.py @@ -6,7 +6,8 @@ from stp_core.common.log import getlogger from plenum.common.startable import Status from plenum.test.greek import genNodeNames -from plenum.test.helper import addNodeBack, ordinal, get_node_by_name +from plenum.test.helper import addNodeBack, ordinal +from plenum.test.node_request.helper import get_node_by_name from plenum.test.test_node import checkNodesConnected, \ checkNodeRemotes from plenum.test.test_stack import CONNECTED, JOINED_NOT_ALLOWED diff --git a/plenum/test/node_catchup/test_catchup_scenarios.py b/plenum/test/node_catchup/test_catchup_scenarios.py index 36bd9e3cbc..48b316fd88 100644 --- a/plenum/test/node_catchup/test_catchup_scenarios.py +++ b/plenum/test/node_catchup/test_catchup_scenarios.py @@ -4,7 +4,8 @@ from stp_core.common.log import getlogger from plenum.common.startable import Mode from plenum.test.delayers import cqDelay -from plenum.test.helper import sdk_send_random_requests, sdk_ensure_pool_functional +from plenum.test.helper import sdk_send_random_requests +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.test_node import checkNodesConnected from plenum.test import waits diff --git a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py index d47a4c3768..ab5cb937a9 100644 --- a/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py +++ b/plenum/test/node_catchup/test_no_catchup_if_got_from_3pc.py @@ -2,7 +2,8 @@ from plenum.common.messages.node_messages import Commit, ConsistencyProof from plenum.test.delayers import cpDelay, cDelay -from plenum.test.helper import send_reqs_batches_and_get_suff_replies, sdk_ensure_pool_functional +from plenum.test.helper import send_reqs_batches_and_get_suff_replies +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data, \ waitNodeDataInequality, waitNodeDataEquality from plenum.test.spy_helpers import getAllReturnVals diff --git a/plenum/test/node_request/helper.py b/plenum/test/node_request/helper.py index b0e1cfcff2..e20ce1a555 100644 --- a/plenum/test/node_request/helper.py +++ b/plenum/test/node_request/helper.py @@ -1,7 +1,25 @@ import operator +from plenum.test.helper import sdk_send_batches_of_random_and_check +from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data + def nodes_by_rank(txnPoolNodeSet): return [t[1] for t in sorted([(node.rank, node) for node in txnPoolNodeSet], key=operator.itemgetter(0))] + + +def sdk_ensure_pool_functional(looper, nodes, sdk_wallet, sdk_pool, + num_reqs=10, num_batches=2): + sdk_send_batches_of_random_and_check(looper, + nodes, + sdk_pool, + sdk_wallet, + num_reqs, + num_batches) + ensure_all_nodes_have_same_data(looper, nodes) + + +def get_node_by_name(txnPoolNodeSet, name): + return next(node for node in txnPoolNodeSet if node.name == name) diff --git a/plenum/test/node_request/test_different_ledger_request_interleave.py b/plenum/test/node_request/test_different_ledger_request_interleave.py index a14aee0f74..57bc5f83d0 100644 --- a/plenum/test/node_request/test_different_ledger_request_interleave.py +++ b/plenum/test/node_request/test_different_ledger_request_interleave.py @@ -1,5 +1,6 @@ from plenum.test.helper import sdk_send_random_and_check, sdk_send_random_requests, \ - sdk_eval_timeout, sdk_get_and_check_replies, sdk_ensure_pool_functional + sdk_eval_timeout, sdk_get_and_check_replies +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.pool_transactions.helper import sdk_add_new_nym, \ prepare_new_node_data, prepare_node_request, sdk_sign_and_send_prepared_request diff --git a/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py b/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py index 4fc1c876bd..1e9f4263b0 100644 --- a/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py +++ b/plenum/test/node_request/test_propagate/test_node_lacks_finalised_requests.py @@ -2,7 +2,8 @@ from plenum.test.delayers import ppgDelay, req_delay from plenum.test.spy_helpers import get_count, getAllReturnVals from plenum.test.test_node import getNonPrimaryReplicas -from plenum.test.helper import sdk_send_random_and_check, sdk_ensure_pool_functional +from plenum.test.helper import sdk_send_random_and_check +from plenum.test.node_request.helper import sdk_ensure_pool_functional @pytest.fixture(scope='function', params=['client_requests', diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index d76af9198f..377fb0d685 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -19,7 +19,8 @@ from plenum.common.util import randomString, hexToFriendly from plenum.test.helper import sdk_sign_request_objects, \ sdk_send_signed_requests, sdk_json_to_request_object, \ - sdk_get_and_check_replies, sdk_ensure_pool_functional + sdk_get_and_check_replies +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.test_client import TestClient, genTestClient from plenum.test.test_node import TestNode, \ ensure_node_disconnected, checkNodesConnected diff --git a/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py b/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py index 09c1c41f0e..1edd3cfe81 100644 --- a/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py +++ b/plenum/test/pool_transactions/test_change_ha_persists_post_nodes_restart.py @@ -1,7 +1,7 @@ from plenum.common.util import hexToFriendly, randomString from stp_core.common.log import getlogger from plenum.test.node_catchup.helper import waitNodeDataEquality -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.pool_transactions.helper import sdk_send_update_node, sdk_pool_refresh, \ sdk_add_new_steward_and_node from plenum.test.test_node import TestNode, checkNodesConnected diff --git a/plenum/test/pool_transactions/test_client_with_pool_txns.py b/plenum/test/pool_transactions/test_client_with_pool_txns.py index bb99c06a29..6748061d82 100644 --- a/plenum/test/pool_transactions/test_client_with_pool_txns.py +++ b/plenum/test/pool_transactions/test_client_with_pool_txns.py @@ -2,7 +2,8 @@ from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger from plenum.test import waits -from plenum.test.helper import sdk_send_random_and_check, sdk_ensure_pool_functional +from plenum.test.helper import sdk_send_random_and_check +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.test_node import checkNodesConnected, TestNode, \ ensureElectionsDone from plenum.common.config_helper import PNodeConfigHelper diff --git a/plenum/test/pool_transactions/test_nodes_data_changed.py b/plenum/test/pool_transactions/test_nodes_data_changed.py index be6e696ee9..9687e96ab4 100644 --- a/plenum/test/pool_transactions/test_nodes_data_changed.py +++ b/plenum/test/pool_transactions/test_nodes_data_changed.py @@ -1,7 +1,7 @@ import pytest from plenum.common.exceptions import RequestRejectedException -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.constants import CLIENT_STACK_SUFFIX from plenum.common.util import randomString, hexToFriendly diff --git a/plenum/test/pool_transactions/test_nodes_with_pool_txns.py b/plenum/test/pool_transactions/test_nodes_with_pool_txns.py index c89d7b2695..afeff5b2c2 100644 --- a/plenum/test/pool_transactions/test_nodes_with_pool_txns.py +++ b/plenum/test/pool_transactions/test_nodes_with_pool_txns.py @@ -13,7 +13,8 @@ from plenum.common.util import getMaxFailures, randomString from plenum.test import waits from plenum.test.helper import sdk_send_random_and_check, \ - sdk_get_and_check_replies, sdk_ensure_pool_functional + sdk_get_and_check_replies +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.pool_transactions.helper import sdk_add_new_node, \ sdk_add_2_nodes, sdk_pool_refresh, sdk_add_new_nym, prepare_new_node_data, \ prepare_node_request, sdk_sign_and_send_prepared_request diff --git a/plenum/test/pool_transactions/test_start_many_nodes.py b/plenum/test/pool_transactions/test_start_many_nodes.py index ecbfa76168..49221e5ced 100644 --- a/plenum/test/pool_transactions/test_start_many_nodes.py +++ b/plenum/test/pool_transactions/test_start_many_nodes.py @@ -2,7 +2,8 @@ from plenum.common.constants import NYM, ROLE, STEWARD from plenum.common.constants import TXN_TYPE -from plenum.test.helper import assertEquality, sdk_ensure_pool_functional +from plenum.test.helper import assertEquality +from plenum.test.node_request.helper import sdk_ensure_pool_functional nodeCount = 8 diff --git a/plenum/test/pool_transactions/test_z_node_key_changed.py b/plenum/test/pool_transactions/test_z_node_key_changed.py index 827cab52f4..d95ca1c779 100644 --- a/plenum/test/pool_transactions/test_z_node_key_changed.py +++ b/plenum/test/pool_transactions/test_z_node_key_changed.py @@ -2,7 +2,7 @@ import base58 import types -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common import stack_manager from plenum.common.keygen_utils import initNodeKeysForBothStacks, \ diff --git a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py index b0d16a04dc..2bec935656 100644 --- a/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py +++ b/plenum/test/primary_selection/test_add_node_to_pool_with_large_ppseqno.py @@ -1,7 +1,8 @@ import pytest from plenum.common.util import randomString -from plenum.test.helper import sdk_send_random_and_check, sdk_ensure_pool_functional +from plenum.test.helper import sdk_send_random_and_check +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.view_change.helper import ensure_several_view_change from plenum.test.pool_transactions.helper import sdk_add_new_steward_and_node from plenum.test.test_node import checkNodesConnected diff --git a/plenum/test/primary_selection/test_primary_selection_pool_txn.py b/plenum/test/primary_selection/test_primary_selection_pool_txn.py index 74f97e9a58..5bd4c42205 100644 --- a/plenum/test/primary_selection/test_primary_selection_pool_txn.py +++ b/plenum/test/primary_selection/test_primary_selection_pool_txn.py @@ -1,5 +1,5 @@ import pytest -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.pool_transactions.helper import sdk_add_2_nodes from plenum.test.primary_selection.helper import check_newly_added_nodes diff --git a/plenum/test/restart/test_restart_node_4_all.py b/plenum/test/restart/test_restart_node_4_all.py index fb1af739ca..6f39800dad 100644 --- a/plenum/test/restart/test_restart_node_4_all.py +++ b/plenum/test/restart/test_restart_node_4_all.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes.py b/plenum/test/restart/test_restart_nodes.py index cac91a004a..8816bb3255 100644 --- a/plenum/test/restart/test_restart_nodes.py +++ b/plenum/test/restart/test_restart_nodes.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode diff --git a/plenum/test/restart/test_restart_nodes_4_all_wp.py b/plenum/test/restart/test_restart_nodes_4_all_wp.py index 3880c9101f..8576f90e95 100644 --- a/plenum/test/restart/test_restart_nodes_4_all_wp.py +++ b/plenum/test/restart/test_restart_nodes_4_all_wp.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_4_np.py b/plenum/test/restart/test_restart_nodes_4_np.py index 9c980dd743..b0a6ab713f 100644 --- a/plenum/test/restart/test_restart_nodes_4_np.py +++ b/plenum/test/restart/test_restart_nodes_4_np.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_6.py b/plenum/test/restart/test_restart_nodes_6.py index 9193d6a942..dd21658233 100644 --- a/plenum/test/restart/test_restart_nodes_6.py +++ b/plenum/test/restart/test_restart_nodes_6.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_6_all.py b/plenum/test/restart/test_restart_nodes_6_all.py index cb798b7ccd..5496e7c9b4 100644 --- a/plenum/test/restart/test_restart_nodes_6_all.py +++ b/plenum/test/restart/test_restart_nodes_6_all.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_6_all_wp.py b/plenum/test/restart/test_restart_nodes_6_all_wp.py index 58355850b2..5aebce9826 100644 --- a/plenum/test/restart/test_restart_nodes_6_all_wp.py +++ b/plenum/test/restart/test_restart_nodes_6_all_wp.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_6_np.py b/plenum/test/restart/test_restart_nodes_6_np.py index 213ac392ec..873ed9ef1a 100644 --- a/plenum/test/restart/test_restart_nodes_6_np.py +++ b/plenum/test/restart/test_restart_nodes_6_np.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_7.py b/plenum/test/restart/test_restart_nodes_7.py index 9547ea4f6b..35b9ef35a8 100644 --- a/plenum/test/restart/test_restart_nodes_7.py +++ b/plenum/test/restart/test_restart_nodes_7.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/restart/test_restart_nodes_7_all.py b/plenum/test/restart/test_restart_nodes_7_all.py index f8176871f8..d11968e92a 100644 --- a/plenum/test/restart/test_restart_nodes_7_all.py +++ b/plenum/test/restart/test_restart_nodes_7_all.py @@ -1,7 +1,7 @@ import pytest from plenum.test.test_node import ensure_node_disconnected, checkNodesConnected from plenum.test import waits -from plenum.test.helper import sdk_ensure_pool_functional +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.common.config_helper import PNodeConfigHelper from plenum.test.test_node import TestNode from plenum.test.restart.test_restart_nodes import get_group, restart_nodes diff --git a/plenum/test/view_change/test_view_change_min_cathup_timeout.py b/plenum/test/view_change/test_view_change_min_cathup_timeout.py index d7bf1ae700..292b5a2a12 100644 --- a/plenum/test/view_change/test_view_change_min_cathup_timeout.py +++ b/plenum/test/view_change/test_view_change_min_cathup_timeout.py @@ -2,7 +2,8 @@ import pytest -from plenum.test.helper import waitForViewChange, sdk_send_random_and_check, sdk_ensure_pool_functional +from plenum.test.helper import waitForViewChange, sdk_send_random_and_check +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.test_node import ensureElectionsDone from stp_core.loop.exceptions import EventuallyTimeoutException diff --git a/plenum/test/view_change/test_view_change_timeout.py b/plenum/test/view_change/test_view_change_timeout.py index a0bcd0b494..ddc0f8d958 100644 --- a/plenum/test/view_change/test_view_change_timeout.py +++ b/plenum/test/view_change/test_view_change_timeout.py @@ -1,7 +1,8 @@ import pytest from plenum.test.delayers import reset_delays_and_process_delayeds, vcd_delay -from plenum.test.helper import waitForViewChange, stopNodes, sdk_ensure_pool_functional +from plenum.test.helper import waitForViewChange, stopNodes +from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.node_catchup.helper import ensure_all_nodes_have_same_data from plenum.test.spy_helpers import get_count, getAllReturnVals from plenum.test.test_node import get_master_primary_node, \ From 75e3ab2f1312662ee353c92070620db55485f3ad Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 2 Apr 2018 19:49:10 +0300 Subject: [PATCH 59/73] little fix Signed-off-by: ArtObr --- plenum/test/test_node_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plenum/test/test_node_basic.py b/plenum/test/test_node_basic.py index c35317f520..733298d373 100644 --- a/plenum/test/test_node_basic.py +++ b/plenum/test/test_node_basic.py @@ -5,8 +5,8 @@ from plenum.test.node_helpers.node_helper import getProtocolInstanceNums from plenum.common.util import getMaxFailures from stp_core.common.util import adict -from plenum.test.helper import checkNodesConnected, \ - sendMessageAndCheckDelivery, msgAll +from plenum.test.helper import sendMessageAndCheckDelivery, msgAll +from plenum.test.test_node import checkNodesConnected from plenum.test.msgs import randomMsg nodeCount = 4 From 4afb8394540e918251e826d033d0fa42fa556534 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 3 Apr 2018 14:32:05 +0300 Subject: [PATCH 60/73] more changes Signed-off-by: ArtObr --- .../test/blacklist/test_blacklist_client.py | 35 ++++++++++++++----- ..._checkpoint_stabilization_after_catchup.py | 4 +-- plenum/test/cli/helper.py | 4 --- .../test/client/test_client_request_nack.py | 8 ----- plenum/test/conftest.py | 9 ----- plenum/test/helper.py | 8 +++++ .../test/node_catchup/test_discard_view_no.py | 5 +-- .../test_remove_request_keys_post_catchup.py | 2 +- .../pool_transactions/test_suspend_node.py | 5 +-- plenum/test/test_req_authenticator.py | 21 +++++++---- 10 files changed, 56 insertions(+), 45 deletions(-) diff --git a/plenum/test/blacklist/test_blacklist_client.py b/plenum/test/blacklist/test_blacklist_client.py index b35d3ce784..30c2e00c1d 100644 --- a/plenum/test/blacklist/test_blacklist_client.py +++ b/plenum/test/blacklist/test_blacklist_client.py @@ -1,26 +1,45 @@ +import json + import pytest +from plenum.common.constants import CURRENT_PROTOCOL_VERSION +from plenum.common.exceptions import RequestNackedException +from plenum.test.helper import sdk_random_request_objects, sdk_send_signed_requests, sdk_send_random_request, \ + sdk_sign_request_objects, sdk_get_and_check_replies from stp_core.loop.eventually import eventually from plenum.test import waits from plenum.test.malicious_behaviors_client import makeClientFaulty, \ sendsUnsignedRequest -@pytest.fixture(scope="module") -def setup(client1): - makeClientFaulty(client1, sendsUnsignedRequest) - - # noinspection PyIncorrectDocstring,PyUnusedLocal,PyShadowingNames -def testDoNotBlacklistClient(setup, looper, txnPoolNodeSet, client1, sent1): +def testDoNotBlacklistClient(looper, txnPoolNodeSet, + sdk_wallet_client, sdk_pool_handle, + poolTxnClientNames): """ Client should be not be blacklisted by node on sending an unsigned request """ - + client_name = poolTxnClientNames[0] + _, did = sdk_wallet_client # No node should blacklist the client + reqs_obj = sdk_random_request_objects(1, identifier=did, + protocol_version=CURRENT_PROTOCOL_VERSION) + req = sdk_sign_request_objects(looper, sdk_wallet_client, reqs_obj)[0] + + # break the signature + request_json = json.loads(req) + request_json['reqId'] = request_json['reqId'] + 1 + req = json.dumps(request_json) + + reqs = sdk_send_signed_requests(sdk_pool_handle, [req]) + + with pytest.raises(RequestNackedException) as e: + sdk_get_and_check_replies(looper, reqs) + assert 'InsufficientCorrectSignatures' in e._excinfo[1].args[0] + def chk(): for node in txnPoolNodeSet: - assert not node.isClientBlacklisted(client1.name) + assert not node.isClientBlacklisted(client_name) timeout = waits.expectedClientToPoolConnectionTimeout(len(txnPoolNodeSet)) looper.run(eventually(chk, retryWait=1, timeout=timeout)) diff --git a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py index 0e35fc1eb2..65cd81aee2 100644 --- a/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py +++ b/plenum/test/checkpoints/test_checkpoint_stabilization_after_catchup.py @@ -9,8 +9,8 @@ def test_second_checkpoint_after_catchup_can_be_stabilized( - chkFreqPatched, looper, txnPoolNodeSet, steward1, sdk_wallet_steward, - sdk_wallet_client, sdk_pool_handle, tdir, client_tdir, tconf, + chkFreqPatched, looper, txnPoolNodeSet, sdk_wallet_steward, + sdk_wallet_client, sdk_pool_handle, tdir, tconf, allPluginsPath): _, new_node = sdk_add_new_steward_and_node( looper, sdk_pool_handle, sdk_wallet_steward, diff --git a/plenum/test/cli/helper.py b/plenum/test/cli/helper.py index dc363436e9..a5b57890dd 100644 --- a/plenum/test/cli/helper.py +++ b/plenum/test/cli/helper.py @@ -17,10 +17,8 @@ from stp_core.common.util import Singleton from stp_core.loop.eventually import eventually from stp_core.common.log import getlogger -from plenum.common.util import getMaxFailures from plenum.test.cli.mock_output import MockOutput from plenum.test.cli.test_keyring import createNewKeyring -from plenum.test.helper import waitForSufficientRepliesForRequests from plenum.test.spy_helpers import getAllArgs from plenum.test.test_client import TestClient from plenum.test.test_node import TestNode, checkPoolReady @@ -28,8 +26,6 @@ from pygments.token import Token from functools import partial from plenum.test import waits -from plenum.common import util -from plenum.common.request import Request logger = getlogger() diff --git a/plenum/test/client/test_client_request_nack.py b/plenum/test/client/test_client_request_nack.py index 2f4ca3e445..23e8cef023 100644 --- a/plenum/test/client/test_client_request_nack.py +++ b/plenum/test/client/test_client_request_nack.py @@ -20,14 +20,6 @@ def restrictiveVerifier(txnPoolNodeSet): n.opVerifiers = [TestVerifier()] -@pytest.fixture(scope="module") -def request1(wallet1): - op = {"type": "buy", - "amount": 999} - req = wallet1.signOp(op) - return req - - @pytest.mark.skip(reason="old style plugin") def testRequestFullRoundTrip(restrictiveVerifier, sdk_pool_handle, diff --git a/plenum/test/conftest.py b/plenum/test/conftest.py index adbf6186f0..28bc9e858e 100644 --- a/plenum/test/conftest.py +++ b/plenum/test/conftest.py @@ -446,15 +446,6 @@ def stewardAndWallet1(looper, txnPoolNodeSet, poolTxnStewardData, client.stop() -@pytest.fixture(scope="module") -def steward1(looper, txnPoolNodeSet, stewardAndWallet1): - steward, wallet = stewardAndWallet1 - looper.add(steward) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward, - *txnPoolNodeSet) - return steward - - @pytest.fixture(scope="module") def client1(looper, clientAndWallet1): client = clientAndWallet1[0] diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 204cb11fbe..6c4e390160 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -807,6 +807,14 @@ def sdk_sign_and_submit_req_obj(looper, pool_handle, sdk_wallet, req_obj): return sdk_send_signed_requests(pool_handle, [s_req])[0] +def sdk_sign_and_submit_op(looper, pool_handle, sdk_wallet, op): + _, did = sdk_wallet + req_obj = sdk_gen_request(op, protocol_version=CURRENT_PROTOCOL_VERSION, + identifier=did) + s_req = sdk_sign_request_objects(looper, sdk_wallet, [req_obj])[0] + return sdk_send_signed_requests(pool_handle, [s_req])[0] + + def sdk_get_reply(looper, sdk_req_resp, timeout=None): req_json, resp_task = sdk_req_resp try: diff --git a/plenum/test/node_catchup/test_discard_view_no.py b/plenum/test/node_catchup/test_discard_view_no.py index 03bddeaad2..ff1275a914 100644 --- a/plenum/test/node_catchup/test_discard_view_no.py +++ b/plenum/test/node_catchup/test_discard_view_no.py @@ -16,7 +16,7 @@ def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet, sdk_node_set_with_node_added_after_some_txns, sdk_new_node_caught_up, - allPluginsPath, wallet1): + allPluginsPath, sdk_wallet_client): """ Node discards 3-phase or ViewChangeDone messages from view nos that it does not know of (view nos before it joined the pool) @@ -40,13 +40,14 @@ def testNodeDiscardMessageFromUnknownView(txnPoolNodeSet, messageTimeout = waits.expectedNodeToNodeMessageDeliveryTime() # 3 pc msg (PrePrepare) needs to be discarded + _, did = sdk_wallet_client primaryRepl = getPrimaryReplica(txnPoolNodeSet) three_pc = PrePrepare( 0, viewNo, 10, get_utc_epoch(), - [[wallet1.defaultId, Request.gen_req_id()]], + [[did, Request.gen_req_id()]], 1, "random digest", DOMAIN_LEDGER_ID, diff --git a/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py b/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py index 6a1ce12f4e..abddc2ec62 100644 --- a/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py +++ b/plenum/test/node_catchup/test_remove_request_keys_post_catchup.py @@ -12,7 +12,7 @@ @pytest.fixture(scope='module', params=['some', 'all']) -def setup(request, looper, txnPoolNodeSet, client1, wallet1): +def setup(request, looper, txnPoolNodeSet): slow_node = getNonPrimaryReplicas(txnPoolNodeSet, 0)[1].node fast_nodes = [n for n in txnPoolNodeSet if n != slow_node] # Delay catchup reply so that the test gets time to make the check, diff --git a/plenum/test/pool_transactions/test_suspend_node.py b/plenum/test/pool_transactions/test_suspend_node.py index bdef22aebc..25e71ff209 100644 --- a/plenum/test/pool_transactions/test_suspend_node.py +++ b/plenum/test/pool_transactions/test_suspend_node.py @@ -1,14 +1,11 @@ import pytest -from plenum.client.client import Client from stp_core.loop.eventually import eventually -from plenum.common.constants import CLIENT_STACK_SUFFIX from plenum.common.util import hexToFriendly from plenum.server.node import Node -from plenum.test.helper import waitForSufficientRepliesForRequests from plenum.test.node_catchup.helper import \ ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.pool_transactions.helper import demote_node, \ - buildPoolClientAndWallet, promote_node + promote_node from plenum.test.test_node import TestNode, checkNodesConnected diff --git a/plenum/test/test_req_authenticator.py b/plenum/test/test_req_authenticator.py index ec675b3398..90aa2094f8 100644 --- a/plenum/test/test_req_authenticator.py +++ b/plenum/test/test_req_authenticator.py @@ -6,6 +6,7 @@ from plenum.common.util import randomString from plenum.server.client_authn import SimpleAuthNr, CoreAuthNr from plenum.server.req_authenticator import ReqAuthenticator +from plenum.test.helper import sdk_sign_and_submit_op from plenum.test.plugin.helper import submitOp from plenum.test.pool_transactions.helper import new_client_request @@ -38,7 +39,10 @@ def test_authenticator_registration(pre_reqs, registration): assert req_authnr.get_authnr_by_type(CoreAuthNr) == core_authnr -def test_authentication(pre_reqs, registration, client1, wallet1): +@pytest.mark.skip(reason='sdk integration') +def test_authentication(looper, pre_reqs, registration, + sdk_wallet_client, + sdk_pool_handle): _, core_authnr, req_authnr = pre_reqs # Remove simple_authnr @@ -51,9 +55,10 @@ def test_authentication(pre_reqs, registration, client1, wallet1): DATA: 1 } # Just creating the request - req = submitOp(wallet1, client1, op) + req = sdk_sign_and_submit_op(looper, sdk_pool_handle, + sdk_wallet_client, op) with pytest.raises(NoAuthenticatorFound): - req_authnr.authenticate(req.as_dict) + req_authnr.authenticate(req[0]) # Empty set for query txn type op = { @@ -62,11 +67,13 @@ def test_authentication(pre_reqs, registration, client1, wallet1): DATA: 1 } # Just creating the request - req = submitOp(wallet1, client1, op) - assert set() == req_authnr.authenticate(req.as_dict) + req = sdk_sign_and_submit_op(looper, sdk_pool_handle, + sdk_wallet_client, op) + assert set() == req_authnr.authenticate(req[0]) # identifier for write type + _, did = sdk_wallet_client req, new_wallet = new_client_request(None, randomString(), wallet1) - core_authnr.addIdr(wallet1.defaultId, - wallet1.getVerkey(wallet1.defaultId)) + core_authnr.addIdr(did, + wallet1.getVerkey(did)) assert req_authnr.authenticate(req.as_dict) == {wallet1.defaultId, } From 260116c3e9df8b411919c6540a02e7aa4f322e29 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 4 Apr 2018 09:20:08 +0300 Subject: [PATCH 61/73] fix Signed-off-by: ArtObr --- plenum/test/script/helper.py | 9 +++------ plenum/test/script/test_change_non_primary_node_ha.py | 6 ++++-- plenum/test/script/test_change_primary_node_ha.py | 6 ++++-- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/plenum/test/script/helper.py b/plenum/test/script/helper.py index d777802052..e01008a37b 100644 --- a/plenum/test/script/helper.py +++ b/plenum/test/script/helper.py @@ -23,7 +23,8 @@ def looper(txnPoolNodesLooper): def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, tconf, shouldBePrimary, tdir, - sdk_pool_handle, sdk_wallet_stewards): + sdk_pool_handle, sdk_wallet_stewards, + sdk_wallet_client): # prepare new ha for node and client stack subjectedNode = None node_index = None @@ -72,11 +73,7 @@ def changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, customTimeout=electionTimeout) # start client and check the node HA - anotherClient, _ = genTestClient(tmpdir=tdirWithClientPoolTxns, - usePoolLedger=True) - looper.add(anotherClient) - looper.run(eventually(anotherClient.ensureConnectedToNodes)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_steward, + sdk_wallet_client, 8) diff --git a/plenum/test/script/test_change_non_primary_node_ha.py b/plenum/test/script/test_change_non_primary_node_ha.py index 152ad1b518..c7f436602f 100644 --- a/plenum/test/script/test_change_non_primary_node_ha.py +++ b/plenum/test/script/test_change_non_primary_node_ha.py @@ -14,7 +14,8 @@ @pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330') def testChangeNodeHaForNonPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnData, poolTxnStewardNames, tconf, tdir, - sdk_pool_handle, sdk_wallet_stewards): + sdk_pool_handle, sdk_wallet_stewards, + sdk_wallet_client): changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, @@ -22,4 +23,5 @@ def testChangeNodeHaForNonPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns shouldBePrimary=False, tdir=tdir, sdk_pool_handle=sdk_pool_handle, - sdk_wallet_stewards=sdk_wallet_stewards) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client) diff --git a/plenum/test/script/test_change_primary_node_ha.py b/plenum/test/script/test_change_primary_node_ha.py index c766ad61a0..601f5aea2c 100644 --- a/plenum/test/script/test_change_primary_node_ha.py +++ b/plenum/test/script/test_change_primary_node_ha.py @@ -15,7 +15,8 @@ @pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330') def testChangeNodeHaForPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnData, poolTxnStewardNames, tconf, tdir, - sdk_pool_handle, sdk_wallet_stewards): + sdk_pool_handle, sdk_wallet_stewards, + sdk_wallet_client): changeNodeHa(looper, txnPoolNodeSet, tdirWithClientPoolTxns, @@ -23,4 +24,5 @@ def testChangeNodeHaForPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns, shouldBePrimary=True, tdir=tdir, sdk_pool_handle=sdk_pool_handle, - sdk_wallet_stewards=sdk_wallet_stewards) + sdk_wallet_stewards=sdk_wallet_stewards, + sdk_wallet_client=sdk_wallet_client) From 4c15288792c4c9b5ed083ed7f5bada3a1632c999 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Wed, 4 Apr 2018 11:15:44 +0300 Subject: [PATCH 62/73] fix Signed-off-by: ArtObr --- plenum/test/script/test_change_primary_node_ha.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plenum/test/script/test_change_primary_node_ha.py b/plenum/test/script/test_change_primary_node_ha.py index 601f5aea2c..f755db3602 100644 --- a/plenum/test/script/test_change_primary_node_ha.py +++ b/plenum/test/script/test_change_primary_node_ha.py @@ -12,7 +12,7 @@ 'got error while verifying message'] -@pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330') +@pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330', scope='function') def testChangeNodeHaForPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnData, poolTxnStewardNames, tconf, tdir, sdk_pool_handle, sdk_wallet_stewards, From 98002d732c97358ae62192f2134755f4329e0ba7 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 9 Apr 2018 16:05:54 +0300 Subject: [PATCH 63/73] little addition Signed-off-by: ArtObr --- plenum/test/pool_transactions/helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index 760f8511f5..bee0c9baca 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -190,6 +190,7 @@ def sdk_add_new_steward_and_node(looper, def sdk_add_new_nym(looper, sdk_pool_handle, creators_wallet, alias=None, role=None, seed=None): seed = seed or randomString(32) + alias = alias or randomString(5) wh, _ = creators_wallet # filling nym request and getting steward did From febb315e80cd7265cb011959214f9d8e3f82a9d5 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 9 Apr 2018 19:19:28 +0300 Subject: [PATCH 64/73] some fixes Signed-off-by: ArtObr --- plenum/test/pool_transactions/conftest.py | 8 +++++--- plenum/test/pool_transactions/helper.py | 16 ++++++++++------ 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/plenum/test/pool_transactions/conftest.py b/plenum/test/pool_transactions/conftest.py index c1ad33d254..7ace61bc13 100644 --- a/plenum/test/pool_transactions/conftest.py +++ b/plenum/test/pool_transactions/conftest.py @@ -1,11 +1,12 @@ import pytest from plenum.common.util import randomString -from plenum.test.test_node import checkNodesConnected +from plenum.test.test_node import checkNodesConnected, TestNode from plenum.test.pool_transactions.helper import \ sdk_add_new_steward_and_node, sdk_pool_refresh + @pytest.fixture(scope="module") def tconf(tconf, request): # Lowering DELTA since some requests will result in validation errors and @@ -24,7 +25,7 @@ def sdk_node_theta_added(looper, sdk_pool_handle, sdk_wallet_steward, allPluginsPath, - testNodeClass=None, + testNodeClass=TestNode, name=None): new_steward_name = "testClientSteward" + randomString(3) new_node_name = name or "Theta" @@ -36,7 +37,8 @@ def sdk_node_theta_added(looper, new_node_name, tdir, tconf, - allPluginsPath) + allPluginsPath, + nodeClass=testNodeClass) txnPoolNodeSet.append(new_node) looper.run(checkNodesConnected(txnPoolNodeSet)) sdk_pool_refresh(looper, sdk_pool_handle) diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index bee0c9baca..123b237d44 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -121,14 +121,16 @@ def create_and_start_new_node( auto_start, plugin_path, nodeClass, - do_post_node_creation: Callable = None): + do_post_node_creation: Callable = None, + configClass=PNodeConfigHelper): node = new_node(node_name=node_name, tdir=tdir, node_ha=node_ha, client_ha=client_ha, tconf=tconf, plugin_path=plugin_path, - nodeClass=nodeClass) + nodeClass=nodeClass, + configClass=configClass) if do_post_node_creation: do_post_node_creation(node) if auto_start: @@ -143,8 +145,9 @@ def new_node( client_ha, tconf, plugin_path, - nodeClass): - config_helper = PNodeConfigHelper(node_name, tconf, chroot=tdir) + nodeClass, + configClass=PNodeConfigHelper): + config_helper = configClass(node_name, tconf, chroot=tdir) node = nodeClass(node_name, config_helper=config_helper, config=tconf, @@ -214,7 +217,7 @@ def sdk_add_new_node(looper, new_node_name, tdir, tconf, allPluginsPath=None, autoStart=True, nodeClass=TestNode, - transformOpFunc=None, do_post_node_creation: Callable = None, + do_post_node_creation: Callable = None, services=[VALIDATOR]): nodeClass = nodeClass or TestNode sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort = \ @@ -244,7 +247,8 @@ def sdk_add_new_node(looper, (nodeIp, nodePort), (clientIp, clientPort), tconf, autoStart, allPluginsPath, nodeClass, - do_post_node_creation=do_post_node_creation) + do_post_node_creation=do_post_node_creation, + configClass=PNodeConfigHelper) async def prepare_schema_request(wallet, named_seed, alias, role): From c8589671f236062ea926c71a02bcffaa49959edb Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 10 Apr 2018 11:59:21 +0300 Subject: [PATCH 65/73] Changes in helper functions Signed-off-by: ArtObr --- plenum/test/pool_transactions/helper.py | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index 123b237d44..4f9a525179 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -47,11 +47,10 @@ def new_client_request(role, name, creatorWallet): return creatorWallet.signOp(op), wallet -def prepare_new_node_data(tconf, tdir, - newNodeName): +def prepare_new_node_data(tconf, tdir, newNodeName, configClass=PNodeConfigHelper): sigseed = randomString(32).encode() (nodeIp, nodePort), (clientIp, clientPort) = genHa(2) - config_helper = PNodeConfigHelper(newNodeName, tconf, chroot=tdir) + config_helper = configClass(newNodeName, tconf, chroot=tdir) _, verkey, bls_key = initNodeKeysForBothStacks(newNodeName, config_helper.keys_dir, sigseed, override=True) return sigseed, verkey, bls_key, nodeIp, nodePort, clientIp, clientPort @@ -184,7 +183,6 @@ def sdk_add_new_steward_and_node(looper, allPluginsPath, autoStart=autoStart, nodeClass=nodeClass, - transformOpFunc=transformNodeOpFunc, do_post_node_creation=do_post_node_creation, services=services) return new_steward_wallet_handle, new_node @@ -252,15 +250,7 @@ def sdk_add_new_node(looper, async def prepare_schema_request(wallet, named_seed, alias, role): - wh, submitter_did = wallet - (named_did, named_verkey) = await create_and_store_my_did(wh, - json.dumps({ - 'seed': named_seed, - 'cid': True}) - ) - nym_request = await build_nym_request(submitter_did, named_did, named_verkey, - alias, role) - return nym_request, named_did + pass async def prepare_nym_request(wallet, named_seed, alias, role): From 0610058b9ce2b2dd98635e4476a09fb2d6b3b30b Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 16 Apr 2018 11:23:01 +0300 Subject: [PATCH 66/73] Some changes for node's tests Signed-off-by: ArtObr --- plenum/client/client.py | 4 ++++ plenum/common/constants.py | 1 + plenum/test/helper.py | 7 +++++++ plenum/test/pool_transactions/helper.py | 24 ++++++++++++++---------- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/plenum/client/client.py b/plenum/client/client.py index d6ee701de0..dd221b7946 100644 --- a/plenum/client/client.py +++ b/plenum/client/client.py @@ -5,6 +5,7 @@ """ import copy +import json import os import random import time @@ -300,6 +301,9 @@ def submitReqs(self, *reqs: Request) -> Tuple[List[Request], List[str]]: errs = [] for request in reqs: + with open('/home/artem/Developement/client-requests.txt', 'a') as out: + out.write(json.dumps(request.as_dict) + '\n') + out.write('\n\n\n') is_read_only = request.txn_type in self._read_only_requests if self.can_send_request(request): recipients = self._connected_node_names diff --git a/plenum/common/constants.py b/plenum/common/constants.py index 6e6343eee0..ff3813f8b5 100644 --- a/plenum/common/constants.py +++ b/plenum/common/constants.py @@ -118,6 +118,7 @@ STEWARD = Roles.STEWARD.value TRUSTEE = Roles.TRUSTEE.value STEWARD_STRING = 'STEWARD' +TRUSTEE_STRING = 'TRUSTEE' # TXNs NODE = PlenumTransactions.NODE.value diff --git a/plenum/test/helper.py b/plenum/test/helper.py index 202356814c..bf5b01749a 100644 --- a/plenum/test/helper.py +++ b/plenum/test/helper.py @@ -10,6 +10,7 @@ from time import sleep from typing import Tuple, Iterable, Dict, Optional, List, Any, Sequence, Union +import pytest from psutil import Popen import json import asyncio @@ -975,3 +976,9 @@ def sdk_json_couples_to_request_list(json_couples): for json_couple in json_couples: req_list.append(sdk_json_to_request_object(json_couple[0])) return req_list + + +def sdk_get_bad_response(looper, reqs, exception, message): + with pytest.raises(exception) as e: + sdk_get_and_check_replies(looper, reqs) + assert message in e._excinfo[1].args[0] diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index 663546feca..9ebac7d017 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -30,10 +30,8 @@ from stp_core.common.log import getlogger from indy.error import ErrorCode, IndyError - logger = getlogger() - REFRESH_TRY_COUNT = 4 @@ -197,7 +195,8 @@ def sdk_add_new_steward_and_node(looper, def sdk_add_new_nym(looper, sdk_pool_handle, creators_wallet, - alias=None, role=None, seed=None): + alias=None, role=None, seed=None, + dest=None, verkey=None): seed = seed or randomString(32) alias = alias or randomString(5) wh, _ = creators_wallet @@ -206,7 +205,7 @@ def sdk_add_new_nym(looper, sdk_pool_handle, creators_wallet, # if role == None, we are adding client nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(creators_wallet, seed, - alias, role)) + alias, role, dest, verkey)) # sending request using 'sdk_' functions request_couple = sdk_sign_and_send_prepared_request(looper, creators_wallet, @@ -261,13 +260,17 @@ async def prepare_schema_request(wallet, named_seed, alias, role): pass -async def prepare_nym_request(wallet, named_seed, alias, role): +async def prepare_nym_request(wallet, named_seed, alias, + role, dest=None, verkey=None): wh, submitter_did = wallet - (named_did, named_verkey) = await create_and_store_my_did(wh, - json.dumps({ - 'seed': named_seed, - 'cid': True}) - ) + (named_did, named_verkey) = \ + await create_and_store_my_did(wh, + json.dumps({ + 'seed': named_seed, + 'cid': True}) + ) + if dest: named_did = dest + if verkey: named_verkey = verkey nym_request = await build_nym_request(submitter_did, named_did, named_verkey, alias, role) return nym_request, named_did @@ -353,6 +356,7 @@ def sdk_pool_refresh(looper, sdk_pool_handle): else: return + def sdk_build_get_txn_request(looper, steward_did, data): request = looper.loop.run_until_complete( build_get_txn_request(steward_did, data)) From 1b42e81037bdd8a7f9ab394573f989b9b5d2b57d Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 16 Apr 2018 16:46:07 +0300 Subject: [PATCH 67/73] deletions of client Signed-off-by: ArtObr --- plenum/test/plugin/helper.py | 19 ------------------- plenum/test/test_req_authenticator.py | 1 - .../validator_info/test_validator_info.py | 10 ---------- 3 files changed, 30 deletions(-) diff --git a/plenum/test/plugin/helper.py b/plenum/test/plugin/helper.py index ebf92b7ae1..ad6e694372 100644 --- a/plenum/test/plugin/helper.py +++ b/plenum/test/plugin/helper.py @@ -10,24 +10,5 @@ def getPluginPath(name): return os.path.join(curPath, name) -def submitOp(wallet, client, op): - req = wallet.signOp(op) - client.submitReqs(req) - return req - - -class App: - - def __init__(self, wallet: Wallet, client: TestClient, looper): - self.wallet = wallet - self.client = client - self.looper = looper - - def submit(self, op): - req = self.wallet.signOp(op) - self.client.submitReqs(req) - return req - - def makeReason(common, specific): return '{} [caused by {}]'.format(common, specific) diff --git a/plenum/test/test_req_authenticator.py b/plenum/test/test_req_authenticator.py index 90aa2094f8..a2083c4020 100644 --- a/plenum/test/test_req_authenticator.py +++ b/plenum/test/test_req_authenticator.py @@ -7,7 +7,6 @@ from plenum.server.client_authn import SimpleAuthNr, CoreAuthNr from plenum.server.req_authenticator import ReqAuthenticator from plenum.test.helper import sdk_sign_and_submit_op -from plenum.test.plugin.helper import submitOp from plenum.test.pool_transactions.helper import new_client_request diff --git a/plenum/test/validator_info/test_validator_info.py b/plenum/test/validator_info/test_validator_info.py index 34c6142218..89122876bb 100644 --- a/plenum/test/validator_info/test_validator_info.py +++ b/plenum/test/validator_info/test_validator_info.py @@ -252,13 +252,3 @@ def wrapped(): return load_info(info_path) return wrapped - - -@pytest.fixture -def client_and_wallet(txnPoolNodesLooper, tdirWithClientPoolTxns, txnPoolNodeSet): - client, wallet = genTestClient(tmpdir=tdirWithClientPoolTxns, nodes=txnPoolNodeSet, - name='reader', usePoolLedger=True) - txnPoolNodesLooper.add(client) - ensureClientConnectedToNodesAndPoolLedgerSame(txnPoolNodesLooper, client, - *txnPoolNodeSet) - return client, wallet From 58bc5412c33adaf12aa2d51f292529f8a91caa14 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 16 Apr 2018 19:39:26 +0300 Subject: [PATCH 68/73] fixes for test Signed-off-by: ArtObr --- plenum/client/client.py | 3 --- plenum/test/storage/test_hash_stores.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/plenum/client/client.py b/plenum/client/client.py index dd221b7946..57072dad81 100644 --- a/plenum/client/client.py +++ b/plenum/client/client.py @@ -301,9 +301,6 @@ def submitReqs(self, *reqs: Request) -> Tuple[List[Request], List[str]]: errs = [] for request in reqs: - with open('/home/artem/Developement/client-requests.txt', 'a') as out: - out.write(json.dumps(request.as_dict) + '\n') - out.write('\n\n\n') is_read_only = request.txn_type in self._read_only_requests if self.can_send_request(request): recipients = self._connected_node_names diff --git a/plenum/test/storage/test_hash_stores.py b/plenum/test/storage/test_hash_stores.py index 8dde45542c..9a58287f18 100644 --- a/plenum/test/storage/test_hash_stores.py +++ b/plenum/test/storage/test_hash_stores.py @@ -37,7 +37,7 @@ def testReadWrite(hashStore, nodesLeaves): assert onebyone == multiple -def testRecoverLedgerFromHashStore(hashStore, tdir): +def testRecoverLedgerFromHashStore(hashStore, tconf, tdir): cleanup(hashStore) tree = CompactMerkleTree(hashStore=hashStore) ledger = Ledger(tree=tree, dataDir=tdir) From 29249a57416bf13f8b83490b583a56d00260f4fb Mon Sep 17 00:00:00 2001 From: ArtObr Date: Mon, 16 Apr 2018 20:09:48 +0300 Subject: [PATCH 69/73] little fixes Signed-off-by: ArtObr --- plenum/test/script/test_change_primary_node_ha.py | 2 +- plenum/test/storage/test_client_req_rep_store.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plenum/test/script/test_change_primary_node_ha.py b/plenum/test/script/test_change_primary_node_ha.py index 1b1279c4a0..320f61c870 100644 --- a/plenum/test/script/test_change_primary_node_ha.py +++ b/plenum/test/script/test_change_primary_node_ha.py @@ -13,7 +13,7 @@ TestRunningTimeLimitSec = 200 -@pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330', scope='function') +@pytest.mark.skipif('sys.platform == "win32"', reason='SOV-330') def testChangeNodeHaForPrimary(looper, txnPoolNodeSet, tdirWithClientPoolTxns, poolTxnData, poolTxnStewardNames, tconf, tdir, sdk_pool_handle, sdk_wallet_stewards, diff --git a/plenum/test/storage/test_client_req_rep_store.py b/plenum/test/storage/test_client_req_rep_store.py index 07ee174606..598a76d53b 100644 --- a/plenum/test/storage/test_client_req_rep_store.py +++ b/plenum/test/storage/test_client_req_rep_store.py @@ -8,4 +8,4 @@ def testReqAcks(replied1, client1): assert len(client1.nodeReg) == len(client1.reqRepStore.getAcks(identifier, reqId)) assert set(client1.nodeReg.keys()) == \ - set(client1.reqRepStore.getAcks(identifier, reqId)) \ No newline at end of file + set(client1.reqRepStore.getAcks(identifier, reqId)) From d10124f105498198c21c530d3c028313a896d2c4 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 17 Apr 2018 16:51:23 +0300 Subject: [PATCH 70/73] INDY-1209: fix test timeout Signed-off-by: ArtObr --- .../test_6th_node_join_after_view_change_by_primary_restart.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py index a55a8672da..c0059774a7 100644 --- a/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py +++ b/plenum/test/view_change/test_6th_node_join_after_view_change_by_primary_restart.py @@ -117,7 +117,7 @@ def test_6th_node_join_after_view_change_by_master_restart( timeout = waits.expectedPoolCatchupTime(nodeCount=len(pool_of_nodes)) for node in pool_of_nodes: - looper.run(eventually(catchuped, node, timeout=2 * timeout)) + looper.run(eventually(catchuped, node, timeout=3 * timeout)) sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, 2) new_psi_node = add_new_node(looper, From f8b9da889f765ad04de21bdec2bd8415f9d6077e Mon Sep 17 00:00:00 2001 From: ArtObr Date: Fri, 20 Apr 2018 14:02:02 +0300 Subject: [PATCH 71/73] INDY-1209: Fix tests according to review Signed-off-by: ArtObr --- .../test/blacklist/test_blacklist_client.py | 20 ++---- plenum/test/bls/helper.py | 54 ++------------ plenum/test/client/conftest.py | 4 +- plenum/test/client/test_client.py | 57 ++------------- ...st_client_sends_get_request_to_one_node.py | 40 ----------- .../client/test_client_sends_to_less_nodes.py | 21 ------ .../test_faulty_client_with_faulty_node.py | 8 +-- plenum/test/pool_transactions/helper.py | 42 +++++------ .../pool_transactions/test_suspend_node.py | 70 ++++++------------- .../test/storage/test_client_req_rep_store.py | 11 --- plenum/test/test_req_authenticator.py | 16 +++-- 11 files changed, 71 insertions(+), 272 deletions(-) delete mode 100644 plenum/test/client/test_client_sends_get_request_to_one_node.py delete mode 100644 plenum/test/client/test_client_sends_to_less_nodes.py delete mode 100644 plenum/test/storage/test_client_req_rep_store.py diff --git a/plenum/test/blacklist/test_blacklist_client.py b/plenum/test/blacklist/test_blacklist_client.py index 30c2e00c1d..2599f0c338 100644 --- a/plenum/test/blacklist/test_blacklist_client.py +++ b/plenum/test/blacklist/test_blacklist_client.py @@ -4,12 +4,10 @@ from plenum.common.constants import CURRENT_PROTOCOL_VERSION from plenum.common.exceptions import RequestNackedException -from plenum.test.helper import sdk_random_request_objects, sdk_send_signed_requests, sdk_send_random_request, \ - sdk_sign_request_objects, sdk_get_and_check_replies +from plenum.test.helper import sdk_random_request_objects, sdk_send_signed_requests, \ + sdk_get_and_check_replies from stp_core.loop.eventually import eventually from plenum.test import waits -from plenum.test.malicious_behaviors_client import makeClientFaulty, \ - sendsUnsignedRequest # noinspection PyIncorrectDocstring,PyUnusedLocal,PyShadowingNames @@ -22,20 +20,14 @@ def testDoNotBlacklistClient(looper, txnPoolNodeSet, client_name = poolTxnClientNames[0] _, did = sdk_wallet_client # No node should blacklist the client - reqs_obj = sdk_random_request_objects(1, identifier=did, - protocol_version=CURRENT_PROTOCOL_VERSION) - req = sdk_sign_request_objects(looper, sdk_wallet_client, reqs_obj)[0] + req_obj = sdk_random_request_objects(1, identifier=did, + protocol_version=CURRENT_PROTOCOL_VERSION)[0] - # break the signature - request_json = json.loads(req) - request_json['reqId'] = request_json['reqId'] + 1 - req = json.dumps(request_json) - - reqs = sdk_send_signed_requests(sdk_pool_handle, [req]) + reqs = sdk_send_signed_requests(sdk_pool_handle, [json.dumps(req_obj.as_dict)]) with pytest.raises(RequestNackedException) as e: sdk_get_and_check_replies(looper, reqs) - assert 'InsufficientCorrectSignatures' in e._excinfo[1].args[0] + assert 'MissingSignature' in e._excinfo[1].args[0] def chk(): for node in txnPoolNodeSet: diff --git a/plenum/test/bls/helper.py b/plenum/test/bls/helper.py index 1d91e356dc..77a0126d4a 100644 --- a/plenum/test/bls/helper.py +++ b/plenum/test/bls/helper.py @@ -14,9 +14,9 @@ from plenum.common.util import get_utc_epoch, randomString, hexToFriendly from plenum.test.helper import sdk_send_random_and_check from plenum.test.node_request.helper import sdk_ensure_pool_functional -from plenum.test.node_catchup.helper import waitNodeDataEquality, ensure_all_nodes_have_same_data +from plenum.test.node_catchup.helper import waitNodeDataEquality from plenum.test.pool_transactions.helper import sdk_send_update_node, \ - sdk_pool_refresh, sdk_add_new_nym + sdk_pool_refresh from stp_core.common.log import getlogger logger = getlogger() @@ -65,45 +65,6 @@ def sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, assert multi_sigs.count(multi_sigs[0]) == len(multi_sigs) -def sdk_check_bls_multi_sig_after_send(looper, txnPoolNodeSet, - sdk_pool_handle, sdk_wallet_handle, - saved_multi_sigs_count): - # at least two because first request could have no - # signature since state can be clear - number_of_requests = 3 - - # 1. send requests - # Using loop to avoid 3pc batching - state_roots = [] - for i in range(number_of_requests): - sdk_send_random_and_check(looper, txnPoolNodeSet, sdk_pool_handle, - sdk_wallet_handle, 1) - waitNodeDataEquality(looper, txnPoolNodeSet[0], *txnPoolNodeSet[:-1]) - state_roots.append( - state_roots_serializer.serialize( - bytes(txnPoolNodeSet[0].getState(DOMAIN_LEDGER_ID).committedHeadHash))) - - # 2. get all saved multi-sigs - multi_sigs_for_batch = [] - for state_root in state_roots: - multi_sigs = [] - for node in txnPoolNodeSet: - multi_sig = node.bls_bft.bls_store.get(state_root) - if multi_sig: - multi_sigs.append(multi_sig) - multi_sigs_for_batch.append(multi_sigs) - - # 3. check how many multi-sigs are saved - for multi_sigs in multi_sigs_for_batch: - assert len(multi_sigs) == saved_multi_sigs_count, \ - "{} != {}".format(len(multi_sigs), saved_multi_sigs_count) - - # 3. check that bls multi-sig is the same for all nodes we get PrePrepare for (that is for all expect the last one) - for multi_sigs in multi_sigs_for_batch[:-1]: - if multi_sigs: - assert multi_sigs.count(multi_sigs[0]) == len(multi_sigs) - - def process_commits_for_key(key, pre_prepare, bls_bfts): for sender_bls_bft in bls_bfts: commit = create_commit_bls_sig( @@ -208,8 +169,7 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, sdk_pool_handle, sdk_wallet_steward, add_wrong=False, - new_bls=None, - use_in_plenum=True): + new_bls=None): new_blspk = init_bls_keys(node.keys_dir, node.name) key_in_txn = new_bls or new_blspk \ if not add_wrong \ @@ -226,13 +186,7 @@ def sdk_change_bls_key(looper, txnPoolNodeSet, poolSetExceptOne.remove(node) waitNodeDataEquality(looper, node, *poolSetExceptOne) sdk_pool_refresh(looper, sdk_pool_handle) - if (use_in_plenum): - sdk_ensure_pool_functional(looper, txnPoolNodeSet, - sdk_wallet_steward, sdk_pool_handle) - else: - sdk_add_new_nym(looper, sdk_pool_handle, sdk_wallet_steward, - alias=randomString(5)) - ensure_all_nodes_have_same_data(looper, txnPoolNodeSet) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) return new_blspk diff --git a/plenum/test/client/conftest.py b/plenum/test/client/conftest.py index ce56e5d181..31d54d8220 100644 --- a/plenum/test/client/conftest.py +++ b/plenum/test/client/conftest.py @@ -2,6 +2,7 @@ from plenum.test.conftest import getValueFromModule from plenum.client.client import Client +from plenum.test.helper import sdk_json_to_request_object from plenum.test.malicious_behaviors_client import makeClientFaulty @@ -16,4 +17,5 @@ def passThroughReqAcked1(sent1): # Overriding reqAcked1 in conftest.py to do nothing because the client # shouldn't see ReqAck msgs from all the nodes, since it only sent REQUESTs # to some. - return sent1 + request = sdk_json_to_request_object(sent1[0][0]) + return request diff --git a/plenum/test/client/test_client.py b/plenum/test/client/test_client.py index 40b08a156d..d8fdfb351e 100644 --- a/plenum/test/client/test_client.py +++ b/plenum/test/client/test_client.py @@ -6,13 +6,11 @@ from plenum.common.exceptions import MissingSignature from plenum.common.exceptions import NotConnectedToAny from stp_core.common.log import getlogger -from plenum.common.constants import OP_FIELD_NAME, REPLY, REQACK from plenum.common.types import f from plenum.server.node import Node from plenum.test import waits from plenum.test.helper import randomOperation, \ - checkLastClientReqForNode, getRepliesFromClientInbox, \ - waitForSufficientRepliesForRequests, assertLength + checkLastClientReqForNode, sdk_get_and_check_replies, sdk_send_random_request from plenum.test.test_client import genTestClient @@ -20,14 +18,6 @@ F = getMaxFailures(nodeCount) -whitelist = ['signer not configured so not signing', - 'for EmptySignature', - 'discarding message', - 'found legacy entry', - 'public key from disk', - 'verification key from disk', - 'got error while verifying message'] # warnings - logger = getlogger() @@ -113,48 +103,13 @@ def testRequestFullRoundTrip(replied1, client1): # noinspection PyIncorrectDocstring -@pytest.mark.skip(reason='sdk integration') -def testEveryNodeRepliesWithNoFaultyNodes(looper, client1, replied1): - """ - Every node will send a reply to the client when there are no faulty nodes in - the system - """ - - def chk(): - receivedReplies = getRepliesFromClientInbox(client1.inBox, - replied1.reqId) - print(receivedReplies) - assert len(receivedReplies) == nodeCount - - looper.run(eventually(chk)) - - -# noinspection PyIncorrectDocstring -@pytest.mark.skip(reason='sdk integration') -def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, client1, sent1): +def testReplyWhenRequestAlreadyExecuted(looper, txnPoolNodeSet, sdk_pool_handle, + sdk_wallet_client, sent1): """ When a request has already been executed the previously executed reply will be sent again to the client. An acknowledgement will not be sent for a repeated request. """ - waitForSufficientRepliesForRequests(looper, client1, requests=[sent1]) - - originalRequestResponsesLen = nodeCount * 2 - duplicateRequestRepliesLen = nodeCount # for a duplicate request we need to - - message_parts, err_msg = \ - client1.nodestack.prepare_for_sending(sent1, None) - - for part in message_parts: - client1.nodestack._enqueueIntoAllRemotes(part, None) - - def chk(): - assertLength([response for response in client1.inBox - if (response[0].get(f.RESULT.nm) and - response[0][f.RESULT.nm][f.REQ_ID.nm] == sent1.reqId) or - (response[0].get(OP_FIELD_NAME) == REQACK and - response[0].get(f.REQ_ID.nm) == sent1.reqId)], - originalRequestResponsesLen + duplicateRequestRepliesLen) - - responseTimeout = waits.expectedTransactionExecutionTime(nodeCount) - looper.run(eventually(chk, retryWait=1, timeout=responseTimeout)) + sdk_get_and_check_replies(looper, sent1) + req = sdk_send_random_request(looper, sdk_pool_handle, sdk_wallet_client) + sdk_get_and_check_replies(looper, [req]) diff --git a/plenum/test/client/test_client_sends_get_request_to_one_node.py b/plenum/test/client/test_client_sends_get_request_to_one_node.py deleted file mode 100644 index c31b276d0c..0000000000 --- a/plenum/test/client/test_client_sends_get_request_to_one_node.py +++ /dev/null @@ -1,40 +0,0 @@ -import random - -import pytest - -from plenum.test.pool_transactions.helper import sdk_build_get_txn_request, sdk_sign_and_send_prepared_request -from stp_core.common.log import getlogger - -from plenum.test.helper import stopNodes, sdk_send_random_and_check, \ - sdk_get_and_check_replies - -logger = getlogger() - -nodeCount = 4 - - -@pytest.mark.skip(reason='sdk client cannot do that yet') -def test_client_can_send_get_request_to_one_node(looper, - sdk_pool_handle, - sdk_wallet_client, - txnPoolNodeSet): - """ - Check that read only request can be sent - without having connection to all nodes - """ - logger.info("Send set request") - req = sdk_send_random_and_check(looper, txnPoolNodeSet, - sdk_pool_handle, - sdk_wallet_client, 1)[0] - - logger.info("Stopping nodes") - nodes_to_stop = list(txnPoolNodeSet)[1:] - stopNodes(nodes_to_stop, looper) - - logger.info("Send get request") - _, did = sdk_wallet_client - req = sdk_build_get_txn_request(looper, did, req[0]['reqId']) - request = sdk_sign_and_send_prepared_request(looper, sdk_wallet_client, - sdk_pool_handle, req) - replies = sdk_get_and_check_replies(looper, [request]) - pass diff --git a/plenum/test/client/test_client_sends_to_less_nodes.py b/plenum/test/client/test_client_sends_to_less_nodes.py deleted file mode 100644 index 43c8a94a51..0000000000 --- a/plenum/test/client/test_client_sends_to_less_nodes.py +++ /dev/null @@ -1,21 +0,0 @@ -import pytest - -from plenum.test.client.conftest import passThroughReqAcked1 - -from plenum.client.client import Client -from plenum.test.malicious_behaviors_client import \ - genDoesntSendRequestToSomeNodes - -nodeCount = 4 -clientFault = genDoesntSendRequestToSomeNodes(skipCount=3) -reqAcked1 = passThroughReqAcked1 - - -# noinspection PyIncorrectDocstring -@pytest.mark.skip(reason='sdk integration') -def testReplyWhenRequestSentToLessThanFPlusOneNodes(looper, txnPoolNodeSet, - fClient: Client, replied1): - """ - In a system with no faulty nodes, even if the client sends the request to - one node it will get replies from all the nodes. - """ diff --git a/plenum/test/client/test_faulty_client_with_faulty_node.py b/plenum/test/client/test_faulty_client_with_faulty_node.py index 7faf5c88a9..d254a685ec 100644 --- a/plenum/test/client/test_faulty_client_with_faulty_node.py +++ b/plenum/test/client/test_faulty_client_with_faulty_node.py @@ -5,12 +5,8 @@ from plenum.test.helper import whitelistNode from plenum.test.malicious_behaviors_node import makeNodeFaulty, changesRequest -from plenum.test.malicious_behaviors_client import \ - genDoesntSendRequestToSomeNodes - nodeCount = 4 faultyNodes = 1 -clientFault = genDoesntSendRequestToSomeNodes("GammaC", "DeltaC") reqAcked1 = passThroughReqAcked1 whitelist = ['for InvalidSignature', 'discarding message'] @@ -25,12 +21,12 @@ def nodeChangesRequest(txnPoolNodeSet): [node for node in txnPoolNodeSet if node != alpha], InvalidSignature.code) makeNodeFaulty(alpha, changesRequest, ) + for node in txnPoolNodeSet[3:]: node.clientstack.stop() # noinspection PyIncorrectDocstring,PyUnusedLocal,PyShadowingNames -@pytest.mark.skip(reason='sdk integration') def testReplyUnaffectedByFaultyNode(looper, txnPoolNodeSet, nodeChangesRequest, - fClient, replied1): + replied1): """ Client is malicious - sends requests to Alpha and Beta only Node Alpha is malicious - it alters the request diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index b1c86f93c4..394fede004 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -3,24 +3,20 @@ from indy.did import create_and_store_my_did from indy.ledger import build_node_request, build_nym_request, build_get_txn_request from indy.pool import refresh_pool_ledger -from plenum.test.node_catchup.helper import waitNodeDataEquality, \ - ensureClientConnectedToNodesAndPoolLedgerSame -from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.node_catchup.helper import waitNodeDataEquality from stp_core.loop.looper import Looper from stp_core.types import HA from typing import Iterable, Union, Callable from plenum.client.wallet import Wallet -from plenum.common.constants import TXN_TYPE, NYM, ROLE, TARGET_NYM, ALIAS, \ - NODE_PORT, CLIENT_IP, NODE_IP, DATA, NODE, CLIENT_PORT, VERKEY, SERVICES, \ - VALIDATOR, BLS_KEY, STEWARD_STRING +from plenum.common.constants import VERKEY, VALIDATOR, STEWARD_STRING from plenum.common.keygen_utils import initNodeKeysForBothStacks from plenum.common.signer_simple import SimpleSigner from plenum.common.signer_did import DidSigner from plenum.common.util import randomString, hexToFriendly from plenum.test.helper import sdk_sign_request_objects, \ sdk_send_signed_requests, sdk_json_to_request_object, \ - sdk_get_and_check_replies + sdk_get_and_check_replies, sdk_sign_request_strings from plenum.test.node_request.helper import sdk_ensure_pool_functional from plenum.test.test_client import TestClient, genTestClient from plenum.test.test_node import TestNode, \ @@ -28,29 +24,27 @@ from stp_core.network.port_dispenser import genHa from plenum.common.config_helper import PNodeConfigHelper from stp_core.common.log import getlogger -from indy.error import ErrorCode, IndyError logger = getlogger() REFRESH_TRY_COUNT = 4 -def new_client_request(role, name, creatorWallet): - wallet = Wallet(name) - wallet.addIdentifier() - idr = wallet.defaultId - - op = { - TXN_TYPE: NYM, - TARGET_NYM: idr, - ALIAS: name, - VERKEY: wallet.getVerkey(idr) - } - - if role: - op[ROLE] = role - - return creatorWallet.signOp(op), wallet +def new_client_request(role, name, looper, sdk_wallet): + wh, did = sdk_wallet + seed = randomString(32) + (named_did, named_verkey) = looper.loop.run_until_complete( + create_and_store_my_did(wh, + json.dumps({ + 'seed': seed, + 'cid': True}) + )) + nym_request = looper.loop.run_until_complete( + build_nym_request(did, named_did, named_verkey, + name, role)) + + return sdk_sign_request_strings(looper, sdk_wallet, + [json.loads(nym_request)])[0] def prepare_new_node_data(tconf, tdir, newNodeName, configClass=PNodeConfigHelper): diff --git a/plenum/test/pool_transactions/test_suspend_node.py b/plenum/test/pool_transactions/test_suspend_node.py index 25e71ff209..701493f029 100644 --- a/plenum/test/pool_transactions/test_suspend_node.py +++ b/plenum/test/pool_transactions/test_suspend_node.py @@ -1,12 +1,17 @@ import pytest + +from stp_core.network.exceptions import RemoteNotFound + +from plenum.test.helper import sendMessageAndCheckDelivery + +from plenum.test.node_request.helper import sdk_ensure_pool_functional +from plenum.test.view_change.helper import start_stopped_node + from stp_core.loop.eventually import eventually -from plenum.common.util import hexToFriendly from plenum.server.node import Node -from plenum.test.node_catchup.helper import \ - ensureClientConnectedToNodesAndPoolLedgerSame from plenum.test.pool_transactions.helper import demote_node, \ - promote_node -from plenum.test.test_node import TestNode, checkNodesConnected + promote_node, sdk_pool_refresh +from plenum.test.test_node import checkNodesConnected def checkNodeNotInNodeReg(node, nodeName): @@ -17,67 +22,38 @@ def checkNodeNotInNodeReg(node, nodeName): raise ValueError("pass a node or client object as first argument") -@pytest.mark.skip(reason="SOV-383") def testStewardSuspendsNode(looper, txnPoolNodeSet, - tdirWithPoolTxns, tconf, + tdir, tconf, sdk_pool_handle, sdk_wallet_steward, sdk_node_theta_added, poolTxnStewardData, allPluginsPath): new_steward_wallet, new_node = sdk_node_theta_added - newNodeNym = hexToFriendly(new_node.nodestack.verhex) - demote_node(looper, newSteward, newStewardWallet, newNodeNym, new_node.name) + demote_node(looper, new_steward_wallet, sdk_pool_handle, new_node) # Check suspended node does not exist in any nodeReg or remotes of # nodes or clients txnPoolNodeSet = txnPoolNodeSet[:-1] for node in txnPoolNodeSet: looper.run(eventually(checkNodeNotInNodeReg, node, new_node.name)) - - # Check a client can send request and receive replies - # req = sendRandomRequest(newStewardWallet, newSteward) - # waitForSufficientRepliesForRequests(looper, newSteward, - # requests=[req]) - - # Check that a restarted client or node does not connect to the suspended + # Check that a node does not connect to the suspended # node - steward1.stop() - looper.removeProdable(steward1) - steward1, stewardWallet - looper.add(steward1) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, - *txnPoolNodeSet) - looper.run(eventually(checkNodeNotInNodeReg, steward1, new_node.name)) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, new_steward_wallet, sdk_pool_handle) + with pytest.raises(RemoteNotFound): + looper.loop.run_until_complete(sendMessageAndCheckDelivery(txnPoolNodeSet[0], new_node)) new_node.stop() looper.removeProdable(new_node) - # TODO: There is a bug that if a primary node is turned off, it sends - # duplicate Pre-Prepare and gets blacklisted. Here is the gist - # https://gist.github.com/lovesh/c16989616ebb6856f9fa2905c14dc4b7 - oldNodeIdx, oldNode = [(i, n) for i, n in enumerate(txnPoolNodeSet) - if not n.hasPrimary][0] - oldNode.stop() - looper.removeProdable(oldNode) - oldNode = TestNode(oldNode.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, - config=tconf, pluginPaths=allPluginsPath) - looper.add(oldNode) - txnPoolNodeSet[oldNodeIdx] = oldNode - looper.run(checkNodesConnected(txnPoolNodeSet)) - looper.run(eventually(checkNodeNotInNodeReg, oldNode, new_node.name)) - # Check that a node whose suspension is revoked can reconnect to other # nodes and clients can also connect to that node - promote_node(looper, newSteward, newStewardWallet, newNodeNym, - new_node.name) - nodeTheta = TestNode(new_node.name, basedirpath=tdirWithPoolTxns, base_data_dir=tdirWithPoolTxns, - config=tconf, pluginPaths=allPluginsPath, - ha=new_node.nodestack.ha, cliha=new_node.clientstack.ha) - looper.add(nodeTheta) + + promote_node(looper, new_steward_wallet, sdk_pool_handle, new_node) + nodeTheta = start_stopped_node(new_node, looper, tconf, + tdir, allPluginsPath, + delay_instance_change_msgs=False) txnPoolNodeSet.append(nodeTheta) looper.run(checkNodesConnected(txnPoolNodeSet)) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, steward1, - *txnPoolNodeSet) - ensureClientConnectedToNodesAndPoolLedgerSame(looper, newSteward, - *txnPoolNodeSet) + sdk_pool_refresh(looper, sdk_pool_handle) + sdk_ensure_pool_functional(looper, txnPoolNodeSet, sdk_wallet_steward, sdk_pool_handle) diff --git a/plenum/test/storage/test_client_req_rep_store.py b/plenum/test/storage/test_client_req_rep_store.py deleted file mode 100644 index 598a76d53b..0000000000 --- a/plenum/test/storage/test_client_req_rep_store.py +++ /dev/null @@ -1,11 +0,0 @@ -import pytest - - -@pytest.mark.skip(reason='sdk integration') -def testReqAcks(replied1, client1): - reqId = replied1.reqId - identifier = replied1.identifier - assert len(client1.nodeReg) == len(client1.reqRepStore.getAcks(identifier, - reqId)) - assert set(client1.nodeReg.keys()) == \ - set(client1.reqRepStore.getAcks(identifier, reqId)) diff --git a/plenum/test/test_req_authenticator.py b/plenum/test/test_req_authenticator.py index a2083c4020..e5683c45c1 100644 --- a/plenum/test/test_req_authenticator.py +++ b/plenum/test/test_req_authenticator.py @@ -1,5 +1,8 @@ +import json + import pytest +from indy.did import key_for_did from plenum.common.constants import TXN_TYPE, DATA, GET_TXN, DOMAIN_LEDGER_ID from plenum.common.exceptions import NoAuthenticatorFound from plenum.common.types import f @@ -38,7 +41,6 @@ def test_authenticator_registration(pre_reqs, registration): assert req_authnr.get_authnr_by_type(CoreAuthNr) == core_authnr -@pytest.mark.skip(reason='sdk integration') def test_authentication(looper, pre_reqs, registration, sdk_wallet_client, sdk_pool_handle): @@ -55,7 +57,7 @@ def test_authentication(looper, pre_reqs, registration, } # Just creating the request req = sdk_sign_and_submit_op(looper, sdk_pool_handle, - sdk_wallet_client, op) + sdk_wallet_client, op) with pytest.raises(NoAuthenticatorFound): req_authnr.authenticate(req[0]) @@ -67,12 +69,12 @@ def test_authentication(looper, pre_reqs, registration, } # Just creating the request req = sdk_sign_and_submit_op(looper, sdk_pool_handle, - sdk_wallet_client, op) + sdk_wallet_client, op) assert set() == req_authnr.authenticate(req[0]) # identifier for write type - _, did = sdk_wallet_client - req, new_wallet = new_client_request(None, randomString(), wallet1) + wh, did = sdk_wallet_client + req = new_client_request(None, randomString(), looper, sdk_wallet_client) core_authnr.addIdr(did, - wallet1.getVerkey(did)) - assert req_authnr.authenticate(req.as_dict) == {wallet1.defaultId, } + looper.loop.run_until_complete(key_for_did(sdk_pool_handle, wh, did))) + assert req_authnr.authenticate(json.loads(req)) == {did, } From 6fc9a41a67bd17887f25289c1b9b8d9fd9296bf6 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 24 Apr 2018 10:43:24 +0300 Subject: [PATCH 72/73] INDY-1209: Add functionality for skipping verkey in request Signed-off-by: ArtObr --- plenum/test/pool_transactions/helper.py | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/plenum/test/pool_transactions/helper.py b/plenum/test/pool_transactions/helper.py index 394fede004..ff2613e694 100644 --- a/plenum/test/pool_transactions/helper.py +++ b/plenum/test/pool_transactions/helper.py @@ -34,11 +34,7 @@ def new_client_request(role, name, looper, sdk_wallet): wh, did = sdk_wallet seed = randomString(32) (named_did, named_verkey) = looper.loop.run_until_complete( - create_and_store_my_did(wh, - json.dumps({ - 'seed': seed, - 'cid': True}) - )) + create_and_store_my_did(wh, json.dumps({'seed': seed}))) nym_request = looper.loop.run_until_complete( build_nym_request(did, named_did, named_verkey, name, role)) @@ -190,7 +186,7 @@ def sdk_add_new_steward_and_node(looper, def sdk_add_new_nym(looper, sdk_pool_handle, creators_wallet, alias=None, role=None, seed=None, - dest=None, verkey=None): + dest=None, verkey=None,skipverkey=False): seed = seed or randomString(32) alias = alias or randomString(5) wh, _ = creators_wallet @@ -199,7 +195,7 @@ def sdk_add_new_nym(looper, sdk_pool_handle, creators_wallet, # if role == None, we are adding client nym_request, new_did = looper.loop.run_until_complete( prepare_nym_request(creators_wallet, seed, - alias, role, dest, verkey)) + alias, role, dest, verkey, skipverkey)) # sending request using 'sdk_' functions request_couple = sdk_sign_and_send_prepared_request(looper, creators_wallet, @@ -255,16 +251,13 @@ async def prepare_schema_request(wallet, named_seed, alias, role): async def prepare_nym_request(wallet, named_seed, alias, - role, dest=None, verkey=None): + role, dest=None, verkey=None, skipverkey=False): wh, submitter_did = wallet (named_did, named_verkey) = \ - await create_and_store_my_did(wh, - json.dumps({ - 'seed': named_seed, - 'cid': True}) - ) - if dest: named_did = dest - if verkey: named_verkey = verkey + await create_and_store_my_did(wh, json.dumps({'seed': named_seed})) + named_did = dest or named_did + named_verkey = verkey or named_verkey + named_verkey = None if skipverkey else named_verkey nym_request = await build_nym_request(submitter_did, named_did, named_verkey, alias, role) return nym_request, named_did From c019e24039a55cb55ae890381b416935340084f9 Mon Sep 17 00:00:00 2001 From: ArtObr Date: Tue, 24 Apr 2018 12:30:03 +0300 Subject: [PATCH 73/73] INDY-1209: Reset timeout back Signed-off-by: ArtObr --- plenum/test/node_catchup/test_revert_during_catchup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plenum/test/node_catchup/test_revert_during_catchup.py b/plenum/test/node_catchup/test_revert_during_catchup.py index 5129d9be51..d1cb43d588 100644 --- a/plenum/test/node_catchup/test_revert_during_catchup.py +++ b/plenum/test/node_catchup/test_revert_during_catchup.py @@ -42,7 +42,7 @@ def test_slow_node_reverts_unordered_state_during_catchup(looper, slow_master_replica = slow_node.master_replica commit_delay = 150 - catchup_rep_delay = 35 + catchup_rep_delay = 25 # Delay COMMITs to one node slow_node.nodeIbStasher.delay(cDelay(commit_delay, 0))