Merge remote-tracking branch 'public/master' into rc-1.2.35

Signed-off-by: Andrew Nikitin <[email protected]>

Author: Andrew Nikitin

Jenkinsfile

@@ -78,7 +78,6 @@ def stpTestUbuntu = {
             testHelpers.install()
 
             echo 'Ubuntu Test: Test'
-            testHelpers.testJUnit([testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml"])
             testHelpers.testJUnit([testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml"])
         }
     }

Jenkinsfile.cd

@@ -78,7 +78,6 @@ def stpTestUbuntu = {
             testHelpers.install()
 
             echo 'Ubuntu Test: Test'
-            testHelpers.testJUnit([testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml"])
             testHelpers.testJUnit([testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml"])
         }
     }

Jenkinsfile.ci

@@ -89,7 +89,7 @@ def test(options=[:]) {
 
     try {
         if (options.useRunner) {
-            sh "PYTHONASYNCIODEBUG='0' $options.python runner.py --pytest \"$options.python -m pytest\" --dir $options.testDir --output \"$options.resFile\" --test-only-slice \"$options.testOnlySlice\""
+            sh "RUSTPYTHONASYNCIODEBUG='0' $options.python runner.py --pytest \"$options.python -m pytest\" --dir $options.testDir --output \"$options.resFile\" --test-only-slice \"$options.testOnlySlice\""
         } else {
             sh "$options.python -m pytest --junitxml=$options.resFile $options.testDir"
         }
@@ -128,7 +128,6 @@ def staticCodeValidation() {
 
 def tests = [
     stp: { python ->
-        test(testDir: 'stp_raet', resFile: "test-result-stp-raet.${NODE_NAME}.xml", python: python)
         test(testDir: 'stp_zmq', resFile: "test-result-stp-zmq.${NODE_NAME}.xml", python: python)
     },
     ledger: { python ->

README.md

@@ -49,8 +49,7 @@ mechanism that Plenum needs.
     - state storage using python 3 version of Ethereum's Patricia Trie
 - stp:
     - secure transport abstraction
-    - it has two implementations: RAET and ZeroMQ
-    - Although RAET implementation is there, it's not supported anymore, and [ZeroMQ](http://zeromq.org/) is the default secure transport in plenum. 
+    - it has [ZeroMQ](http://zeromq.org/) implementations
 - storage:
     - key-value storage abstractions
     - contains [leveldb](http://leveldb.org/) implementation as the main key-valued storage used in Plenum (for ledger, state, etc.)

build-scripts/ubuntu-1604/Dockerfile

@@ -1,5 +1,9 @@
 FROM ubuntu:16.04
 
+RUN apt-get update -y && apt-get install -y \
+        apt-transport-https \
+        ca-certificates
+
 RUN apt-get update -y && apt-get install -y \
     # common stuff
         git \
@@ -9,12 +13,19 @@ RUN apt-get update -y && apt-get install -y \
         python3-pip \
         python-setuptools \
         python3-venv \
-    # fmp
+    # fpm
         ruby \
         ruby-dev \
         rubygems \
         gcc \
-        make
+        make \
+    # rocksdb python wrapper
+        libbz2-dev \
+        zlib1g-dev \
+        liblz4-dev \
+        libsnappy-dev
+
+RUN pip3 install -U setuptools
 
 # install fpm
 RUN gem install --no-ri --no-rdoc fpm

build-scripts/ubuntu-1604/build-3rd-parties.sh

@@ -5,6 +5,23 @@ set -x
 
 OUTPUT_PATH=${1:-.}
 
+function build_rocksdb_deb {
+    VERSION=$1
+    VERSION_TAG="rocksdb-$VERSION"
+
+    git clone https://github.com/evernym/rocksdb.git /tmp/rocksdb
+    cd /tmp/rocksdb
+    git checkout $VERSION_TAG
+    sed -i 's/-m [email protected]/-m "Hyperledger <[email protected]>"/g' \
+        ./build_tools/make_package.sh
+    PORTABLE=1 EXTRA_CFLAGS="-fPIC" EXTRA_CXXFLAGS="-fPIC" ./build_tools/make_package.sh $VERSION
+    cp ./package/rocksdb_${VERSION}_amd64.deb $OUTPUT_PATH
+    # Install it in the system as it is needed by python-rocksdb.
+    make install
+    cd -
+    rm -rf /tmp/rocksdb
+}
+
 function build_from_pypi {
     PACKAGE_NAME=$1
 
@@ -17,8 +34,13 @@ function build_from_pypi {
     PREREM_TMP=prerm-${PACKAGE_NAME}
     cp postinst ${POSTINST_TMP}
     cp prerm ${PREREM_TMP}
-    sed -i 's/{package_name}/python3-'${PACKAGE_NAME}'/' ${POSTINST_TMP}
-    sed -i 's/{package_name}/python3-'${PACKAGE_NAME}'/' ${PREREM_TMP}
+    if [[ ${PACKAGE_NAME} =~ ^python-* ]]; then
+        PACKAGE_NAME_TMP="${PACKAGE_NAME/python-/}"
+    else
+        PACKAGE_NAME_TMP=$PACKAGE_NAME
+    fi
+    sed -i 's/{package_name}/python3-'${PACKAGE_NAME_TMP}'/' ${POSTINST_TMP}
+    sed -i 's/{package_name}/python3-'${PACKAGE_NAME_TMP}'/' ${PREREM_TMP}
 
     fpm --input-type "python" \
         --output-type "deb" \
@@ -38,14 +60,21 @@ function build_from_pypi {
     rm ${PREREM_TMP}
 }
 
+# Build rocksdb at first
+build_rocksdb_deb 5.8.8
+
 build_from_pypi ioflo 1.5.4
 build_from_pypi orderedset 2.0
 build_from_pypi base58 0.2.4
 build_from_pypi prompt-toolkit 0.57
 build_from_pypi rlp 0.5.1
 build_from_pypi sha3 0.2.1
-build_from_pypi raet 0.6.6
+build_from_pypi libnacl 1.6.1
+build_from_pypi six 1.11.0
 build_from_pypi pyzmq 16.0.2
 build_from_pypi intervaltree 2.1.0
 build_from_pypi portalocker 0.5.7
 build_from_pypi sortedcontainers 1.5.7
+build_from_pypi setuptools 38.5.2
+# TODO: add libsnappy dependency for python-rocksdb package
+build_from_pypi python-rocksdb 0.6.9

ci/ubuntu.dockerfile

@@ -5,10 +5,18 @@ ARG uid=1000
 ARG user=indy
 ARG venv=venv
 
+RUN echo "To invalidate cache"
+
 RUN apt-get update -y && apt-get install -y \
     python3-nacl \
     libindy-crypto=0.2.0 \
-    libindy
+    libindy=1.3.1~452 \
+# rocksdb python wrapper
+    libbz2-dev \
+    zlib1g-dev \
+    liblz4-dev \
+    libsnappy-dev \
+    rocksdb=5.8.8
 
 RUN indy_ci_add_user $uid $user $venv
 

ledger/ledger.py

@@ -10,16 +10,20 @@
 from ledger.tree_hasher import TreeHasher
 from ledger.util import F, ConsistencyVerificationFailed
 from storage.kv_store import KeyValueStorage
-from storage.kv_store_leveldb_int_keys import KeyValueStorageLeveldbIntKeys
+from storage.helper import initKeyValueStorageIntKeys
+from plenum.common.config_util import getConfig
 
 
 class Ledger(ImmutableStore):
     @staticmethod
     def _defaultStore(dataDir,
                       logName,
                       ensureDurability,
-                      open=True) -> KeyValueStorage:
-        return KeyValueStorageLeveldbIntKeys(dataDir, logName, open)
+                      open=True,
+                      config=None) -> KeyValueStorage:
+        config = config or getConfig()
+        return initKeyValueStorageIntKeys(config.transactionLogDefaultStorage,
+                                          dataDir, logName, open)
 
     def __init__(self,
                  tree: MerkleTree,
@@ -29,7 +33,8 @@ def __init__(self,
                  fileName: str = None,
                  ensureDurability: bool = True,
                  transactionLogStore: KeyValueStorage = None,
-                 genesis_txn_initiator: GenesisTxnInitiator = None):
+                 genesis_txn_initiator: GenesisTxnInitiator = None,
+                 config=None):
         """
         :param tree: an implementation of MerkleTree
         :param dataDir: the directory where the transaction log is stored
@@ -42,6 +47,7 @@ def __init__(self,
 
         self.dataDir = dataDir
         self.tree = tree
+        self.config = config or getConfig()
         self.txn_serializer = txn_serializer or ledger_txn_serializer  # type: MappingSerializer
         # type: MappingSerializer
         self.hash_serializer = hash_serializer or ledger_hash_serializer
@@ -204,7 +210,8 @@ def start(self, loop=None, ensureDurability=True):
                 self._customTransactionLogStore or \
                 self._defaultStore(self.dataDir,
                                    self._transactionLogName,
-                                   ensureDurability)
+                                   ensureDurability,
+                                   config=self.config)
             if self._transactionLog.closed:
                 self._transactionLog.open()
             if self.tree.hashStore.closed:
@@ -222,8 +229,11 @@ def reset(self):
     # TODO: rename getAllTxn to get_txn_slice with required parameters frm to
     # add get_txn_all without args.
     def getAllTxn(self, frm: int = None, to: int = None):
-        yield from ((int(seq_no), self.txn_serializer.deserialize(txn))
-                    for seq_no, txn in self._transactionLog.iterator(start=frm, end=to))
+        for seq_no, txn in self._transactionLog.iterator(start=frm, end=to):
+            if to is None or int(seq_no) <= to:
+                yield (int(seq_no), self.txn_serializer.deserialize(txn))
+            else:
+                break
 
     @staticmethod
     def hashToStr(h):

ledger/test/conftest.py

@@ -7,7 +7,7 @@
 from common.serializers.signing_serializer import SigningSerializer
 from ledger.genesis_txn.genesis_txn_file_util import create_genesis_txn_init_ledger
 from ledger.test.helper import create_ledger, create_ledger_text_file_storage, \
-    create_ledger_chunked_file_storage, create_ledger_leveldb_file_storage
+    create_ledger_chunked_file_storage, create_ledger_leveldb_storage, create_ledger_rocksdb_storage
 
 
 @pytest.fixture(scope='module')
@@ -75,32 +75,38 @@ def hash_serializer(request):
         return CompactSerializer(orderedFields)
 
 
-@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage', 'LeveldbStorage'])
+@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage',
+                                                'LeveldbStorage', 'RocksdbStorage'])
 def ledger(request, genesis_txn_file, tempdir, txn_serializer, hash_serializer):
     ledger = create_ledger(request, txn_serializer,
                            hash_serializer, tempdir, genesis_txn_file)
     yield ledger
     ledger.stop()
 
 
-@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage', 'LeveldbStorage'])
+@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage',
+                                                'LeveldbStorage', 'RocksdbStorage'])
 def create_ledger_callable(request):
     if request.param == 'TextFileStorage':
         return create_ledger_text_file_storage
     elif request.param == 'ChunkedFileStorage':
         return create_ledger_chunked_file_storage
     elif request.param == 'LeveldbStorage':
-        return create_ledger_leveldb_file_storage
+        return create_ledger_leveldb_storage
+    elif request.param == 'RocksdbStorage':
+        return create_ledger_rocksdb_storage
 
 
-@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage', 'LeveldbStorage'])
+@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage',
+                                                'LeveldbStorage', 'RocksdbStorage'])
 def ledger_no_genesis(request, tempdir, txn_serializer, hash_serializer):
     ledger = create_ledger(request, txn_serializer, hash_serializer, tempdir)
     yield ledger
     ledger.stop()
 
 
-@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage', 'LeveldbStorage'])
+@pytest.yield_fixture(scope="function", params=['TextFileStorage', 'ChunkedFileStorage',
+                                                'LeveldbStorage', 'RocksdbStorage'])
 def ledger_with_genesis(request, init_genesis_txn_file, tempdir, txn_serializer, hash_serializer):
     ledger = create_ledger(request, txn_serializer,
                            hash_serializer, tempdir, init_genesis_txn_file)

ledger/test/helper.py

@@ -12,6 +12,7 @@
 from storage.binary_serializer_based_file_store import BinarySerializerBasedFileStore
 from storage.chunked_file_store import ChunkedFileStore
 from storage.kv_store_leveldb_int_keys import KeyValueStorageLeveldbIntKeys
+from storage.kv_store_rocksdb_int_keys import KeyValueStorageRocksdbIntKeys
 from storage.text_file_store import TextFileStore
 
 
@@ -70,7 +71,9 @@ def create_ledger(request, txn_serializer, hash_serializer, tempdir, init_genesi
     elif request.param == 'ChunkedFileStorage':
         return create_ledger_chunked_file_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file)
     elif request.param == 'LeveldbStorage':
-        return create_ledger_leveldb_file_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file)
+        return create_ledger_leveldb_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file)
+    elif request.param == 'RocksdbStorage':
+        return create_ledger_rocksdb_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file)
 
 
 def create_ledger_text_file_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file=None):
@@ -92,12 +95,18 @@ def create_ledger_text_file_storage(txn_serializer, hash_serializer, tempdir, in
     return __create_ledger(store, txn_serializer, hash_serializer, tempdir, init_genesis_txn_file)
 
 
-def create_ledger_leveldb_file_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file=None):
+def create_ledger_leveldb_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file=None):
     store = KeyValueStorageLeveldbIntKeys(tempdir,
                                           'transactions')
     return __create_ledger(store, txn_serializer, hash_serializer, tempdir, init_genesis_txn_file)
 
 
+def create_ledger_rocksdb_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file=None):
+    store = KeyValueStorageRocksdbIntKeys(tempdir,
+                                          'transactions')
+    return __create_ledger(store, txn_serializer, hash_serializer, tempdir, init_genesis_txn_file)
+
+
 def create_ledger_chunked_file_storage(txn_serializer, hash_serializer, tempdir, init_genesis_txn_file=None):
     chunk_creator = None
     db_name = 'transactions'

ledger/test/test_utils.py

@@ -0,0 +1,15 @@
+import random
+
+from ledger.util import has_nth_bit_set
+
+
+def test_nth_bit_set():
+    """
+    Check if bits of randomly chosen integers are set or unset and compare
+    with the binary representation.
+    """
+    for _ in range(0, 10000):
+        number = random.randint(0, 100000000)
+        bits = bin(number)[2:]
+        for i, b in enumerate(reversed(bits)):
+            assert has_nth_bit_set(number, i) == (int(b) == 1)

ledger/util.py

@@ -32,6 +32,16 @@ def highest_bit_set(i):
     return hiBit
 
 
+def has_nth_bit_set(number, n):
+    """
+    Check if a specific bit is set in a number.
+    :param number: The number to check.
+    :param n: The bit if set
+    :return:
+    """
+    return ((1 << n) & number) > 0
+
+
 def highestPowerOf2LessThan(n):
     return n.bit_length() - 1
 

plenum/bls/bls_store.py

@@ -1,5 +1,5 @@
 from common.serializers.serialization import multi_sig_store_serializer
-from plenum.persistence.storage import initKeyValueStorage
+from storage.helper import initKeyValueStorage
 from crypto.bls.bls_multi_signature import MultiSignature
 from typing import Optional
 

plenum/cli/cli.py

@@ -40,7 +40,6 @@
 from stp_core.crypto.util import cleanSeed, seedFromHex
 from stp_core.network.port_dispenser import genHa
 from stp_core.types import HA
-from stp_raet.util import getLocalEstateData
 from plenum.common.config_helper import PNodeConfigHelper
 
 import configparser
@@ -231,20 +230,13 @@ def __init__(self, looper, basedirpath: str, ledger_base_dir: str, nodeReg=None,
             eventloop=eventloop,
             output=out)
 
-        # RAETVerbosity = getRAETLogLevelFromConfig("RAETLogLevelCli",
-        #                                           Console.Wordage.mute,
-        #                                           self.config)
-        # RAETLogFile = getRAETLogFilePath("RAETLogFilePathCli", self.config)
         # Patch stdout in something that will always print *above* the prompt
         # when something is written to stdout.
         sys.stdout = self.cli.stdout_proxy()
 
         if logFileName:
             Logger().enableFileLogging(logFileName)
 
-        # TODO: If we want RAET logging in CLI we need fix this. See INDY-315.
-        # Logger().setupRaet(RAETVerbosity, RAETLogFile)
-
         self.logger = getlogger("cli")
         self.print("\n{}-CLI (c) 2017 Evernym, Inc.".format(self.properName))
         self._actions = []
@@ -1062,8 +1054,7 @@ def newClient(self, clientName,
             if not areKeysSetup(clientName, self.basedirpath):
                 client_addr = genHa(ip='0.0.0.0')
             else:
-                client_addr = tuple(getLocalEstateData(clientName,
-                                                       self.basedirpath)['ha'])
+                raise Exception("Usage of deprecated code")
             nodeReg = None if self.nodeRegLoadedFromFile else self.cliNodeReg
             client = self.ClientClass(clientName,
                                       ha=client_addr,