Merge pull request #1 from merklescience/dev-add-coin

add coin price

.gitignore

@@ -48,3 +48,5 @@ coverage.xml
 .venv
 venv/
 ENV/
+
+.idea/

bitcoinetl/cli/export_all.py

@@ -25,7 +25,7 @@
 import re
 
 from datetime import datetime, timedelta
-from bitcoinetl.enumeration.chain import Chain
+from bitcoinetl.enumeration.chain import Chain, CoinPriceType
 from bitcoinetl.jobs.export_all import export_all as do_export_all
 from bitcoinetl.service.btc_block_range_service import BtcBlockRangeService
 from bitcoinetl.rpc.bitcoin_rpc import BitcoinRpc
@@ -96,7 +96,10 @@ def get_partitions(start, end, partition_batch_size, provider_uri):
 @click.option('-c', '--chain', default=Chain.BITCOIN, type=click.Choice(Chain.ALL),
               help='The type of chain.')
 @click.option('--enrich', default=False, type=bool, help='Enable filling in transactions inputs fields.')
-def export_all(start, end, partition_batch_size, provider_uri, output_dir, max_workers, export_batch_size, chain, enrich):
+@click.option('--coin-price-type', default=CoinPriceType.empty, type=int,
+              help='Enable querying CryptoCompare for coin prices. 0 for no price, 1 for daily price, 2 for hourly price.')
+def export_all(start, end, partition_batch_size, provider_uri, output_dir, max_workers, export_batch_size, chain, enrich, coin_price_type):
     """Exports all data for a range of blocks."""
     do_export_all(chain, get_partitions(start, end, partition_batch_size, provider_uri),
-                  output_dir, provider_uri, max_workers, export_batch_size, enrich)
+                output_dir, provider_uri, max_workers, export_batch_size, enrich,
+                coin_price_type)

bitcoinetl/cli/export_blocks_and_transactions.py

@@ -23,7 +23,7 @@
 
 import click
 
-from bitcoinetl.enumeration.chain import Chain
+from bitcoinetl.enumeration.chain import Chain, CoinPriceType
 from bitcoinetl.jobs.export_blocks_job import ExportBlocksJob
 from bitcoinetl.jobs.exporters.blocks_and_transactions_item_exporter import blocks_and_transactions_item_exporter
 from bitcoinetl.rpc.bitcoin_rpc import BitcoinRpc
@@ -48,8 +48,11 @@
                    'If not provided transactions will not be exported. Use "-" for stdout')
 @click.option('-c', '--chain', default=Chain.BITCOIN, type=click.Choice(Chain.ALL),
               help='The type of chain')
+@click.option('--coin-price-type', default=CoinPriceType.empty, type=int,
+              help='Enable querying CryptoCompare for coin prices. 0 for no price, 1 for daily price, 2 for hourly price.')
 def export_blocks_and_transactions(start_block, end_block, batch_size, provider_uri,
-                                   max_workers, blocks_output, transactions_output, chain):
+                                   max_workers, blocks_output, transactions_output, chain,
+                                   coin_price_type):
     """Export blocks and transactions."""
     if blocks_output is None and transactions_output is None:
         raise ValueError('Either --blocks-output or --transactions-output options must be provided')
@@ -63,5 +66,7 @@ def export_blocks_and_transactions(start_block, end_block, batch_size, provider_
         item_exporter=blocks_and_transactions_item_exporter(blocks_output, transactions_output),
         chain=chain,
         export_blocks=blocks_output is not None,
-        export_transactions=transactions_output is not None)
+        export_transactions=transactions_output is not None,
+        coin_price_type=coin_price_type
+    )
     job.run()

bitcoinetl/domain/block.py

@@ -24,6 +24,7 @@
 
 class BtcBlock(object):
 
+
     def __init__(self):
         self.hash = None
         self.size = None
@@ -36,8 +37,26 @@ def __init__(self):
         self.nonce = None
         self.bits = None
         self.coinbase_param = None
+        self.transaction_count = None
 
         self.transactions = []
 
+        # New fields added
+        self.transaction_ids = []
+
+        self.version_hex = None
+        self.median_timestamp = None
+        self.difficulty = None
+        self.chain_work = None
+        self.previous_block_hash = None
+        self.next_block_hash = None
+        self.input_value = None
+
+        self.block_reward = None
+        self.transaction_fees = None
+        self.coin_price_usd = None
+        self.coinbase_txid = None
+        self.coinbase_param_decoded = None
+
     def has_full_transactions(self):
         return len(self.transactions) > 0 and isinstance(self.transactions[0], BtcTransaction)

bitcoinetl/domain/transaction.py

@@ -43,6 +43,15 @@ def __init__(self):
         self.join_splits = []
         self.value_balance = 0
 
+        # New fields
+        self.transaction_id = None
+        self.weight = None
+        self.input_count = None
+        self.input_value = None
+        self.output_count = None
+        self.output_value = None
+        self.coin_price_usd = None
+
     def add_input(self, input):
         if len(self.inputs) > 0:
             input.index = self.inputs[len(self.inputs) - 1].index + 1

bitcoinetl/domain/transaction_input.py

@@ -23,9 +23,12 @@
 
 class BtcTransactionInput(object):
     def __init__(self):
+        self.create_transaction_id = None
+        self.create_output_index = None
+
+        self.spending_transaction_id = None
         self.index = None
-        self.spent_transaction_hash = None
-        self.spent_output_index = None
+
         self.script_asm = None
         self.script_hex = None
         self.coinbase_param = None
@@ -37,4 +40,4 @@ def __init__(self):
         self.value = None
 
     def is_coinbase(self):
-        return self.coinbase_param is not None or self.spent_transaction_hash is None
+        return self.coinbase_param is not None or self.create_transaction_id is None

bitcoinetl/domain/transaction_output.py

@@ -31,3 +31,7 @@ def __init__(self):
 
         self.addresses = []
         self.value = None
+        self.witness = []
+
+        self.create_transaction_id = None
+        self.spending_transaction_id = None

bitcoinetl/enumeration/chain.py

@@ -10,3 +10,23 @@ class Chain:
     ALL = [BITCOIN, BITCOIN_CASH, DOGECOIN, LITECOIN, DASH, ZCASH, MONACOIN]
     # Old API doesn't support verbosity for getblock which doesn't allow querying all transactions in a block in 1 go.
     HAVE_OLD_API = [BITCOIN_CASH, DOGECOIN, DASH, MONACOIN]
+
+    @classmethod
+    def ticker_symbol(cls, chain):
+        symbols = {
+            'bitcoin': 'BTC',
+            'bitcoin_cash': 'BCH',
+            'dogecoin': 'DOGE',
+            'litecoin': 'LTC',
+            'dash': 'DASH',
+            'zcash': 'ZEC',
+            'monacoin': 'MONA',
+        }
+        return symbols.get(chain, None)
+
+
+class CoinPriceType:
+
+    empty = 0
+    daily = 1
+    hourly = 2

bitcoinetl/jobs/export_all.py

@@ -40,7 +40,10 @@
 logger = logging.getLogger('export_all')
 
 
-def export_all(chain, partitions, output_dir, provider_uri, max_workers, batch_size, enrich):
+def export_all(
+        chain, partitions, output_dir, provider_uri, max_workers, batch_size, enrich,
+        coin_price_type
+    ):
     for batch_start_block, batch_end_block, partition_dir, *args in partitions:
         # # # start # # #
 
@@ -101,7 +104,9 @@ def export_all(chain, partitions, output_dir, provider_uri, max_workers, batch_s
             max_workers=max_workers,
             item_exporter=blocks_and_transactions_item_exporter(blocks_file, transactions_file),
             export_blocks=blocks_file is not None,
-            export_transactions=transactions_file is not None)
+            export_transactions=transactions_file is not None,
+            coin_price_type=coin_price_type,
+            )
         job.run()
 
         if enrich == True:

bitcoinetl/jobs/export_blocks_job.py

@@ -27,6 +27,7 @@
 from blockchainetl.executors.batch_work_executor import BatchWorkExecutor
 from blockchainetl.jobs.base_job import BaseJob
 from blockchainetl.utils import validate_range
+from bitcoinetl.enumeration.chain import CoinPriceType
 
 
 # Exports blocks and transactions
@@ -41,8 +42,10 @@ def __init__(
             item_exporter,
             chain,
             export_blocks=True,
-            export_transactions=True):
+            export_transactions=True,
+            coin_price_type=CoinPriceType.empty):
         validate_range(start_block, end_block)
+
         self.start_block = start_block
         self.end_block = end_block
 
@@ -54,7 +57,7 @@ def __init__(
         if not self.export_blocks and not self.export_transactions:
             raise ValueError('At least one of export_blocks or export_transactions must be True')
 
-        self.btc_service = BtcService(bitcoin_rpc, chain)
+        self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type)
         self.block_mapper = BtcBlockMapper()
         self.transaction_mapper = BtcTransactionMapper()
 

bitcoinetl/jobs/exporters/blocks_and_transactions_item_exporter.py

@@ -23,41 +23,56 @@
 
 from blockchainetl.jobs.exporters.composite_item_exporter import CompositeItemExporter
 
+
 BLOCK_FIELDS_TO_EXPORT = [
-    'hash',
-    'size',
-    'stripped_size',
-    'weight',
-    'number',
-    'version',
-    'merkle_root',
-    'timestamp',
-    'nonce',
-    'bits',
-    'coinbase_param',
-    'transaction_count'
+    "hash",
+    "number",
+    "timestamp",
+    "median_timestamp",
+    "merkle_root",
+    "coinbase_param",
+    "coinbase_param_decoded",
+    "coinbase_txid",
+    "previous_block_hash",
+    "next_block_hash",
+    "nonce",
+    "difficulty",
+    "chain_work",
+    "version",
+    "version_hex",
+    "size",
+    "stripped_size",
+    "weight",
+    "bits",
+    "transaction_count",
+    "transaction_fees",
+    "block_reward",
+    "input_value",
+    "transaction_ids",
+    "coin_price_usd",
 ]
 
+
 TRANSACTION_FIELDS_TO_EXPORT = [
+    'transaction_id',
     'hash',
-    'size',
-    'virtual_size',
-    'version',
-    'lock_time',
     'block_number',
     'block_hash',
     'block_timestamp',
     'is_coinbase',
+    'lock_time',
+    'size',
+    'virtual_size',
+    'weight',
+    'version',
     'index',
-
-    'inputs',
-    'outputs',
-
     'input_count',
     'output_count',
     'input_value',
     'output_value',
-    'fee'
+    'inputs',
+    'outputs',
+    'coin_price_usd',
 ]
 
 

bitcoinetl/mappers/block_mapper.py

@@ -58,6 +58,16 @@ def json_dict_to_block(self, json_dict):
 
             block.transaction_count = len(raw_transactions)
 
+        # New fields
+        block.transaction_count = json_dict.get("nTx")
+        block.version_hex = json_dict.get("versionHex")
+        block.median_timestamp = json_dict.get("mediantime")
+        block.difficulty = int(json_dict.get("difficulty"))
+        block.chain_work = json_dict.get("chainwork")
+        block.coinbase_txid = json_dict.get("coinbase_txid")
+        block.previous_block_hash = json_dict.get("previousblockhash")
+        block.coin_price_usd = json_dict.get('coin_price_usd')
+        block.transaction_ids = [tx.transaction_id for tx in block.transactions]
         return block
 
     def block_to_dict(self, block):
@@ -74,7 +84,17 @@ def block_to_dict(self, block):
             'nonce': block.nonce,
             'bits': block.bits,
             'coinbase_param': block.coinbase_param,
-            'transaction_count': len(block.transactions)
+            'coinbase_param_decoded': block.coinbase_param_decoded,
+            'coinbase_txid': block.coinbase_txid,
+            'transaction_count': block.transaction_count,
+            'block_reward': block.block_reward,
+            'version_hex': block.version_hex,
+            'median_timestamp': block.median_timestamp,
+            'difficulty': block.difficulty,
+            'chain_work': block.chain_work,
+            'previous_block_hash': block.previous_block_hash,
+            "coin_price_usd": block.coin_price_usd,
+            "transaction_ids": block.transaction_ids
         }