diff --git a/.gitignore b/.gitignore index 77a4b16..ed2a59a 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ coverage.xml .venv venv/ ENV/ + +.idea/ diff --git a/bitcoinetl/cli/export_all.py b/bitcoinetl/cli/export_all.py index e235f9a..2628e8a 100644 --- a/bitcoinetl/cli/export_all.py +++ b/bitcoinetl/cli/export_all.py @@ -25,7 +25,7 @@ import re from datetime import datetime, timedelta -from bitcoinetl.enumeration.chain import Chain +from bitcoinetl.enumeration.chain import Chain, CoinPriceType from bitcoinetl.jobs.export_all import export_all as do_export_all from bitcoinetl.service.btc_block_range_service import BtcBlockRangeService from bitcoinetl.rpc.bitcoin_rpc import BitcoinRpc @@ -96,7 +96,10 @@ def get_partitions(start, end, partition_batch_size, provider_uri): @click.option('-c', '--chain', default=Chain.BITCOIN, type=click.Choice(Chain.ALL), help='The type of chain.') @click.option('--enrich', default=False, type=bool, help='Enable filling in transactions inputs fields.') -def export_all(start, end, partition_batch_size, provider_uri, output_dir, max_workers, export_batch_size, chain, enrich): +@click.option('--coin-price-type', default=CoinPriceType.empty, type=int, + help='Enable querying CryptoCompare for coin prices. 0 for no price, 1 for daily price, 2 for hourly price.') +def export_all(start, end, partition_batch_size, provider_uri, output_dir, max_workers, export_batch_size, chain, enrich, coin_price_type): """Exports all data for a range of blocks.""" do_export_all(chain, get_partitions(start, end, partition_batch_size, provider_uri), - output_dir, provider_uri, max_workers, export_batch_size, enrich) + output_dir, provider_uri, max_workers, export_batch_size, enrich, + coin_price_type) diff --git a/bitcoinetl/cli/export_blocks_and_transactions.py b/bitcoinetl/cli/export_blocks_and_transactions.py index 1ab1a09..9d75e08 100644 --- a/bitcoinetl/cli/export_blocks_and_transactions.py +++ b/bitcoinetl/cli/export_blocks_and_transactions.py @@ -23,7 +23,7 @@ import click -from bitcoinetl.enumeration.chain import Chain +from bitcoinetl.enumeration.chain import Chain, CoinPriceType from bitcoinetl.jobs.export_blocks_job import ExportBlocksJob from bitcoinetl.jobs.exporters.blocks_and_transactions_item_exporter import blocks_and_transactions_item_exporter from bitcoinetl.rpc.bitcoin_rpc import BitcoinRpc @@ -48,8 +48,11 @@ 'If not provided transactions will not be exported. Use "-" for stdout') @click.option('-c', '--chain', default=Chain.BITCOIN, type=click.Choice(Chain.ALL), help='The type of chain') +@click.option('--coin-price-type', default=CoinPriceType.empty, type=int, + help='Enable querying CryptoCompare for coin prices. 0 for no price, 1 for daily price, 2 for hourly price.') def export_blocks_and_transactions(start_block, end_block, batch_size, provider_uri, - max_workers, blocks_output, transactions_output, chain): + max_workers, blocks_output, transactions_output, chain, + coin_price_type): """Export blocks and transactions.""" if blocks_output is None and transactions_output is None: raise ValueError('Either --blocks-output or --transactions-output options must be provided') @@ -63,5 +66,7 @@ def export_blocks_and_transactions(start_block, end_block, batch_size, provider_ item_exporter=blocks_and_transactions_item_exporter(blocks_output, transactions_output), chain=chain, export_blocks=blocks_output is not None, - export_transactions=transactions_output is not None) + export_transactions=transactions_output is not None, + coin_price_type=coin_price_type + ) job.run() diff --git a/bitcoinetl/domain/block.py b/bitcoinetl/domain/block.py index 97a8219..5e5ee7d 100644 --- a/bitcoinetl/domain/block.py +++ b/bitcoinetl/domain/block.py @@ -24,6 +24,7 @@ class BtcBlock(object): + def __init__(self): self.hash = None self.size = None @@ -36,8 +37,26 @@ def __init__(self): self.nonce = None self.bits = None self.coinbase_param = None + self.transaction_count = None self.transactions = [] + # New fields added + self.transaction_ids = [] + + self.version_hex = None + self.median_timestamp = None + self.difficulty = None + self.chain_work = None + self.previous_block_hash = None + self.next_block_hash = None + self.input_value = None + + self.block_reward = None + self.transaction_fees = None + self.coin_price_usd = None + self.coinbase_txid = None + self.coinbase_param_decoded = None + def has_full_transactions(self): return len(self.transactions) > 0 and isinstance(self.transactions[0], BtcTransaction) diff --git a/bitcoinetl/domain/transaction.py b/bitcoinetl/domain/transaction.py index ef56275..de66cad 100644 --- a/bitcoinetl/domain/transaction.py +++ b/bitcoinetl/domain/transaction.py @@ -43,6 +43,15 @@ def __init__(self): self.join_splits = [] self.value_balance = 0 + # New fields + self.transaction_id = None + self.weight = None + self.input_count = None + self.input_value = None + self.output_count = None + self.output_value = None + self.coin_price_usd = None + def add_input(self, input): if len(self.inputs) > 0: input.index = self.inputs[len(self.inputs) - 1].index + 1 diff --git a/bitcoinetl/domain/transaction_input.py b/bitcoinetl/domain/transaction_input.py index 95a3bf6..d2a39d8 100644 --- a/bitcoinetl/domain/transaction_input.py +++ b/bitcoinetl/domain/transaction_input.py @@ -23,9 +23,12 @@ class BtcTransactionInput(object): def __init__(self): + self.create_transaction_id = None + self.create_output_index = None + + self.spending_transaction_id = None self.index = None - self.spent_transaction_hash = None - self.spent_output_index = None + self.script_asm = None self.script_hex = None self.coinbase_param = None @@ -37,4 +40,4 @@ def __init__(self): self.value = None def is_coinbase(self): - return self.coinbase_param is not None or self.spent_transaction_hash is None + return self.coinbase_param is not None or self.create_transaction_id is None diff --git a/bitcoinetl/domain/transaction_output.py b/bitcoinetl/domain/transaction_output.py index 2d3b2ca..f5c53bf 100644 --- a/bitcoinetl/domain/transaction_output.py +++ b/bitcoinetl/domain/transaction_output.py @@ -31,3 +31,7 @@ def __init__(self): self.addresses = [] self.value = None + self.witness = [] + + self.create_transaction_id = None + self.spending_transaction_id = None diff --git a/bitcoinetl/enumeration/chain.py b/bitcoinetl/enumeration/chain.py index d54728b..f115d9b 100644 --- a/bitcoinetl/enumeration/chain.py +++ b/bitcoinetl/enumeration/chain.py @@ -10,3 +10,23 @@ class Chain: ALL = [BITCOIN, BITCOIN_CASH, DOGECOIN, LITECOIN, DASH, ZCASH, MONACOIN] # Old API doesn't support verbosity for getblock which doesn't allow querying all transactions in a block in 1 go. HAVE_OLD_API = [BITCOIN_CASH, DOGECOIN, DASH, MONACOIN] + + @classmethod + def ticker_symbol(cls, chain): + symbols = { + 'bitcoin': 'BTC', + 'bitcoin_cash': 'BCH', + 'dogecoin': 'DOGE', + 'litecoin': 'LTC', + 'dash': 'DASH', + 'zcash': 'ZEC', + 'monacoin': 'MONA', + } + return symbols.get(chain, None) + + +class CoinPriceType: + + empty = 0 + daily = 1 + hourly = 2 diff --git a/bitcoinetl/jobs/export_all.py b/bitcoinetl/jobs/export_all.py index 8a33591..e3a75d2 100644 --- a/bitcoinetl/jobs/export_all.py +++ b/bitcoinetl/jobs/export_all.py @@ -40,7 +40,10 @@ logger = logging.getLogger('export_all') -def export_all(chain, partitions, output_dir, provider_uri, max_workers, batch_size, enrich): +def export_all( + chain, partitions, output_dir, provider_uri, max_workers, batch_size, enrich, + coin_price_type + ): for batch_start_block, batch_end_block, partition_dir, *args in partitions: # # # start # # # @@ -101,7 +104,9 @@ def export_all(chain, partitions, output_dir, provider_uri, max_workers, batch_s max_workers=max_workers, item_exporter=blocks_and_transactions_item_exporter(blocks_file, transactions_file), export_blocks=blocks_file is not None, - export_transactions=transactions_file is not None) + export_transactions=transactions_file is not None, + coin_price_type=coin_price_type, + ) job.run() if enrich == True: diff --git a/bitcoinetl/jobs/export_blocks_job.py b/bitcoinetl/jobs/export_blocks_job.py index c4b384c..e19b2a1 100644 --- a/bitcoinetl/jobs/export_blocks_job.py +++ b/bitcoinetl/jobs/export_blocks_job.py @@ -27,6 +27,7 @@ from blockchainetl.executors.batch_work_executor import BatchWorkExecutor from blockchainetl.jobs.base_job import BaseJob from blockchainetl.utils import validate_range +from bitcoinetl.enumeration.chain import CoinPriceType # Exports blocks and transactions @@ -41,8 +42,10 @@ def __init__( item_exporter, chain, export_blocks=True, - export_transactions=True): + export_transactions=True, + coin_price_type=CoinPriceType.empty): validate_range(start_block, end_block) + self.start_block = start_block self.end_block = end_block @@ -54,7 +57,7 @@ def __init__( if not self.export_blocks and not self.export_transactions: raise ValueError('At least one of export_blocks or export_transactions must be True') - self.btc_service = BtcService(bitcoin_rpc, chain) + self.btc_service = BtcService(bitcoin_rpc, chain, coin_price_type) self.block_mapper = BtcBlockMapper() self.transaction_mapper = BtcTransactionMapper() diff --git a/bitcoinetl/jobs/exporters/blocks_and_transactions_item_exporter.py b/bitcoinetl/jobs/exporters/blocks_and_transactions_item_exporter.py index c1e8f0e..764a5ad 100644 --- a/bitcoinetl/jobs/exporters/blocks_and_transactions_item_exporter.py +++ b/bitcoinetl/jobs/exporters/blocks_and_transactions_item_exporter.py @@ -23,41 +23,56 @@ from blockchainetl.jobs.exporters.composite_item_exporter import CompositeItemExporter + BLOCK_FIELDS_TO_EXPORT = [ - 'hash', - 'size', - 'stripped_size', - 'weight', - 'number', - 'version', - 'merkle_root', - 'timestamp', - 'nonce', - 'bits', - 'coinbase_param', - 'transaction_count' + "hash", + "number", + "timestamp", + "median_timestamp", + "merkle_root", + "coinbase_param", + "coinbase_param_decoded", + "coinbase_txid", + "previous_block_hash", + "next_block_hash", + "nonce", + "difficulty", + "chain_work", + "version", + "version_hex", + "size", + "stripped_size", + "weight", + "bits", + "transaction_count", + "transaction_fees", + "block_reward", + "input_value", + "transaction_ids", + "coin_price_usd", ] + TRANSACTION_FIELDS_TO_EXPORT = [ + 'transaction_id', 'hash', - 'size', - 'virtual_size', - 'version', - 'lock_time', 'block_number', 'block_hash', 'block_timestamp', 'is_coinbase', + 'lock_time', + 'size', + 'virtual_size', + 'weight', + 'version', 'index', - - 'inputs', - 'outputs', - 'input_count', 'output_count', 'input_value', 'output_value', - 'fee' + 'inputs', + 'outputs', + 'coin_price_usd', ] diff --git a/bitcoinetl/mappers/block_mapper.py b/bitcoinetl/mappers/block_mapper.py index fce6095..806d391 100644 --- a/bitcoinetl/mappers/block_mapper.py +++ b/bitcoinetl/mappers/block_mapper.py @@ -58,6 +58,16 @@ def json_dict_to_block(self, json_dict): block.transaction_count = len(raw_transactions) + # New fields + block.transaction_count = json_dict.get("nTx") + block.version_hex = json_dict.get("versionHex") + block.median_timestamp = json_dict.get("mediantime") + block.difficulty = int(json_dict.get("difficulty")) + block.chain_work = json_dict.get("chainwork") + block.coinbase_txid = json_dict.get("coinbase_txid") + block.previous_block_hash = json_dict.get("previousblockhash") + block.coin_price_usd = json_dict.get('coin_price_usd') + block.transaction_ids = [tx.transaction_id for tx in block.transactions] return block def block_to_dict(self, block): @@ -74,7 +84,17 @@ def block_to_dict(self, block): 'nonce': block.nonce, 'bits': block.bits, 'coinbase_param': block.coinbase_param, - 'transaction_count': len(block.transactions) + 'coinbase_param_decoded': block.coinbase_param_decoded, + 'coinbase_txid': block.coinbase_txid, + 'transaction_count': block.transaction_count, + 'block_reward': block.block_reward, + 'version_hex': block.version_hex, + 'median_timestamp': block.median_timestamp, + 'difficulty': block.difficulty, + 'chain_work': block.chain_work, + 'previous_block_hash': block.previous_block_hash, + "coin_price_usd": block.coin_price_usd, + "transaction_ids": block.transaction_ids } diff --git a/bitcoinetl/mappers/transaction_input_mapper.py b/bitcoinetl/mappers/transaction_input_mapper.py index 9d58058..468ed04 100644 --- a/bitcoinetl/mappers/transaction_input_mapper.py +++ b/bitcoinetl/mappers/transaction_input_mapper.py @@ -25,24 +25,28 @@ class BtcTransactionInputMapper(object): - def vin_to_inputs(self, vin): + def vin_to_inputs(self, vin, spending_transaction_id=None): inputs = [] index = 0 for item in (vin or []): - input = self.json_dict_to_input(item) + input = self.json_dict_to_input(json_dict=item, spending_transaction_id=spending_transaction_id) input.index = index index = index + 1 inputs.append(input) return inputs - def json_dict_to_input(self, json_dict): + def json_dict_to_input(self, json_dict, spending_transaction_id=None): input = BtcTransactionInput() - input.spent_transaction_hash = json_dict.get('txid') - input.spent_output_index = json_dict.get('vout') + input.create_transaction_id = json_dict.get('txid') + input.create_output_index = json_dict.get('vout') + + input.spending_transaction_id = spending_transaction_id + input.coinbase_param = json_dict.get('coinbase') input.sequence = json_dict.get('sequence') + if 'scriptSig' in json_dict: input.script_asm = (json_dict.get('scriptSig')).get('asm') input.script_hex = (json_dict.get('scriptSig')).get('hex') @@ -54,16 +58,21 @@ def inputs_to_dicts(self, inputs): for input in inputs: item = { 'index': input.index, - 'spent_transaction_hash': input.spent_transaction_hash, - 'spent_output_index': input.spent_output_index, + 'create_transaction_id': input.create_transaction_id, + 'spending_transaction_id': input.spending_transaction_id, + 'create_output_index': input.create_output_index, + 'sequence': input.sequence, + 'script_asm': input.script_asm, 'script_hex': input.script_hex, - 'sequence': input.sequence, + 'required_signatures': input.required_signatures, - 'type': input.type, 'addresses': input.addresses, 'value': input.value, + 'type': input.type, } + if input.coinbase_param: + item['coinbase_param'] = input.coinbase_param result.append(item) return result @@ -72,8 +81,8 @@ def dicts_to_inputs(self, dicts): for dict in dicts: input = BtcTransactionInput() input.index = dict.get('index') - input.spent_transaction_hash = dict.get('spent_transaction_hash') - input.spent_output_index = dict.get('spent_output_index') + input.create_transaction_id = dict.get('create_transaction_id') + input.create_output_index = dict.get('create_output_index') input.script_asm = dict.get('script_asm') input.script_hex = dict.get('script_hex') input.sequence = dict.get('sequence') @@ -81,6 +90,7 @@ def dicts_to_inputs(self, dicts): input.type = dict.get('type') input.addresses = dict.get('addresses') input.value = dict.get('value') + input.spending_transaction_id = dict.get('spending_transaction_id') result.append(input) return result diff --git a/bitcoinetl/mappers/transaction_mapper.py b/bitcoinetl/mappers/transaction_mapper.py index 93f2f29..c1094b6 100644 --- a/bitcoinetl/mappers/transaction_mapper.py +++ b/bitcoinetl/mappers/transaction_mapper.py @@ -36,13 +36,14 @@ def __init__(self): self.transaction_output_mapper = BtcTransactionOutputMapper() self.join_split_mapper = BtcJoinSplitMapper() - def json_dict_to_transaction(self, json_dict, block=None, index=None): + def json_dict_to_transaction(self, json_dict, block=None, index=None, coin_price_usd=None): transaction = BtcTransaction() - transaction.hash = json_dict.get('txid') + transaction.hash = json_dict.get('hash') transaction.size = json_dict.get('size') transaction.virtual_size = json_dict.get('vsize') transaction.version = json_dict.get('version') transaction.lock_time = json_dict.get('locktime') + transaction.transaction_id = json_dict.get('txid') if block is not None: transaction.block_number = block.number @@ -58,19 +59,33 @@ def json_dict_to_transaction(self, json_dict, block=None, index=None): if index is not None: transaction.index = index - transaction.inputs = self.transaction_input_mapper.vin_to_inputs(json_dict.get('vin')) - transaction.outputs = self.transaction_output_mapper.vout_to_outputs(json_dict.get('vout')) + transaction.inputs = self.transaction_input_mapper.vin_to_inputs( + vin=json_dict.get('vin'), + spending_transaction_id=transaction.transaction_id + ) + transaction.outputs = self.transaction_output_mapper.vout_to_outputs( + vout=json_dict.get('vout'), + create_transaction_id=transaction.transaction_id + ) # Only Zcash transaction.join_splits = self.join_split_mapper.vjoinsplit_to_join_splits(json_dict.get('vjoinsplit')) transaction.value_balance = bitcoin_to_satoshi(json_dict.get('valueBalance')) + # New fields + transaction.coin_price_usd = coin_price_usd + transaction.weight = json_dict.get('weight') + transaction.output_addresses = self.get_output_addresses(transaction) return transaction + def get_output_addresses(self, transaction): + return [','.join(output.addresses) if output.addresses else output.addresses for output in transaction.outputs] + def transaction_to_dict(self, transaction): result = { 'type': 'transaction', 'hash': transaction.hash, + 'transaction_id': transaction.transaction_id, 'size': transaction.size, 'virtual_size': transaction.virtual_size, 'version': transaction.version, @@ -89,12 +104,16 @@ def transaction_to_dict(self, transaction): 'input_value': transaction.calculate_input_value(), 'output_value': transaction.calculate_output_value(), 'fee': transaction.calculate_fee(), + 'coin_price_usd': transaction.coin_price_usd, + 'weight': transaction.weight, + 'output_addresses': transaction.output_addresses } return result def dict_to_transaction(self, dict): transaction = BtcTransaction() transaction.hash = dict.get('hash') + transaction.transaction_id = dict.get('transaction_id') transaction.size = dict.get('size') transaction.virtual_size = dict.get('virtual_size') transaction.version = dict.get('version') @@ -104,6 +123,15 @@ def dict_to_transaction(self, dict): transaction.block_timestamp = dict.get('block_timestamp') transaction.is_coinbase = dict.get('is_coinbase') transaction.index = dict.get('index') + transaction.coin_price_usd = dict.get('coin_price_usd') + transaction.weight = dict.get('weight') + transaction.output_addresses = dict.get('output_addresses') + transaction.input_addresses = dict.get('input_addresses') + transaction.input_count = dict.get('input_count') + transaction.input_value = dict.get('input_value') + transaction.output_count = dict.get('output_count') + transaction.output_value = dict.get('output_value') + transaction.fee = dict.get('fee') transaction.inputs = self.transaction_input_mapper.dicts_to_inputs(dict.get('inputs')) transaction.outputs = self.transaction_output_mapper.dicts_to_outputs(dict.get('outputs')) diff --git a/bitcoinetl/mappers/transaction_output_mapper.py b/bitcoinetl/mappers/transaction_output_mapper.py index 6974006..1466db5 100644 --- a/bitcoinetl/mappers/transaction_output_mapper.py +++ b/bitcoinetl/mappers/transaction_output_mapper.py @@ -26,20 +26,22 @@ class BtcTransactionOutputMapper(object): - def vout_to_outputs(self, vout): + def vout_to_outputs(self, vout, create_transaction_id=None): outputs = [] for item in (vout or []): - output = self.json_dict_to_output(item) + output = self.json_dict_to_output(json_dict=item, create_transaction_id=create_transaction_id) outputs.append(output) return outputs - def json_dict_to_output(self, json_dict): + def json_dict_to_output(self, json_dict, create_transaction_id=None): output = BtcTransactionOutput() output.index = json_dict.get('n') output.addresses = json_dict.get('addresses') output.txinwitness = json_dict.get('txinwitness') output.value = bitcoin_to_satoshi(json_dict.get('value')) + output.create_transaction_id = create_transaction_id + if 'scriptPubKey' in json_dict: script_pub_key = json_dict.get('scriptPubKey') output.script_asm = script_pub_key.get('asm') @@ -55,13 +57,20 @@ def outputs_to_dicts(self, outputs): for output in outputs: item = { 'index': output.index, + 'create_transaction_id': output.create_transaction_id, + 'spending_transaction_id': None, + 'script_asm': output.script_asm, 'script_hex': output.script_hex, - 'required_signatures': output.required_signatures, + 'type': output.type, 'addresses': output.addresses, - 'value': output.value + 'value': output.value, + 'required_signatures': output.required_signatures, } + if output.txinwitness: + item['witness'] = output.txinwitness + result.append(item) return result @@ -76,6 +85,9 @@ def dicts_to_outputs(self, dicts): input.type = dict.get('type') input.addresses = dict.get('addresses') input.value = dict.get('value') + input.witness = dict.get('witness') + input.create_transaction_id = dict.get('create_transaction_id') + input.spending_transaction_id = dict.get('spending_transaction_id') result.append(input) return result diff --git a/bitcoinetl/service/btc_service.py b/bitcoinetl/service/btc_service.py index d768618..0f71b94 100644 --- a/bitcoinetl/service/btc_service.py +++ b/bitcoinetl/service/btc_service.py @@ -22,7 +22,7 @@ from bitcoinetl.domain.transaction_input import BtcTransactionInput from bitcoinetl.domain.transaction_output import BtcTransactionOutput -from bitcoinetl.enumeration.chain import Chain +from bitcoinetl.enumeration.chain import Chain, CoinPriceType from bitcoinetl.json_rpc_requests import generate_get_block_hash_by_number_json_rpc, \ generate_get_block_by_hash_json_rpc, generate_get_transaction_by_id_json_rpc from bitcoinetl.mappers.block_mapper import BtcBlockMapper @@ -30,14 +30,22 @@ from bitcoinetl.service.btc_script_service import script_hex_to_non_standard_address from bitcoinetl.service.genesis_transactions import GENESIS_TRANSACTIONS from blockchainetl.utils import rpc_response_batch_to_results, dynamic_batch_iterator +from blockchainetl.cryptocompare import ( + get_coin_price, + get_hour_id_from_ts, + get_day_id_from_ts, + get_ts_from_hour_id, + get_ts_from_day_id +) class BtcService(object): - def __init__(self, bitcoin_rpc, chain=Chain.BITCOIN): + def __init__(self, bitcoin_rpc, chain=Chain.BITCOIN, coin_price_type=CoinPriceType.empty): self.bitcoin_rpc = bitcoin_rpc self.block_mapper = BtcBlockMapper() self.transaction_mapper = BtcTransactionMapper() self.chain = chain + self.coin_price_type = coin_price_type def get_block(self, block_number, with_transactions=False): block_hashes = self.get_block_hashes([block_number]) @@ -73,10 +81,14 @@ def get_blocks_by_hashes(self, block_hash_batch, with_transactions=True): if self.chain in Chain.HAVE_OLD_API and with_transactions: self._fetch_transactions(blocks) + self._add_coin_price_to_blocks(blocks, self.coin_price_type) + for block in blocks: self._remove_coinbase_input(block) + if block.has_full_transactions(): for transaction in block.transactions: + self._add_coin_price_to_transaction(transaction, block.coin_price_usd) self._add_non_standard_addresses(transaction) if self.chain == Chain.ZCASH: self._add_shielded_inputs_and_outputs(transaction) @@ -144,6 +156,7 @@ def _remove_coinbase_input(self, block): if block.has_full_transactions(): for transaction in block.transactions: coinbase_inputs = [input for input in transaction.inputs if input.is_coinbase()] + if len(coinbase_inputs) > 1: raise ValueError('There must be no more than 1 coinbase input in any transaction. Was {}, hash {}' .format(len(coinbase_inputs), transaction.hash)) @@ -153,6 +166,14 @@ def _remove_coinbase_input(self, block): transaction.inputs = [input for input in transaction.inputs if not input.is_coinbase()] transaction.is_coinbase = True + block.coinbase_param = coinbase_input.coinbase_param + block.coinbase_param_decoded = bytes.fromhex(coinbase_input.coinbase_param).decode('utf-8', 'replace') + block.coinbase_tx = transaction + block.coinbase_txid = transaction.transaction_id + + block.block_reward = self.get_block_reward(block) + transaction.input_count = 0 + def _add_non_standard_addresses(self, transaction): for output in transaction.outputs: if output.addresses is None or len(output.addresses) == 0: @@ -186,5 +207,41 @@ def _add_shielded_inputs_and_outputs(self, transaction): output.value = -transaction.value_balance transaction.add_output(output) + def get_block_reward(self, block): + return block.coinbase_tx.calculate_output_value() + + def _add_coin_price_to_blocks(self, blocks, coin_price_type): + from_currency_code = Chain.ticker_symbol(self.chain) + + if not from_currency_code or coin_price_type == CoinPriceType.empty: + return + + elif coin_price_type == CoinPriceType.hourly: + block_hour_ids = list(set([get_hour_id_from_ts(block.timestamp) for block in blocks])) + block_hours_ts = {hour_id: get_ts_from_hour_id(hour_id) for hour_id in block_hour_ids} + coin_price_hours = { + hour_id: get_coin_price(from_currency_code=from_currency_code, timestamp=hour_ts, resource="histohour") + for hour_id, hour_ts in block_hours_ts.items() + } + + for block in blocks: + block_hour_id = get_hour_id_from_ts(block.timestamp) + block.coin_price_usd = coin_price_hours[block_hour_id] + + elif coin_price_type == CoinPriceType.daily: + block_day_ids = list(set([get_day_id_from_ts(block.timestamp) for block in blocks])) + block_days_ts = {day_id: get_ts_from_day_id(day_id) for day_id in block_day_ids} + coin_price_days = { + day_id: get_coin_price(from_currency_code=from_currency_code, timestamp=day_ts, resource="histoday") + for day_id, day_ts in block_days_ts.items() + } + + for block in blocks: + block_day_id = get_day_id_from_ts(block.timestamp) + block.coin_price_usd = coin_price_days[block_day_id] + + def _add_coin_price_to_transaction(self, transaction, coin_price_usd): + transaction.coin_price_usd = coin_price_usd + ADDRESS_TYPE_SHIELDED = 'shielded' diff --git a/blockchainetl/cryptocompare.py b/blockchainetl/cryptocompare.py new file mode 100644 index 0000000..ffd835d --- /dev/null +++ b/blockchainetl/cryptocompare.py @@ -0,0 +1,125 @@ +# MIT License +# +# Copyright (c) 2019 Nirmal AK, nirmal@merklescience.com +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import os +import requests +from time import time +from math import floor +from datetime import datetime, timedelta + + +CRYPTOCOMPARE_API_KEY = os.getenv("CRYPTOCOMPARE_API_KEY", "") + + +class CryptoCompareRequestException(Exception): + pass + + +def get_hour_id_from_ts(timestamp: int) -> int: + """ + returns the number of hours elapsed since 1st Jan 2000 + """ + base_ts = datetime(2000, 1, 1).timestamp() + seconds_to_hour = 60 * 60 + return floor((int(timestamp) - base_ts) / seconds_to_hour) + + +def get_day_id_from_ts(timestamp: int) -> int: + """ + returns the number of days elapsed since 1st Jan 2000 + """ + base_ts = datetime(2000, 1, 1).timestamp() + seconds_to_day = 60 * 60 * 24 + return floor((int(timestamp) - base_ts) / seconds_to_day) + + +def get_ts_from_hour_id(hour_id: int) -> int: + base_date = datetime(2000, 1, 1) + reference_date = base_date + timedelta(hours=hour_id) + return floor(reference_date.timestamp()) + + +def get_ts_from_day_id(day_id: int) -> int: + base_date = datetime(2000, 1, 1) + reference_date = base_date + timedelta(days=day_id) + return floor(reference_date.timestamp()) + + +def _make_request( + resource: str, + from_currency_code: str, + to_currency_code: str, + timestamp: int, + access_token: str, + exchange_code: str, + num_records: int, + api_version: str + ) -> requests.Response: + """ + API documentation for cryptocompare can be found at https://min-api.cryptocompare.com/documentation + """ + base_url = f"https://min-api.cryptocompare.com/data/{api_version}/{resource}" + params = { + "fsym": from_currency_code, + "tsym": to_currency_code, + "e": exchange_code, + "limit": num_records, + "toTs": timestamp, + "api_key": access_token + } + return requests.get(base_url, params=params) + + +def get_coin_price( + from_currency_code: str, + timestamp: int, + resource="histohour", + to_currency_code: str="USD", + exchange_code: str="CCCAGG", + num_records: int=1, + api_version: str ="v2", + access_token: str=CRYPTOCOMPARE_API_KEY, + ): + """ + Prices are retrieved from hourly price resource as prices + are available for historical data from when available + """ + response = _make_request( + resource=resource, + from_currency_code=from_currency_code, + to_currency_code=to_currency_code, + timestamp=int(timestamp), + access_token=access_token, + exchange_code=exchange_code, + num_records=num_records, + api_version=api_version, + ) + if not response.status_code == 200: + raise CryptoCompareRequestException + + payload = response.json() + if payload["Type"] != 100: + raise CryptoCompareRequestException(payload.get("Message", "")) + + data = payload["Data"]["Data"] + avg_price = sum(item["open"] for item in data) / len(data) + return round(avg_price, 2)