Code arrangement refactor. Optimizes utxo_parser (output file 70-ish …

…% smaller).

bitcoin_tools/plots.py → bitcoin_tools/analysis/plots.py

@@ -1,10 +1,11 @@
-import numpy as np
-import matplotlib.pyplot as plt
-import matplotlib as mpl
 from json import loads, dumps
-from bitcoin_tools.utils import load_conf_file
-from bitcoin_tools.utxo_dump import accumulate_dust
 
+import matplotlib as mpl
+import matplotlib.pyplot as plt
+import numpy as np
+
+from bitcoin_tools.utils import load_conf_file
+from bitcoin_tools.utxo.utxo_dump import accumulate_dust
 
 label_size = 12
 mpl.rcParams['xtick.labelsize'] = label_size
@@ -98,7 +99,7 @@ def plot_distribution(xs, ys, title, xlabel, ylabel, log_axis=False, save_fig=Fa
 
     # Output result
     if save_fig:
-        plt.savefig(cfg.figs_path + '/' + save_fig + '.pdf', format='pdf', dpi=600)
+        plt.savefig(cfg.figs_path + save_fig + '.pdf', format='pdf', dpi=600)
     else:
         plt.show()
 
@@ -176,32 +177,35 @@ def plot_from_file_dict(x_attribute, y="dust", fin=None, data=None, percentage=N
     else:
         # Decides the type of chart to be plot.
         if y == "dust":
-            data_type = "dust_utxos"
+            data_type = ["dust_utxos", "lm_utxos"]
             if not percentage:
-                ylabel = "Number of dust UTXOs"
+                ylabel = "Number of utxos"
             else:
-                ylabel = "Percentage of dust UTXOs"
+                ylabel = "Percentage of utxos"
                 total = "total_utxos"
         elif y == "value":
-            data_type = "dust_value"
+            data_type = ["dust_value", "lm_value"]
             if not percentage:
-                ylabel = "Total dust (Satoshis)"
+                ylabel = "Value (Satoshi)"
             else:
-                ylabel = "Percentage of dust value"
+                ylabel = "Percentage of total value"
                 total = "total_value"
         elif y == "data_len":
-            data_type = "dust_data_len"
+            data_type = ["dust_data_len", "lm_data_len"]
             if not percentage:
-                ylabel = "Total size of dust UTXOs (bytes)"
+                ylabel = "Utxos' size (bytes)"
             else:
-                ylabel = "Percentage of dust size"
+                ylabel = "Percentage of total utxos' size"
                 total = "total_data_len"
         else:
             raise ValueError('Unrecognized y value')
 
+        xs = []
+        ys = []
         # Sort the data
-        xs = sorted(data[data_type].keys(), key=int)
-        ys = sorted(data[data_type].values(), key=int)
+        for i in data_type:
+            xs.append(sorted(data[i].keys(), key=int))
+            ys.append(sorted(data[i].values(), key=int))
 
         title = ""
         if not xlabel:
@@ -210,40 +214,11 @@ def plot_from_file_dict(x_attribute, y="dust", fin=None, data=None, percentage=N
         # If percentage is set, a chart with y axis as a percentage (dividing every single y value by the
         # corresponding total value) is created.
         if percentage:
-            ys = [i / float(data[total]) * 100 for i in ys]
+            for i in range(len(ys)):
+                if isinstance(ys[i], list):
+                    ys[i] = [j / float(data[total]) * 100 for j in ys[i]]
+                elif isinstance(ys[i], int):
+                    ys[i] = ys[i] / float(data[total]) * 100
 
         # And finally plots the chart.
         plot_distribution(xs, ys, title, xlabel, ylabel, log_axis, save_fig, legend, legend_loc, font_size)
-
-# Generate plots from tx data (from parsed_txs.txt)
-#plot_from_file("height", save_fig="tx_height")
-#plot_from_file("num_utxos", xlabel="Number of utxos per tx", save_fig="tx_num_utxos")
-#plot_from_file("num_utxos", xlabel="Number of utxos per tx", log_axis="x", save_fig="tx_num_utxos_logx")
-#plot_from_file("total_len", xlabel="Total length (bytes)", save_fig="tx_total_len")
-#plot_from_file("total_len", xlabel="Total length (bytes)",  log_axis="x", save_fig="tx_total_len_logx")
-#plot_from_file("version", save_fig="tx_version")
-#plot_from_file("total_value", log_axis="x", save_fig="tx_total_value_logx")
-
-# Generate plots from utxo data (from parsed_utxo.txt)
-#plot_from_file("tx_height", y="utxo", save_fig="utxo_tx_height")
-#plot_from_file("amount", y="utxo", log_axis="x", save_fig="utxo_amount_logx")
-#plot_from_file("index", y="utxo", save_fig="utxo_index")
-#plot_from_file("index", y="utxo", log_axis="x", save_fig="utxo_index_logx")
-#plot_from_file("out_type", y="utxo", save_fig="utxo_out_type")
-#plot_from_file("out_type", y="utxo", log_axis="x", save_fig="utxo_out_type_logx")
-#plot_from_file("utxo_data_len", y="utxo", save_fig="utxo_utxo_data_len")
-#plot_from_file("utxo_data_len", y="utxo", log_axis="x", save_fig="utxo_utxo_data_len_logx")
-
-# Generate plots for dust analysis (including percentage scale).
-plot_from_file_dict("fee_per_byte", "dust", fin="parsed_utxos.txt", save_fig="dust_utxos")
-
-fin = open(cfg.data_path + 'dust.txt', 'r')
-data = loads(fin.read())
-
-plot_from_file_dict("fee_per_byte", "dust", data=data, percentage=True, save_fig="perc_dust_utxos")
-
-plot_from_file_dict("fee_per_byte", "value", data=data, save_fig="dust_value")
-plot_from_file_dict("fee_per_byte", "value", data=data, percentage=True, save_fig="perc_dust_value")
-
-plot_from_file_dict("fee_per_byte", "data_len", data=data, save_fig="dust_data_len")
-plot_from_file_dict("fee_per_byte", "data_len", data=data, percentage=True, save_fig="perc_dust_data_len")

bitcoin_tools/transaction.py → bitcoin_tools/core/transaction.py

@@ -1,13 +1,14 @@
 from binascii import a2b_hex, b2a_hex
+from copy import deepcopy
+from hashlib import sha256
+
+from ecdsa import SigningKey
 
-from bitcoin_tools.keys import serialize_pk, ecdsa_tx_sign
-from bitcoin_tools.script import InputScript, OutputScript, Script, SIGHASH_ALL, SIGHASH_SINGLE, SIGHASH_NONE, \
+from bitcoin_tools.core.keys import serialize_pk, ecdsa_tx_sign
+from bitcoin_tools.core.script import InputScript, OutputScript, Script, SIGHASH_ALL, SIGHASH_SINGLE, SIGHASH_NONE, \
     SIGHASH_ANYONECANPAY
 from bitcoin_tools.utils import change_endianness, encode_varint, int2bytes, is_public_key, is_btc_addr, \
     parse_element, parse_varint, get_prev_ScriptPubKey
-from copy import deepcopy
-from ecdsa import SigningKey
-from hashlib import sha256
 
 
 class TX:

bitcoin_tools/utxo/ldb_analysis.py

@@ -0,0 +1,46 @@
+from bitcoin_tools.utils import load_conf_file
+from utxo_dump import transaction_dump, utxo_dump
+from bitcoin_tools.analysis.plots import plot_from_file, plot_from_file_dict
+from json import loads
+
+# Load config file
+cfg = load_conf_file()
+
+# Parses transactions and utxos from the dumped data.
+# transaction_dump("utxos.txt", "parsed_txs.txt")
+# utxo_dump("utxos.txt", "parsed_utxos.txt")
+# utxo_dump("utxos.txt", "parsed_non_std_utxos.txt", non_std_only=True)
+
+# Generate plots from tx data (from parsed_txs.txt)
+#plot_from_file("height", save_fig="tx_height")
+#plot_from_file("num_utxos", xlabel="Number of utxos per tx", save_fig="tx_num_utxos")
+#plot_from_file("num_utxos", xlabel="Number of utxos per tx", log_axis="x", save_fig="tx_num_utxos_logx")
+#plot_from_file("total_len", xlabel="Total length (bytes)", save_fig="tx_total_len")
+#plot_from_file("total_len", xlabel="Total length (bytes)",  log_axis="x", save_fig="tx_total_len_logx")
+#plot_from_file("version", save_fig="tx_version")
+#plot_from_file("total_value", log_axis="x", save_fig="tx_total_value_logx")
+
+# Generate plots from utxo data (from parsed_utxo.txt)
+#plot_from_file("tx_height", y="utxo", save_fig="utxo_tx_height")
+#plot_from_file("amount", y="utxo", log_axis="x", save_fig="utxo_amount_logx")
+#plot_from_file("index", y="utxo", save_fig="utxo_index")
+#plot_from_file("index", y="utxo", log_axis="x", save_fig="utxo_index_logx")
+#plot_from_file("out_type", y="utxo", save_fig="utxo_out_type")
+#plot_from_file("out_type", y="utxo", log_axis="x", save_fig="utxo_out_type_logx")
+#plot_from_file("utxo_data_len", y="utxo", save_fig="utxo_utxo_data_len")
+#plot_from_file("utxo_data_len", y="utxo", log_axis="x", save_fig="utxo_utxo_data_len_logx")
+
+# Generate plots for dust analysis (including percentage scale).
+# plot_from_file_dict("fee_per_byte", "dust", fin="parsed_utxos.txt", save_fig="dust_utxos")
+
+fin = open(cfg.data_path + 'dust.txt', 'r')
+data = loads(fin.read())
+
+plot_from_file_dict("fee_per_byte", "dust", data=data, save_fig="dust_utxos", legend=True)
+plot_from_file_dict("fee_per_byte", "dust", data=data, percentage=True, save_fig="perc_dust_utxos")
+
+plot_from_file_dict("fee_per_byte", "value", data=data, save_fig="dust_value")
+plot_from_file_dict("fee_per_byte", "value", data=data, percentage=True, save_fig="perc_dust_value")
+
+plot_from_file_dict("fee_per_byte", "data_len", data=data, save_fig="dust_data_len")
+plot_from_file_dict("fee_per_byte", "data_len", data=data, percentage=True, save_fig="perc_dust_data_len")

bitcoin_tools/utxo_dump.py → bitcoin_tools/utxo/utxo_dump.py

@@ -20,7 +20,6 @@ def check_multisig(script):
 
 
 def get_min_input_size(out, height, count_p2sh=False):
-
     out_type = out["out_type"]
     script = out["data"]
 
@@ -103,7 +102,7 @@ def transaction_dump(fin_name, fout_name):
         result = {"tx_id": change_endianness(data["key"][2:]),
                   "num_utxos": len(utxo.get("outs")),
                   "total_value": imprt,
-                  "total_len": (len(data["key"]) + len(data["value"]))/2,
+                  "total_len": (len(data["key"]) + len(data["value"])) / 2,
                   "height": utxo["height"],
                   "coinbase": utxo["coinbase"],
                   "version": utxo["version"]}
@@ -124,32 +123,36 @@ def utxo_dump(fin_name, fout_name, count_p2sh=False, non_std_only=False):
     # Standard UTXO types
     std_types = [0, 1, 2, 3, 4, 5]
 
-    # Dust dictionary initialisation. It contains one entry for each fee-per-byte step in range min, max.
-    dust = {str(fee_per_byte): 0 for fee_per_byte in range(MIN_FEE_PER_BYTE, MAX_FEE_PER_BYTE, FEE_STEP)}
-
     for line in fin:
         data = loads(line[:-1])
         utxo = decode_utxo(data["value"])
 
         for out in utxo.get("outs"):
             # Checks whether we are looking for every type of UTXO or just for non-standard ones.
             if not non_std_only or (non_std_only and out["out_type"] not in std_types):
-
                 # Calculates the dust threshold for every UTXO value and every fee per byte ratio between min and max.
                 min_size = get_min_input_size(out, utxo["height"], count_p2sh)
-                for fee_per_byte in range(MIN_FEE_PER_BYTE, MAX_FEE_PER_BYTE, FEE_STEP):
-                    # Dust dictionary is updated with whether the UTXO is dust for that current ratio (fee_per_byte)
-                    # or not (i.e: 1 or 0).
-                    if out["amount"] < min_size * fee_per_byte:
-                        dust[str(fee_per_byte)] = 1
-                    else:
-                        dust[str(fee_per_byte)] = 0
+                # Initialize dust, lm and the fee_per_byte ratio.
+                dust = 0
+                lm = 0
+                fee_per_byte = MIN_FEE_PER_BYTE
+                # Check whether the utxo is dust/lm for the fee_per_byte range.
+                while MAX_FEE_PER_BYTE > fee_per_byte and lm == 0:
+                    # Set the dust and loss_making thresholds.
+                    if dust is 0 and min_size * fee_per_byte > out["amount"] / 3:
+                        dust = fee_per_byte
+                    if lm is 0 and out["amount"] < min_size * fee_per_byte:
+                        lm = fee_per_byte
+
+                    # Increase the ratio
+                    fee_per_byte += FEE_STEP
 
                 # Builds the output dictionary
                 result = {"tx_id": change_endianness(data["key"][2:]),
                           "tx_height": utxo["height"],
-                          "utxo_data_len": len(out["data"])/2,
-                          "dust": dust}
+                          "utxo_data_len": len(out["data"]) / 2,
+                          "dust": dust,
+                          "loss_making": lm}
 
                 # Updates the dictionary with the remaining data from out, and stores it in disk.
                 result.update(out)
@@ -164,8 +167,12 @@ def accumulate_dust(fin_name):
     fin = open(cfg.data_path + fin_name, 'r')
 
     dust = {str(fee_per_byte): 0 for fee_per_byte in range(MIN_FEE_PER_BYTE, MAX_FEE_PER_BYTE, FEE_STEP)}
-    value = deepcopy(dust)
-    data_len = deepcopy(dust)
+    value_dust = deepcopy(dust)
+    data_len_dust = deepcopy(dust)
+
+    lm = deepcopy(dust)
+    value_lm = deepcopy(dust)
+    data_len_lm = deepcopy(dust)
 
     total_utxo = 0
     total_value = 0
@@ -175,21 +182,21 @@ def accumulate_dust(fin_name):
         data = loads(line[:-1])
 
         for fee_per_byte in range(MIN_FEE_PER_BYTE, MAX_FEE_PER_BYTE, FEE_STEP):
-            if data["dust"][str(fee_per_byte)]:
+            if fee_per_byte >= data["dust"] != 0:
                 dust[str(fee_per_byte)] += 1
-                value[str(fee_per_byte)] += data["amount"]
-                data_len[str(fee_per_byte)] += data["utxo_data_len"]
+                value_dust[str(fee_per_byte)] += data["amount"]
+                data_len_dust[str(fee_per_byte)] += data["utxo_data_len"]
+            if fee_per_byte >= data["loss_making"] != 0:
+                lm[str(fee_per_byte)] += 1
+                value_lm[str(fee_per_byte)] += data["amount"]
+                data_len_lm[str(fee_per_byte)] += data["utxo_data_len"]
 
         total_utxo = total_utxo + 1
         total_value += data["amount"]
         total_data_len += data["utxo_data_len"]
 
     fin.close()
 
-    return {"dust_utxos": dust, "dust_value": value, "dust_data_len": data_len, "total_utxos": total_utxo,
-            "total_value": total_value, "total_data_len": total_data_len}
-
-
-transaction_dump("utxos.txt", "parsed_txs.txt")
-utxo_dump("utxos.txt", "parsed_utxos.txt")
-# utxo_dump("utxos.txt", "parsed_non_std_utxos.txt", non_std_only=True)
+    return {"dust_utxos": dust, "dust_value": value_dust, "dust_data_len": data_len_dust,
+            "lm_utxos": lm, "lm_value": value_lm, "lm_data_len": data_len_lm,
+            "total_utxos": total_utxo, "total_value": total_value, "total_data_len": total_data_len}

bitcoin_tools/wallet.py

@@ -1,13 +1,14 @@
-from bitcoin_tools.constants import PUBKEY_HASH, TESTNET_PUBKEY_HASH, WIF, TESTNET_WIF
 from binascii import a2b_hex, b2a_hex
 from hashlib import new, sha256
 from os import mkdir, path
+from re import match
 
-from bitcoin_tools.utils import load_conf_file
-from keys import serialize_pk, serialize_sk
 from base58 import b58encode, b58decode
 from qrcode import make as qr_make
-from re import match
+
+from bitcoin_tools.constants import PUBKEY_HASH, TESTNET_PUBKEY_HASH, WIF, TESTNET_WIF
+from bitcoin_tools.core.keys import serialize_pk, serialize_sk
+from bitcoin_tools.utils import load_conf_file
 
 
 def hash_160(pk):

examples/advanced_raw_tx_creation.py

@@ -1,5 +1,4 @@
-from bitcoin_tools.keys import load_keys
-from bitcoin_tools.transaction import TX
+from bitcoin_tools.core.transaction import TX
 
 #################################################
 #       Advanced Raw transaction building       #

examples/basic_raw_tx_creation.py

@@ -1,5 +1,6 @@
 from bitcoin_tools.keys import load_keys
-from bitcoin_tools.transaction import TX
+
+from bitcoin_tools.core.transaction import TX
 
 #################################################
 #               Key loading                     #

examples/key_management.py

@@ -1,4 +1,4 @@
-from bitcoin_tools.keys import generate_keys, store_keys
+from bitcoin_tools.core.keys import generate_keys, store_keys
 from bitcoin_tools.wallet import generate_wif, generate_btc_addr
 
 #################################################

examples/tx_analysis.py

@@ -1,4 +1,4 @@
-from bitcoin_tools.transaction import TX
+from bitcoin_tools.core.transaction import TX
 
 #################################################
 #           Hex transaction analysis            #

test/scripts_test.py

@@ -1,7 +1,8 @@
-from bitcoin_tools.script import OutputScript, InputScript, Script
-from bitcoin_tools.wallet import hash_160
 from binascii import b2a_hex
 
+from bitcoin_tools.core.script import OutputScript, InputScript, Script
+from bitcoin_tools.wallet import hash_160
+
 pk = "04a01f076082a713a82b47ace012934052bcf3359c964f1963d00def84a34e7b0345efeefe037f4b0a4e160cc40a7fac052523da88398630c07ef3c54b47aa6046"
 signature = "3045022100df7b7e5cda14ddf91290e02ea10786e03eb11ee36ec02dd862fe9a326bbcb7fd02203f5b4496b667e6e281cc654a2da9e4f08660c620a1051337fa8965f727eb191901"
 btc_addr = "mgwpBW3g4diqasfxzWDgSi5fBrsFKmNdva"

test/transactions.py

@@ -1,5 +1,5 @@
-from bitcoin_tools.keys import serialize_pk, load_keys
-from bitcoin_tools.transaction import TX
+from bitcoin_tools.core.keys import serialize_pk, load_keys
+from bitcoin_tools.core.transaction import TX
 
 # BUILD TRANSACTIONS