Taproot Support and Python 3 Migration

bitcoin_tools/analysis/plots.py

@@ -78,7 +78,7 @@ def plot_distribution(xs, ys, title, xlabel, ylabel, log_axis=None, save_fig=Fal
         plt.plot(xs, ys)  # marker='o'
     else:
         for i in range(len(xs)):
-            plt.plot(xs[i], ys[i], ' ', linestyle='solid')  # marker='o'
+            plt.plot(xs[i], ys[i],  linestyle='solid')  # marker='o'
 
     # Plot title and xy labels
     plt.title(title, {'color': 'k', 'fontsize': font_size})
@@ -102,6 +102,10 @@ def plot_distribution(xs, ys, title, xlabel, ylabel, log_axis=None, save_fig=Fal
         ymin, ymax = plt.ylim()
         plt.ylim(ymin, y_sup_lim)
 
+    tick_val = [0, 100000, 200000, 300000, 400000, 500000, 600000, 700000, 800000]
+    tick_lab = ['0', '100K', '200K', '300K', '400K', '500K', '600K', '700K', '800K']
+    plt.xticks(tick_val, tick_lab)
+
     # Output result
     if save_fig:
         plt.savefig(CFG.figs_path + save_fig + '.pdf', format='pdf', dpi=600)

bitcoin_tools/analysis/status/README.md

@@ -3,7 +3,7 @@
 **STATUS** (**ST**atistical **A**nalysis **T**ool for **U**txo **S**et) is an open source tool that provides an easy way to access, decode and analyze data from the Bitcoin's `utxo set`. The accompanying working paper further explains its design, application, and presents results of a recently performed analysis: [https://eprint.iacr.org/2017/1095.pdf](https://eprint.iacr.org/2017/1095.pdf)
 
 
-STATUS is coded in Python 2 and works for both the existing versions of Bitcoin Core's `utxo set`, that is, the first defined format (versions 0.8 - 0.14) and the recently defined one (version 0.15).
+STATUS is developed and tested with Python 3.11 and works for both the existing versions of Bitcoin Core's `utxo set`, that is, the first defined format (versions 0.8 - 0.14) and the recently defined one (version 0.15).
 
 STATUS works, from now on, with 0.15 format. For 0.8-0.14 version refer to `ldb_0.14` branch.
 

bitcoin_tools/analysis/status/data_dump.py

@@ -1,7 +1,7 @@
 from bitcoin_tools import CFG
 from bitcoin_tools.analysis.status import FEE_STEP
 from bitcoin_tools.analysis.status.utils import check_multisig, get_min_input_size, roundup_rate, check_multisig_type, \
-    get_serialized_size_fast, get_est_input_size, load_estimation_data, check_native_segwit
+    get_serialized_size_fast, get_est_input_size, load_estimation_data, check_native_segwit, check_native_taproot
 import ujson
 
 
@@ -118,10 +118,14 @@ def utxo_dump(fin_name, fout_name, coin, count_p2sh=False, non_std_only=False):
                 else:
                     multisig = check_multisig_type(out["data"])
                     segwit = check_native_segwit(out["data"])
+                    taproot = check_native_taproot(out["data"])
+
                     if multisig:
                         non_std_type = multisig
                     elif segwit[0]:
                         non_std_type = segwit[1]
+                    elif taproot[0]:
+                        non_std_type = taproot[1]
                     else:
                         non_std_type = False
 

bitcoin_tools/analysis/status/kill_at_heigh.py

@@ -45,10 +45,10 @@ def kill_line(line, kill_at_height, pid):
         try:
             height_str = re.search(' height=\d+ ', line)
             block_height = height_str.group()[8:-1]
-            print "    Block height {} read".format(block_height)
+            print ("    Block height {} read".format(block_height))
             if block_height == kill_at_height:
                 kill(pid, SIGTERM)
-                print "Process with pid {} KILLED!!!".format(pid)
+                print ("Process with pid {} KILLED!!!".format(pid))
                 exit()
 
         except AttributeError as err:
@@ -79,7 +79,7 @@ def follow(thefile, kill_at_height, pid, start_at=START_AT_ORIGIN):
         if line and not kill_line(line, kill_at_height, pid):
             continue
         if not line:
-            print "Waiting {} for new data...".format(SLEEP_INT)
+            print ("Waiting {} for new data...".format(SLEEP_INT))
             sleep(SLEEP_INT)
             continue
 
@@ -107,15 +107,15 @@ def follow(thefile, kill_at_height, pid, start_at=START_AT_ORIGIN):
                 start = START_AT_ORIGIN
 
         if not kill_at_height or not pid:
-            print "Usage: "
-            print "    python kill_at_heigh.py -k block_heigh -p pid [-f file] [-o] [-e] "
+            print ("Usage: ")
+            print ("    python kill_at_heigh.py -k block_heigh -p pid [-f file] [-o] [-e] ")
             exit()
 
-        print "Starting to monitor file {} at {}".format(log_filename, start)
-        print "I'm going to kill process with pid {} at height {}".format(pid, kill_at_height)
+        print ("Starting to monitor file {} at {}".format(log_filename, start))
+        print ("I'm going to kill process with pid {} at height {}".format(pid, kill_at_height))
 
         logfile = open(log_filename, "r")
         follow(logfile, kill_at_height, pid, start_at=start)
 
     except IOError as err:
-        print "Bitcoind log file {} not found".format(log_filename)
+        print ("Bitcoind log file {} not found".format(log_filename))

bitcoin_tools/analysis/status/plots.py

@@ -104,20 +104,20 @@ def overview_from_file(tx_fin_name, utxo_fin_name):
 
     samples = get_samples(['num_utxos', 'total_len', 'height'], fin_name=tx_fin_name)
 
-    print "\t Max height: ", str(max(samples['height']))
-    print "\t Num. of tx: ", str(len(samples['num_utxos']))
-    print "\t Num. of UTXOs: ", str(sum(samples['num_utxos']))
-    print "\t Avg. num. of UTXOs per tx: ", str(np.mean(samples['num_utxos']))
-    print "\t Std. num. of UTXOs per tx: ", str(np.std(samples['num_utxos']))
-    print "\t Median num. of UTXOs per tx: ", str(np.median(samples['num_utxos']))
+    print ("\t Max height: ", str(max(samples['height'])))
+    print ("\t Num. of tx: ", str(len(samples['num_utxos'])))
+    print ("\t Num. of UTXOs: ", str(sum(samples['num_utxos'])))
+    print ("\t Avg. num. of UTXOs per tx: ", str(np.mean(samples['num_utxos'])))
+    print ("\t Std. num. of UTXOs per tx: ", str(np.std(samples['num_utxos'])))
+    print ("\t Median num. of UTXOs per tx: ", str(np.median(samples['num_utxos'])))
 
     len_attribute = "total_len"
 
-    print "\t Size of the (serialized) UTXO set: ", str(np.sum(samples[len_attribute]))
+    print ("\t Size of the (serialized) UTXO set: ", str(np.sum(samples[len_attribute])))
 
     samples = get_samples("register_len", fin_name=utxo_fin_name)
     len_attribute = "register_len"
 
-    print "\t Avg. size per register: ", str(np.mean(samples[len_attribute]))
-    print "\t Std. size per register: ", str(np.std(samples[len_attribute]))
-    print "\t Median size per register: ", str(np.median(samples[len_attribute]))
+    print ("\t Avg. size per register: ", str(np.mean(samples[len_attribute])))
+    print ("\t Std. size per register: ", str(np.std(samples[len_attribute])))
+    print ("\t Median size per register: ", str(np.median(samples[len_attribute])))

bitcoin_tools/analysis/status/run_analysis.py

@@ -6,6 +6,7 @@
 from bitcoin_tools import CFG
 from getopt import getopt
 from sys import argv
+import copy
 
 
 def set_out_names(count_p2sh, non_std_only):
@@ -57,10 +58,10 @@ def non_std_outs_analysis(samples):
     # and 3-3, and put the rest into "Other".
 
     groups = [[u'multisig-1-3'], [u'multisig-1-2'], [u'multisig-1-1'], [u'multisig-3-3'], [u'multisig-2-2'],
-              [u'multisig-2-3'], ["P2WSH"], ["P2WPKH"], [False, u'multisig-OP_NOTIF-OP_NOTIF',
+              [u'multisig-2-3'], ["P2WSH"], ["P2WPKH"],["P2TR"], [False, u'multisig-OP_NOTIF-OP_NOTIF',
                                   u'multisig-<2153484f55544f555420544f2023424954434f494e2d41535345545320202020202020202'
                                   u'0202020202020202020202020202020202020202020202020202020>-1']]
-    labels = ['M. 1-3', 'M. 1-2', 'M. 1-1', 'M. 3-3', 'M. 2-2', 'M. 2-3', "P2WSH", "P2WPKH", 'Other']
+    labels = ['M. 1-3', 'M. 1-2', 'M. 1-1', 'M. 3-3', 'M. 2-2', 'M. 2-3', "P2WSH", "P2WPKH","P2TR", 'Other']
 
     out_name = "utxo_non_std_type"
 
@@ -69,6 +70,26 @@ def non_std_outs_analysis(samples):
                                                    "#A69229", "#B69229", "#F69229"], labels_out=True)
 
 
+def all_outs_analysis(samples):
+
+    for i in range(len(samples["out_type"])):
+        if samples["non_std_type"][i] == "std":
+            samples["non_std_type"][i] = samples["out_type"][i]
+
+    samples_special = samples.pop("non_std_type")
+
+    groups = [[0], [2, 3, 4, 5], [1], [u'multisig-1-3', u'multisig-1-2', u'multisig-1-1', u'multisig-3-3',
+                  u'multisig-2-2', u'multisig-2-3'], ["P2WSH"], ["P2WPKH"], ["P2TR"]]
+
+    labels = ['P2PKH', 'P2PK', 'P2SH', 'MULTISIG', "P2WSH",
+              "P2WPKH", "P2TR"]
+
+    out_name = "utxo_all_type"
+
+    plot_pie_chart_from_samples(samples=samples_special, save_fig=out_name, labels=labels, groups=groups, title="",
+                                labels_out=True)
+
+
 def tx_based_analysis(tx_fin_name):
     """
     Performs a transaction based analysis from a given input file (resulting from a transaction dump of the chainstate)
@@ -127,7 +148,7 @@ def utxo_based_analysis(utxo_fin_name):
     log_axis = [False, 'x', [False, 'x'], [False, 'x'], [False, 'x'], [False, 'x']]
 
     x_attributes_pie = ['out_type', 'out_type']
-    xlabels_pie = [['C-even', 'C-odd', 'U-even', 'U-odd'], ['P2PKH', 'P2PK', 'P2SH', 'Other']]
+    xlabels_pie = [['C-even', 'C-odd', 'U-even', 'U-odd'], ['P2PKH', 'P2PK', 'P2SH']]
     out_names_pie = ["utxo_pk_types", "utxo_types"]
     pie_groups = [[[2], [3], [4], [5]], [[0], [2, 3, 4, 5], [1]]]
 
@@ -136,17 +157,19 @@ def utxo_based_analysis(utxo_fin_name):
     # Since the attributes for the pie chart are already included in the normal chart, we won't pass them to the
     # sampling function.
     samples = get_samples(x_attributes + [x_attribute_special], fin_name=utxo_fin_name)
+    samples_all = copy.deepcopy(samples)
     samples_special = samples.pop(x_attribute_special)
 
-    for attribute, label, log, out in zip(x_attributes, xlabels, log_axis, out_names):
-        xs, ys = get_cdf(samples[attribute], normalize=True)
-        plots_from_samples(xs=xs, ys=ys, xlabel=label, log_axis=log, save_fig=out, ylabel="Number of UTXOs")
+    #for attribute, label, log, out in zip(x_attributes, xlabels, log_axis, out_names):
+    #    xs, ys = get_cdf(samples[attribute], normalize=True)
+    #    plots_from_samples(xs=xs, ys=ys, xlabel=label, log_axis=log, save_fig=out, ylabel="Number of UTXOs")
 
-    for attribute, label, out, groups in (zip(x_attributes_pie, xlabels_pie, out_names_pie, pie_groups)):
-        plot_pie_chart_from_samples(samples=samples[attribute], save_fig=out, labels=label, title="", groups=groups,
-                                    colors=["#165873", "#428C5C", "#4EA64B", "#ADD96C"], labels_out=True)
-    # Special case: non-standard
-    non_std_outs_analysis(samples_special)
+    #for attribute, label, out, groups in (zip(x_attributes_pie, xlabels_pie, out_names_pie, pie_groups)):
+    #    plot_pie_chart_from_samples(samples=samples[attribute], save_fig=out, labels=label, title="", groups=groups,
+    #                                colors=["#165873", "#428C5C", "#4EA64B", "#ADD96C"], labels_out=True)
+    # Special case: non-standardf
+    #non_std_outs_analysis(samples_special)
+    all_outs_analysis(samples_all)
 
 
 def dust_analysis(utxo_fin_name, f_dust, fltr=None):
@@ -244,6 +267,7 @@ def utxo_based_analysis_with_filters(utxo_fin_name):
                lambda x: x["out_type"] in [2, 3, 4, 5],
                lambda x: x["non_std_type"] == "P2WPKH",
                lambda x: x["non_std_type"] == "P2WSH",
+               lambda x: x["non_std_type"] == "P2TR",
                lambda x: x["non_std_type"] is not False and "multisig" in x["non_std_type"],
                lambda x: x["non_std_type"] is False,
                lambda x: x["amount"] == 1,
@@ -256,7 +280,7 @@ def utxo_based_analysis_with_filters(utxo_fin_name):
                lambda x: x["out_type"] == 1,
                lambda x: x["amount"] == 1]
 
-    legends = [['P2PKH', 'P2SH', 'P2PK', 'P2WPKH', 'P2WSH', 'Multisig', 'Other'],
+    legends = [['P2PKH', 'P2SH', 'P2PK', 'P2WPKH', 'P2WSH', 'P2TR' , 'Multisig', 'Other'],
                ['$=1$', '$1 < x \leq 10$', '$10 < x \leq 10^2$', '$10^2 < x \leq 10^4$', '$10^4 < x \leq 10^6$',
                 '$10^6 < x \leq 10^8$', '$10^8 < x$'], ['P2SH'], ['Amount = 1']]
     comparative = [True, True, False, False]
@@ -322,33 +346,33 @@ def run_experiment(coin, chainstate, count_p2sh, non_std_only):
     f_utxos, f_parsed_txs, f_parsed_utxos, f_dust = set_out_names(count_p2sh, non_std_only)
 
     # Parse all the data in the chainstate.
-    print "Parsing the chainstate."
+    print("Parsing the chainstate.")
     parse_ldb(f_utxos, fin_name=chainstate)
 
     # Parses transactions and utxos from the dumped data.
-    print "Adding meta-data for transactions and UTXOs."
+    print("Adding meta-data for transactions and UTXOs.")
     transaction_dump(f_utxos, f_parsed_txs)
     utxo_dump(f_utxos, f_parsed_utxos, coin, count_p2sh=count_p2sh, non_std_only=non_std_only)
 
     # Print basic stats from data
-    print "Running overview analysis."
+    print("Running overview analysis.")
     overview_from_file(f_parsed_txs, f_parsed_utxos)
 
     # Generate plots from tx data (from f_parsed_txs)
-    print "Running transaction based analysis."
+    print("Running transaction based analysis.")
     tx_based_analysis(f_parsed_txs)
 
     # Generate plots from utxo data (from f_parsed_utxos)
-    print "Running UTXO based analysis."
+    print ("Running UTXO based analysis.")
     utxo_based_analysis(f_parsed_utxos)
 
     # # Aggregates dust and generates plots.
-    print "Running dust analysis."
+    print ("Running dust analysis.")
     dust_analysis(f_parsed_utxos, f_dust)
     dust_analysis_all_fees(f_parsed_utxos)
 
     # Generate plots with filters
-    print "Running analysis with filters."
+    print ("Running analysis with filters.")
     utxo_based_analysis_with_filters(f_parsed_utxos)
     tx_based_analysis_with_filters(f_parsed_txs)
 

bitcoin_tools/analysis/status/run_comparative_analysis.py

@@ -134,13 +134,13 @@ def run_experiment(f_dust, f_parsed_utxos, f_parsed_txs):
     :rtype: None
     """
 
-    print "Running comparative data analysis."
+    print ("Running comparative data analysis.")
     # Comparative dust analysis between different snapshots
     fin_names = ['height-' + str(i) + 'K/' + f_parsed_utxos + '.json' for i in range(100, 550, 50)]
     dust_files = ['height-' + str(i) + 'K/' + f_dust + '.json' for i in range(100, 550, 50)]
     legend = [str(i) + 'K' for i in range(100, 550, 50)]
 
-    print "Comparing dust from different snapshots."
+    print ("Comparing dust from different snapshots.")
     # Get dust files from different dates to compare (Change / Add the ones you'll need)
     compare_dust(dust_files=dust_files, legend=legend)
 
@@ -154,12 +154,12 @@ def run_experiment(f_dust, f_parsed_utxos, f_parsed_txs):
 
     # Comparative analysis between different snapshots
     # UTXO amount comparison
-    print "Comparing UTXO amount from different snapshots."
+    print ("Comparing UTXO amount from different snapshots.")
     compare_attribute(fin_names=fin_names, x_attribute='amount', xlabel='Amount (Satoshi)', legend=legend,
                       out_name='cmp_utxo_amount')
 
     # UTXO size comparison
-    print "Comparing UTXO size from different snapshots."
+    print ("Comparing UTXO size from different snapshots.")
     compare_attribute(fin_names=fin_names, x_attribute='register_len', xlabel='Size (bytes)', legend=legend,
                       out_name='cmp_utxo_size')