From 84000a81c21bbd69ffea6d53a7966ef05ec2c044 Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 10:43:00 -0800 Subject: [PATCH 01/10] dynamically generating requirements.txt --- .gitignore | 3 +++ Makefile | 19 +++++++++++++++++++ requirements.txt | 4 ---- 3 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 .gitignore create mode 100644 Makefile delete mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ce4db3b --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +requirements.txt +*.csv +__pycache__ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..e7a10d9 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +QUIET ?= -OO # run with QUIET= to see debugging messages +CHAINSTATE ?= $(HOME)/.bitcoin/chainstate +BALANCES ?= balances.csv +export +run: requirements.txt btcposbal2csv.py + # stop bitcoind if running, to avoid corrupting chainstate + pip3 install --user --requirement $< + if $$(pidof bitcoind); then \ + bitcoin-cli stop; \ + python3 $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ + bitcoind; \ + else \ + python3 $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ + fi +requirements.txt: + for requirement in plyvel base58 sqlite3 hashlib; do \ + python3 -c "import $$requirement" 2>/dev/null || \ + (echo $$requirement >> $@; true); \ + done diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index cc1501a..0000000 --- a/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -hashlib -plyvel -base58 -sqlite3 \ No newline at end of file From a69f660f0419fc6ad743de46941b1f45af3f5e72 Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 10:48:34 -0800 Subject: [PATCH 02/10] make it run under either python2 or python3 --- .gitignore | 2 ++ Makefile | 14 ++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index ce4db3b..48be754 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ requirements.txt *.csv __pycache__ +*.pyo +*.pyc diff --git a/Makefile b/Makefile index e7a10d9..89caac0 100644 --- a/Makefile +++ b/Makefile @@ -1,19 +1,25 @@ QUIET ?= -OO # run with QUIET= to see debugging messages CHAINSTATE ?= $(HOME)/.bitcoin/chainstate BALANCES ?= balances.csv +PYTHON ?= python +ifeq ($(PYTHON),python3) + PIP := pip3 +else + PIP := pip +endif export run: requirements.txt btcposbal2csv.py # stop bitcoind if running, to avoid corrupting chainstate - pip3 install --user --requirement $< + $(PIP) install --user --requirement $< if $$(pidof bitcoind); then \ bitcoin-cli stop; \ - python3 $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ + $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ bitcoind; \ else \ - python3 $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ + $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ fi requirements.txt: for requirement in plyvel base58 sqlite3 hashlib; do \ - python3 -c "import $$requirement" 2>/dev/null || \ + $(PYTHON) -c "import $$requirement" 2>/dev/null || \ (echo $$requirement >> $@; true); \ done From 603d12943c00e8c5b74ddf9f8b23846c63322a7e Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 10:57:49 -0800 Subject: [PATCH 03/10] added warning to install libleveldb-dev --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 89caac0..ddcdd76 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,7 @@ run: requirements.txt btcposbal2csv.py $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ fi requirements.txt: + # WARNING: may be necessary to `sudo apt install libleveldb-dev` for requirement in plyvel base58 sqlite3 hashlib; do \ $(PYTHON) -c "import $$requirement" 2>/dev/null || \ (echo $$requirement >> $@; true); \ From 2102a1ce9c0c4f8b527532edb7f76f822da4a56b Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 11:09:41 -0800 Subject: [PATCH 04/10] made some updates to the readme --- readme.md | 43 ++++++++++++++++++++++++++++--------------- 1 file changed, 28 insertions(+), 15 deletions(-) diff --git a/readme.md b/readme.md index dfef80b..5c07574 100644 --- a/readme.md +++ b/readme.md @@ -7,21 +7,25 @@ python 2.7 pip #### To install: -run pip install -r requirements.txt +`make requirements.txt` +`pip install -r requirements.txt` or install following packages with pip manualy -* hashlib +* hashlib (already included in modern Python installations) * plyvel * base58 -* sqlite3 +* sqlite3 (already included in modern Python installations) #### Usage -To use you will need copy of chainstate database as created by [bitcoin core](https://bitcoin.org/en/bitcoin-core/) - client. I've not tried different clients. +To use you will need copy of chainstate database as created by +[bitcoin core](https://bitcoin.org/en/bitcoin-core/) client. I've not +tried different clients. -To get current addresses with positive balance, let the full node client sync with the network. -**Stop** the bitcoin-core client before running this utility. If you not stop the client, the database might get corrupted. -Then run this program with path to chainstate directory (usualy $HOME/.bitcoin/chainstate). +To get current addresses with positive balance, let the full node client +sync with the network. **Stop** the bitcoin-core client before running this +utility. If you not stop the client, the database might get corrupted. +Then run this program with path to chainstate directory +(usualy $HOME/.bitcoin/chainstate). Show help ``` @@ -31,11 +35,18 @@ python btcposbal2csv.py -h Following will read from `/home/USER/.bitcoin/chainstate`, and write result to `/home/USER/addresses_with_balance.csv`. ``` python btcposbal2csv.py /home/USER/.bitcoin/chainstate /home/USER/addresses_with_balance.csv + +Or you can just accept the defaults in the Makefile, and run: `make` ``` ##### Notice -* That the output may not be complete as there are some transactions which are not understood by the decoding lib, or that which do not have "address" at all. Such transactions are not processed. Number of them and the total ammount is displayed after the analysis. -* The output csv file only reflects the chainstate leveldb at your disk. So it will always be few blocks behind the network as you need to stop the bitcoin-core client. +* The output may not be complete, as there are some transactions which are + not understood by the decoding lib, or which do not have "address" at all. + Such transactions are not processed. The number of them and the total amount + is displayed after the analysis. +* The output csv file only reflects the chainstate leveldb on your disk. So + it will always be few blocks behind the network, as you need to stop the + bitcoin-core client. #### Converting to RIPEMD160 Per request, I'm adding script which is able to convert BTC address to RIPEMD160 representation. @@ -47,10 +58,12 @@ python convert2ripemd160.py /home/USER/addresses_with_balance.csv ``` #### Acknowledgement -This utility builds on very nice [bitcoin_tools](https://github.com/sr-gi/bitcoin_tools/) lib, - which does the UTXO parsing. +This utility builds on the very nice +[bitcoin_tools](https://github.com/sr-gi/bitcoin_tools/) lib, +which does the UTXO parsing. #### Support -If you like this utility, please consider supporting the bitcoin_tools library which does all the heavy lifting. +If you like this utility, please consider supporting the bitcoin_tools library, +which does all the heavy lifting. -If this particular functionality made your life easier you can support coffee consumption in BTC -1FxC1mgJkad63beJcECfZMRaFSf4PBLr2f. +If this particular functionality made your life easier, you can support coffee +consumption in BTC, 1FxC1mgJkad63beJcECfZMRaFSf4PBLr2f. From e45888953a62777cf65c04adb5e09bbd3da7f32a Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 13:14:36 -0800 Subject: [PATCH 05/10] python3 compatibility --- utils.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/utils.py b/utils.py index 830e3b0..2fb239b 100644 --- a/utils.py +++ b/utils.py @@ -11,6 +11,11 @@ # Fee per byte range NSPECIALSCRIPTS = 6 +# python3 compatibility +if chr(189) == b'\xbd': + BYTECHR = chr +else: + BYTECHR = lambda c: bytes((c,)) def txout_decompress(x): """ Decompresses the Satoshi amount of a UTXO stored in the LevelDB. Code is a port from the Bitcoin Core C++ @@ -134,7 +139,11 @@ def decode_utxo(coin, outpoint, version=0.15): else: # First we will parse all the data encoded in the outpoint, that is, the transaction id and index of the utxo. # Check that the input data corresponds to a transaction. - assert outpoint[:2] == '43' + try: + assert outpoint[:2] == b'43' + except AssertionError: + raise AssertionError('First two characters of %r are not "43"' % + outpoint) # Check the provided outpoint has at least the minimum length (1 byte of key code, 32 bytes tx id, 1 byte index) assert len(outpoint) >= 68 # Get the transaction id (LE) by parsing the next 32 bytes of the outpoint. @@ -314,7 +323,7 @@ def parse_ldb(fin_name, version=0.15, types=(0, 1)): db = plyvel.DB(fin_name, compression=None) # Change with path to chainstate # Load obfuscation key (if it exists) - o_key = db.get((unhexlify("0e00") + "obfuscate_key")) + o_key = db.get((unhexlify("0e00") + b"obfuscate_key")) # If the key exists, the leading byte indicates the length of the key (8 byte by default). If there is no key, # 8-byte zeros are used (since the key will be XORed with the given values). @@ -394,7 +403,7 @@ def deobfuscate_value(obfuscation_key, value): # Get the extended obfuscation key by concatenating the obfuscation key with itself until it is as large as the # value to be de-obfuscated. if l_obf < l_value: - extended_key = (obfuscation_key * ((l_value / l_obf) + 1))[:l_value] + extended_key = (obfuscation_key * ((l_value // l_obf) + 1))[:l_value] else: extended_key = obfuscation_key[:l_value] @@ -448,7 +457,7 @@ def hash_160_to_btc_address(h160, v): h160 = unhexlify(h160) # Add the network version leading the previously calculated RIPEMD-160 hash. - vh160 = chr(v) + h160 + vh160 = BYTECHR(v) + h160 # Double sha256. h = sha256(sha256(vh160).digest()).digest() # Add the two first bytes of the result as a checksum tailing the RIPEMD-160 hash. From 90570f539c7a8de37acef6b835e7ff44a0c85972 Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 22:07:28 +0000 Subject: [PATCH 06/10] fixed some more python2/python3 compatibility issues. --- btcposbal2csv.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/btcposbal2csv.py b/btcposbal2csv.py index 6562139..a79b889 100644 --- a/btcposbal2csv.py +++ b/btcposbal2csv.py @@ -105,7 +105,7 @@ def in_mem(in_args): else: add_dict[add] = [val, height] - for key in add_dict.iterkeys(): + for key in add_dict: ll = add_dict[key] yield key, ll[0], ll[1] @@ -201,8 +201,10 @@ def low_mem(in_args): for address, sat_val, block_height in add_iter: if sat_val == 0: continue + # make this work the same on Python2 and Python3 + decoded = str(address.decode()) w.append( - address + ',' + str(sat_val) + ',' + str(block_height) + ','.join((decoded, str(sat_val), str(block_height))) ) c += 1 if c == 1000: @@ -212,4 +214,4 @@ def low_mem(in_args): if c > 0: f.write('\n'.join(w) + '\n') f.write('\n') - print('writen to %s' % args.out) + print('written to %s' % args.out) From 9739569bc742cbdc0e465ea4a46f4dbddd6f771d Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 22:19:05 +0000 Subject: [PATCH 07/10] works identically under python2 and 3 now, at least without options --- Makefile | 2 +- utils.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index ddcdd76..9cea271 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ else PIP := pip endif export -run: requirements.txt btcposbal2csv.py +$(BALANCES): requirements.txt btcposbal2csv.py # stop bitcoind if running, to avoid corrupting chainstate $(PIP) install --user --requirement $< if $$(pidof bitcoind); then \ diff --git a/utils.py b/utils.py index 2fb239b..4652e64 100644 --- a/utils.py +++ b/utils.py @@ -32,10 +32,10 @@ def txout_decompress(x): return 0 x -= 1 e = x % 10 - x /= 10 + x //= 10 if e < 9: d = (x % 9) + 1 - x /= 9 + x //= 9 n = x * 10 + d else: n = x + 1 From afcb391f10aca6942539d95dead85d37d0a9ed90 Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 14:39:53 -0800 Subject: [PATCH 08/10] slight cleanup of README --- readme.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/readme.md b/readme.md index 5c07574..1988055 100644 --- a/readme.md +++ b/readme.md @@ -25,19 +25,18 @@ To get current addresses with positive balance, let the full node client sync with the network. **Stop** the bitcoin-core client before running this utility. If you not stop the client, the database might get corrupted. Then run this program with path to chainstate directory -(usualy $HOME/.bitcoin/chainstate). +(usually $HOME/.bitcoin/chainstate). Show help ``` python btcposbal2csv.py -h ``` #### Example: -Following will read from `/home/USER/.bitcoin/chainstate`, and write result to `/home/USER/addresses_with_balance.csv`. -``` -python btcposbal2csv.py /home/USER/.bitcoin/chainstate /home/USER/addresses_with_balance.csv +Following will read from `$HOME/.bitcoin/chainstate`, and write result to `$HOME/addresses_with_balance.csv`. + +`python btcposbal2csv.py $HOME/.bitcoin/chainstate $HOME/addresses_with_balance.csv` Or you can just accept the defaults in the Makefile, and run: `make` -``` ##### Notice * The output may not be complete, as there are some transactions which are @@ -54,7 +53,7 @@ BTC address must be in fist column, RIPEMD160 is added to csv. Output goes to st Example: ``` -python convert2ripemd160.py /home/USER/addresses_with_balance.csv +python convert2ripemd160.py $HOME/addresses_with_balance.csv ``` #### Acknowledgement From b0298ee1284309f7b8ce292b9a8b5c1fc8a9d47f Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 17:08:43 -0800 Subject: [PATCH 09/10] bugfix --- Makefile | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9cea271..2e5d469 100644 --- a/Makefile +++ b/Makefile @@ -11,12 +11,18 @@ export $(BALANCES): requirements.txt btcposbal2csv.py # stop bitcoind if running, to avoid corrupting chainstate $(PIP) install --user --requirement $< - if $$(pidof bitcoind); then \ + if pidof bitcoind >/dev/null; then \ bitcoin-cli stop; \ - $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ + while pidof bitcoind >/dev/null; do sleep 1; done; \ + $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES)tmp; \ bitcoind; \ else \ - $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES); \ + $(PYTHON) $(QUIET) ./$(word 2, $+) $(CHAINSTATE) $(BALANCES)tmp; \ + fi + if [ "$$(wc -c $(BALANCES)tmp | awk '{print 1}')" -gt 1 ]; then \ + mv $(BALANCES)tmp $(BALANCES); \ + else \ + echo No balances found in chainstate >&2; \ fi requirements.txt: # WARNING: may be necessary to `sudo apt install libleveldb-dev` From d34afa47ce4c18150a93238f2498a0bdb36ab3c7 Mon Sep 17 00:00:00 2001 From: John Comeau Date: Tue, 23 Feb 2021 17:09:25 -0800 Subject: [PATCH 10/10] ignoring more cruft --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 48be754..3b2f324 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ requirements.txt __pycache__ *.pyo *.pyc +*.txt +*.csvtmp