From ee06bedf3c33580f1896e1971335aa86afc3d1bf Mon Sep 17 00:00:00 2001 From: Kousthub Raja Date: Sun, 27 Feb 2022 23:42:05 +0530 Subject: [PATCH 1/8] miner addition changes --- app/codes/kycwallet.py | 3 ++- app/codes/minermanager.py | 28 ++++++++++++++++++++++++++++ app/codes/tokenmanager.py | 5 ++--- app/codes/transactionmanager.py | 4 ++-- app/migrations/init_db.py | 8 ++++++++ app/types.py | 3 ++- 6 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 app/codes/minermanager.py diff --git a/app/codes/kycwallet.py b/app/codes/kycwallet.py index 86540da..6b8df8a 100644 --- a/app/codes/kycwallet.py +++ b/app/codes/kycwallet.py @@ -8,6 +8,7 @@ import base64 import sqlite3 +from .utils import get_time_ms from ..constants import TMP_PATH, NEWRL_DB from .transactionmanager import Transactionmanager @@ -113,7 +114,7 @@ def generate_wallet(kyccustodian, kycdocs, ownertype, jurisd, wallet_specific_da def create_add_wallet_transaction(wallet): transaction_data = { - 'timestamp': str(datetime.datetime.now()), + 'timestamp': get_time_ms(), 'type': 1, 'currency': 'INR', 'fee': 0.0, diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py new file mode 100644 index 0000000..d0396f0 --- /dev/null +++ b/app/codes/minermanager.py @@ -0,0 +1,28 @@ +"""Miner update functions""" +import datetime + +from .auth.auth import get_wallet +from .signmanager import sign_transaction +from ..types import TRANSACTION_MINER_ADDITION +from .utils import get_time_ms +from .transactionmanager import Transactionmanager + + +def miner_addition_transaction(): + transaction_data = { + 'timestamp': get_time_ms(), + 'type': TRANSACTION_MINER_ADDITION, + 'currency': "NWRL", + 'fee': 0.0, + 'descr': "Miner addition", + 'valid': 1, + 'block_index': 0, + 'specific_data': {} + } + + transaction = Transactionmanager() + transaction_data = {'transaction': transaction_data, 'signatures': []} + transaction.transactioncreator(transaction_data) + wallet = get_wallet() + signed_transaction = sign_transaction(wallet, transaction_data) + return signed_transaction diff --git a/app/codes/tokenmanager.py b/app/codes/tokenmanager.py index d33419e..422f927 100644 --- a/app/codes/tokenmanager.py +++ b/app/codes/tokenmanager.py @@ -1,13 +1,12 @@ # Python programm to create object that enables addition of a block -import datetime - +from .utils import get_time_ms from .transactionmanager import Transactionmanager def create_token_transaction(token_data): transaction = { - 'timestamp': str(datetime.datetime.now()), + 'timestamp': get_time_ms(), 'type': 2, 'currency': "INR", 'fee': 0.0, diff --git a/app/codes/transactionmanager.py b/app/codes/transactionmanager.py index ea4f23a..03330d2 100644 --- a/app/codes/transactionmanager.py +++ b/app/codes/transactionmanager.py @@ -8,7 +8,7 @@ import base64 import sqlite3 -from ..types import TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION, TRANSCATION_TOKEN_CREATION +from ..types import TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION, TRANSACTION_TOKEN_CREATION from .chainscanner import get_wallet_token_balance from ..constants import ALLOWED_CUSTODIANS_FILE, MEMPOOL_PATH, NEWRL_DB from .utils import get_time_ms @@ -540,7 +540,7 @@ def get_valid_addresses(transaction): if transaction_type == TRANSACTION_WALLET_CREATION: # Custodian needs to sign valid_addresses.append( transaction['specific_data']['custodian_wallet']) - if transaction_type == TRANSCATION_TOKEN_CREATION: # Custodian needs to sign + if transaction_type == TRANSACTION_TOKEN_CREATION: # Custodian needs to sign valid_addresses.append(transaction['specific_data']['custodian']) if transaction_type == TRANSACTION_SMART_CONTRACT: valid_addresses = get_sc_validadds(transaction) diff --git a/app/migrations/init_db.py b/app/migrations/init_db.py index 85b5236..3424f8a 100644 --- a/app/migrations/init_db.py +++ b/app/migrations/init_db.py @@ -16,6 +16,7 @@ def clear_db(): cur.execute('DROP TABLE IF EXISTS transactions') cur.execute('DROP TABLE IF EXISTS transfers') cur.execute('DROP TABLE IF EXISTS contracts') + cur.execute('DROP TABLE IF EXISTS miners') con.commit() con.close() @@ -111,6 +112,13 @@ def init_db(): legalparams TEXT) ''') + cur.execute(''' + CREATE TABLE IF NOT EXISTS miners + (id text NOT NULL PRIMARY KEY, + address text NOT NULL, + wallet_address text, + ) + ''') con.commit() con.close() diff --git a/app/types.py b/app/types.py index bc26c5b..59fdc99 100644 --- a/app/types.py +++ b/app/types.py @@ -1,8 +1,9 @@ """Type declarations""" TRANSACTION_WALLET_CREATION = 1 -TRANSCATION_TOKEN_CREATION = 2 +TRANSACTION_TOKEN_CREATION = 2 TRANSACTION_SMART_CONTRACT = 3 TRANSACTION_TWO_WAY_TRANSFER = 4 TRANSACTION_ONE_WAY_TRANSFER = 5 TRANSACTION_TRUST_SCORE_CHANGE = 6 +TRANSACTION_MINER_ADDITION = 7 From 0e4348a41c36e69aaaa3b8e94ed06b4d7fe05f9d Mon Sep 17 00:00:00 2001 From: Kousthub Raja Date: Sun, 27 Feb 2022 23:42:05 +0530 Subject: [PATCH 2/8] miner addition changes --- app/codes/kycwallet.py | 3 ++- app/codes/minermanager.py | 28 ++++++++++++++++++++++++++++ app/codes/tokenmanager.py | 5 ++--- app/codes/transactionmanager.py | 6 ++++-- app/migrations/init_db.py | 8 ++++++++ app/ntypes.py | 5 +++-- 6 files changed, 47 insertions(+), 8 deletions(-) create mode 100644 app/codes/minermanager.py diff --git a/app/codes/kycwallet.py b/app/codes/kycwallet.py index 86540da..6b8df8a 100644 --- a/app/codes/kycwallet.py +++ b/app/codes/kycwallet.py @@ -8,6 +8,7 @@ import base64 import sqlite3 +from .utils import get_time_ms from ..constants import TMP_PATH, NEWRL_DB from .transactionmanager import Transactionmanager @@ -113,7 +114,7 @@ def generate_wallet(kyccustodian, kycdocs, ownertype, jurisd, wallet_specific_da def create_add_wallet_transaction(wallet): transaction_data = { - 'timestamp': str(datetime.datetime.now()), + 'timestamp': get_time_ms(), 'type': 1, 'currency': 'INR', 'fee': 0.0, diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py new file mode 100644 index 0000000..d0396f0 --- /dev/null +++ b/app/codes/minermanager.py @@ -0,0 +1,28 @@ +"""Miner update functions""" +import datetime + +from .auth.auth import get_wallet +from .signmanager import sign_transaction +from ..types import TRANSACTION_MINER_ADDITION +from .utils import get_time_ms +from .transactionmanager import Transactionmanager + + +def miner_addition_transaction(): + transaction_data = { + 'timestamp': get_time_ms(), + 'type': TRANSACTION_MINER_ADDITION, + 'currency': "NWRL", + 'fee': 0.0, + 'descr': "Miner addition", + 'valid': 1, + 'block_index': 0, + 'specific_data': {} + } + + transaction = Transactionmanager() + transaction_data = {'transaction': transaction_data, 'signatures': []} + transaction.transactioncreator(transaction_data) + wallet = get_wallet() + signed_transaction = sign_transaction(wallet, transaction_data) + return signed_transaction diff --git a/app/codes/tokenmanager.py b/app/codes/tokenmanager.py index d33419e..422f927 100644 --- a/app/codes/tokenmanager.py +++ b/app/codes/tokenmanager.py @@ -1,13 +1,12 @@ # Python programm to create object that enables addition of a block -import datetime - +from .utils import get_time_ms from .transactionmanager import Transactionmanager def create_token_transaction(token_data): transaction = { - 'timestamp': str(datetime.datetime.now()), + 'timestamp': get_time_ms(), 'type': 2, 'currency': "INR", 'fee': 0.0, diff --git a/app/codes/transactionmanager.py b/app/codes/transactionmanager.py index 01063f6..33da76d 100644 --- a/app/codes/transactionmanager.py +++ b/app/codes/transactionmanager.py @@ -8,7 +8,9 @@ import base64 import sqlite3 -from ..ntypes import TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION, TRANSCATION_TOKEN_CREATION + +from ..ntypes import TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION, TRANSACTION_TOKEN_CREATION + from .chainscanner import get_wallet_token_balance from ..constants import ALLOWED_CUSTODIANS_FILE, MEMPOOL_PATH, NEWRL_DB from .utils import get_time_ms @@ -540,7 +542,7 @@ def get_valid_addresses(transaction): if transaction_type == TRANSACTION_WALLET_CREATION: # Custodian needs to sign valid_addresses.append( transaction['specific_data']['custodian_wallet']) - if transaction_type == TRANSCATION_TOKEN_CREATION: # Custodian needs to sign + if transaction_type == TRANSACTION_TOKEN_CREATION: # Custodian needs to sign valid_addresses.append(transaction['specific_data']['custodian']) if transaction_type == TRANSACTION_SMART_CONTRACT: valid_addresses = get_sc_validadds(transaction) diff --git a/app/migrations/init_db.py b/app/migrations/init_db.py index 6041d92..00c6dce 100644 --- a/app/migrations/init_db.py +++ b/app/migrations/init_db.py @@ -16,6 +16,7 @@ def clear_db(): cur.execute('DROP TABLE IF EXISTS transactions') cur.execute('DROP TABLE IF EXISTS transfers') cur.execute('DROP TABLE IF EXISTS contracts') + cur.execute('DROP TABLE IF EXISTS miners') con.commit() con.close() @@ -112,6 +113,13 @@ def init_db(): legalparams TEXT) ''') + cur.execute(''' + CREATE TABLE IF NOT EXISTS miners + (id text NOT NULL PRIMARY KEY, + address text NOT NULL, + wallet_address text, + ) + ''') con.commit() con.close() diff --git a/app/ntypes.py b/app/ntypes.py index 6870c77..d60a913 100644 --- a/app/ntypes.py +++ b/app/ntypes.py @@ -1,12 +1,13 @@ """Type declarations""" TRANSACTION_WALLET_CREATION = 1 -TRANSCATION_TOKEN_CREATION = 2 +TRANSACTION_TOKEN_CREATION = 2 TRANSACTION_SMART_CONTRACT = 3 TRANSACTION_TWO_WAY_TRANSFER = 4 TRANSACTION_ONE_WAY_TRANSFER = 5 TRANSACTION_TRUST_SCORE_CHANGE = 6 +TRANSACTION_MINER_ADDITION = 7 NEWRL_TOKEN_CODE = 'NWRL' NUSD_TOKEN_CODE = 'NUSD' -NEWRL_TOKEN_NAME = 'Newrl' \ No newline at end of file +NEWRL_TOKEN_NAME = 'Newrl' From 23f0b59f7c5eee7f711aa229a2b5ac116aa1c177 Mon Sep 17 00:00:00 2001 From: Kousthub Raja Date: Mon, 28 Feb 2022 10:33:51 +0530 Subject: [PATCH 3/8] miner add transaction, mine timer --- app/codes/auth/auth.py | 1 - app/codes/clock/global_time.py | 15 ++- app/codes/db_updater.py | 7 ++ app/codes/minermanager.py | 100 ++++++++++++++++-- app/codes/p2p/peers.py | 4 +- app/codes/p2p/utils.py | 5 + app/codes/state_updater.py | 20 ++-- app/codes/transactionmanager.py | 22 ++-- app/codes/validator.py | 37 +++---- app/constants.py | 1 + app/main.py | 22 ++-- app/migrations/init_db.py | 6 +- .../migrations/3_init_newrl_tokens.py | 13 ++- app/tests/conftest.py | 7 +- app/tests/test_fee_rewards.py | 8 +- app/tests/test_miner_committee.py | 35 ++++++ 16 files changed, 244 insertions(+), 59 deletions(-) create mode 100644 app/tests/test_miner_committee.py diff --git a/app/codes/auth/auth.py b/app/codes/auth/auth.py index fe16f93..53f6186 100644 --- a/app/codes/auth/auth.py +++ b/app/codes/auth/auth.py @@ -36,7 +36,6 @@ def get_auth(): 'public': wallet['public'], } auth_data['signature'] = sign_object(private_key, auth_data) - print('auth', auth_data) return auth_data except: auth_data = {} diff --git a/app/codes/clock/global_time.py b/app/codes/clock/global_time.py index 008afec..acbbade 100644 --- a/app/codes/clock/global_time.py +++ b/app/codes/clock/global_time.py @@ -2,7 +2,9 @@ import time import requests import threading -from ...constants import BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL +from ..minermanager import broadcast_miner_update, should_i_mine +from ...constants import BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL, TIME_MINER_BROADCAST_INTERVAL +from ..updater import run_updater def get_global_epoch(): @@ -23,7 +25,9 @@ def no_receipt_timeout(): def mine(): - print('Mining block.') + if should_i_mine(): + print('Mining block.') + run_updater() def start_receipt_timeout(): @@ -37,6 +41,13 @@ def start_mining_clock(): timer.start() +def start_miner_broadcast_clock(): + print('Broadcasting miner update') + broadcast_miner_update() + timer = threading.Timer(TIME_MINER_BROADCAST_INTERVAL, start_miner_broadcast_clock) + timer.start() + + def get_time_difference(): """Return the time difference between local and global in seconds""" global_epoch = get_global_epoch() diff --git a/app/codes/db_updater.py b/app/codes/db_updater.py index c494f2c..71d6869 100644 --- a/app/codes/db_updater.py +++ b/app/codes/db_updater.py @@ -269,3 +269,10 @@ def input_to_dict(ipval): else: callparams = ipval return callparams + + +def add_miner(cur, wallet_address, network_address, broadcast_timestamp): + cur.execute('''INSERT OR REPLACE INTO miners + (id, wallet_address, network_address, last_broadcast_timestamp) + VALUES (?, ?, ?, ?)''', + (wallet_address, wallet_address, network_address, broadcast_timestamp)) diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py index d0396f0..fcdb0e0 100644 --- a/app/codes/minermanager.py +++ b/app/codes/minermanager.py @@ -1,28 +1,112 @@ """Miner update functions""" -import datetime +import sqlite3 +import random +from .blockchain import get_last_block_hash +from .p2p.utils import get_my_address +from ..constants import COMMITTEE_SIZE, NEWRL_DB from .auth.auth import get_wallet from .signmanager import sign_transaction -from ..types import TRANSACTION_MINER_ADDITION +from ..ntypes import TRANSACTION_MINER_ADDITION from .utils import get_time_ms from .transactionmanager import Transactionmanager +from .validator import validate def miner_addition_transaction(): + wallet = get_wallet() + my_address = get_my_address() + timestamp = get_time_ms() transaction_data = { - 'timestamp': get_time_ms(), + 'timestamp': timestamp, 'type': TRANSACTION_MINER_ADDITION, 'currency': "NWRL", 'fee': 0.0, 'descr': "Miner addition", 'valid': 1, 'block_index': 0, - 'specific_data': {} + 'specific_data': { + 'wallet_address': wallet['address'], + 'network_address': my_address, + 'broadcast_timestamp': timestamp + } } - transaction = Transactionmanager() + transaction_manager = Transactionmanager() transaction_data = {'transaction': transaction_data, 'signatures': []} - transaction.transactioncreator(transaction_data) - wallet = get_wallet() - signed_transaction = sign_transaction(wallet, transaction_data) + transaction_manager.transactioncreator(transaction_data) + transaction = transaction_manager.get_transaction_complete() + signed_transaction = sign_transaction(wallet, transaction) return signed_transaction + + +def get_miner_status(wallet_address): + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + miner_cursor = cur.execute( + 'SELECT wallet_address, network_address, last_broadcast_timestamp FROM miners WHERE wallet_address=?', (wallet_address, )).fetchone() + if miner_cursor is None: + return None + miner_info = { + 'wallet_address': miner_cursor[0], + 'network_address': miner_cursor[1], + 'broadcast_timestamp': miner_cursor[2] + } + return miner_info + + +def get_my_miner_status(): + wallet = get_wallet() + my_status = get_miner_status(wallet['address']) + return my_status + + +def broadcast_miner_update(): + transaction = miner_addition_transaction() + validate(transaction) + + +def get_miner_list(): + con = sqlite3.connect(NEWRL_DB) + con.row_factory = sqlite3.Row + cur = con.cursor() + miner_cursor = cur.execute( + 'SELECT wallet_address, network_address, last_broadcast_timestamp FROM miners ORDER BY wallet_address ASC').fetchall() + miners = [dict(m) for m in miner_cursor] + return miners + + +def get_miner_for_current_block(): + last_block = get_last_block_hash() + + if not last_block: + return + + # TODO - Use hash instead of index + # random.seed takes integer arguments; hash need to convert to int + random.seed(last_block['index']) + + miners = get_miner_list() + miner = random.choice(miners) + return miner + + +def get_committee_for_current_block(): + last_block = get_last_block_hash() + + if not last_block: + return + + random.seed(last_block['index']) + + miners = get_miner_list() + committee = random.choices(miners, k=COMMITTEE_SIZE) + return committee + + +def should_i_mine(): + my_wallet = get_wallet() + miner = get_miner_for_current_block() + if miner['wallet_address'] == my_wallet['address']: + return True + return False \ No newline at end of file diff --git a/app/codes/p2p/peers.py b/app/codes/p2p/peers.py index 429f0c9..ca5a0e1 100644 --- a/app/codes/p2p/peers.py +++ b/app/codes/p2p/peers.py @@ -9,6 +9,7 @@ from app.migrations.init import init_newrl from ...constants import BOOTSTRAP_NODES, REQUEST_TIMEOUT, NEWRL_P2P_DB, NEWRL_PORT, MY_ADDRESS from ..auth.auth import get_auth +from .utils import get_my_address logging.basicConfig(level=logging.INFO) @@ -154,9 +155,6 @@ def update_peers(): print('Error updating software on peer', str(e)) return True -def get_my_address(): - return requests.get('https://api.ipify.org?format=json').json()['ip'] - def update_my_address(): MY_ADDRESS = get_my_address() diff --git a/app/codes/p2p/utils.py b/app/codes/p2p/utils.py index ea7c019..f794d61 100644 --- a/app/codes/p2p/utils.py +++ b/app/codes/p2p/utils.py @@ -1,4 +1,5 @@ import sqlite3 +import requests from ...constants import NEWRL_P2P_DB @@ -11,3 +12,7 @@ def get_peers(): peer_cursor = cur.execute('SELECT * FROM peers').fetchall() peers = [dict(ix) for ix in peer_cursor] return peers + + +def get_my_address(): + return requests.get('https://api.ipify.org?format=json').json()['ip'] diff --git a/app/codes/state_updater.py b/app/codes/state_updater.py index 578462b..17d676f 100644 --- a/app/codes/state_updater.py +++ b/app/codes/state_updater.py @@ -6,7 +6,7 @@ from ..constants import NEWRL_DB from .db_updater import * -from ..ntypes import NEWRL_TOKEN_CODE, NEWRL_TOKEN_NAME +from ..ntypes import NEWRL_TOKEN_CODE, NEWRL_TOKEN_NAME, TRANSACTION_MINER_ADDITION, TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TOKEN_CREATION, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION def update_db_states(cur, block): @@ -42,13 +42,13 @@ def update_db_states(cur, block): def update_state_from_transaction(cur, transaction_type, transaction_data, transaction_code, transaction_timestamp): - if transaction_type == 1: # this is a wallet creation transaction + if transaction_type == TRANSACTION_WALLET_CREATION: # this is a wallet creation transaction add_wallet_pid(cur, transaction_data) - if transaction_type == 2: # this is a token creation or addition transaction + if transaction_type == TRANSACTION_TOKEN_CREATION: # this is a token creation or addition transaction add_token(cur, transaction_data, transaction_code) - if transaction_type == 4 or transaction_type == 5: # this is a transfer tx + if transaction_type == TRANSACTION_TWO_WAY_TRANSFER or transaction_type == TRANSACTION_ONE_WAY_TRANSFER: # this is a transfer tx sender1 = transaction_data['wallet1'] sender2 = transaction_data['wallet2'] @@ -62,14 +62,14 @@ def update_state_from_transaction(cur, transaction_type, transaction_data, trans transfer_tokens_and_update_balances( cur, sender2, sender1, tokencode2, amount2) - if transaction_type == 6: # score update transaction + if transaction_type == TRANSACTION_TRUST_SCORE_CHANGE: # score update transaction personid1 = get_pid_from_wallet(cur, transaction_data['address1']) personid2 = get_pid_from_wallet(cur, transaction_data['address2']) new_score = transaction_data['new_score'] tstamp = transaction_timestamp update_trust_score(cur, personid1, personid2, new_score, tstamp) - if transaction_type == 3: # smart contract transaction + if transaction_type == TRANSACTION_SMART_CONTRACT: # smart contract transaction funct = transaction_data['function'] if funct == "setup": # sc is being set up contract = dict(transaction_data['params']) @@ -84,6 +84,14 @@ def update_state_from_transaction(cur, transaction_type, transaction_data, trans # sc_instance = nusd1(transaction['specific_data']['address']) funct = getattr(sc_instance, funct) funct(cur, transaction_data['params']) + + if transaction_type == TRANSACTION_MINER_ADDITION: + add_miner( + cur, + transaction_data['wallet_address'], + transaction_data['network_address'], + transaction_data['broadcast_timestamp'], + ) def add_block_reward(cur, creator, blockindex): diff --git a/app/codes/transactionmanager.py b/app/codes/transactionmanager.py index 33da76d..3e7a11a 100644 --- a/app/codes/transactionmanager.py +++ b/app/codes/transactionmanager.py @@ -9,7 +9,7 @@ import sqlite3 -from ..ntypes import TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION, TRANSACTION_TOKEN_CREATION +from ..ntypes import TRANSACTION_MINER_ADDITION, TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION, TRANSACTION_TOKEN_CREATION from .chainscanner import get_wallet_token_balance from ..constants import ALLOWED_CUSTODIANS_FILE, MEMPOOL_PATH, NEWRL_DB @@ -201,7 +201,7 @@ def econvalidator(self): # from mempool only include transactions that reduce balance and not those that increase # check if the sender has enough balance to spend self.validity = 0 - if self.transaction['type'] == 1: + if self.transaction['type'] == TRANSACTION_WALLET_CREATION: custodian = self.transaction['specific_data']['custodian_wallet'] walletaddress = self.transaction['specific_data']['wallet_address'] if not is_wallet_valid(custodian): @@ -246,7 +246,7 @@ def econvalidator(self): self.validity = 0 # self.validity=0 - if self.transaction['type'] == 2: # token addition transaction + if self.transaction['type'] == TRANSACTION_TOKEN_CREATION: # token addition transaction firstowner = self.transaction['specific_data']['first_owner'] custodian = self.transaction['specific_data']['custodian'] fovalidity = False @@ -300,7 +300,7 @@ def econvalidator(self): "Tokencode provided does not exist. Will append as new one.") self.validity = 1 # tokencode is provided by user - if self.transaction['type'] == 3: + if self.transaction['type'] == TRANSACTION_SMART_CONTRACT: self.validity = 1 for wallet in self.transaction['specific_data']['signers']: if not is_wallet_valid(wallet): @@ -311,7 +311,7 @@ def econvalidator(self): self.validity = 0 # self.validity=0 - if self.transaction['type'] == 4 or self.transaction['type'] == 5: + if self.transaction['type'] == TRANSACTION_TWO_WAY_TRANSFER or self.transaction['type'] == TRANSACTION_ONE_WAY_TRANSFER: ttype = self.transaction['type'] startingbalance1 = 0 startingbalance2 = 0 @@ -402,7 +402,7 @@ def econvalidator(self): # self.transaction['valid']=1; self.validity = 1 - if self.transaction['type'] == 6: # score change transaction + if self.transaction['type'] == TRANSACTION_TRUST_SCORE_CHANGE: # score change transaction ttype = self.transaction['type'] # personid1 = self.transaction['specific_data']['personid1'] # personid2 = self.transaction['specific_data']['personid2'] @@ -428,6 +428,14 @@ def econvalidator(self): self.validity = 0 else: self.validity = 1 + + if self.transaction['type'] == TRANSACTION_MINER_ADDITION: + # No checks for fee in the beginning + if not is_wallet_valid(self.transaction['specific_data']['wallet_address']): + print("Miner wallet not in chain") + self.validity = 0 + else: + self.validity = 1 if self.validity == 1: return True @@ -554,4 +562,6 @@ def get_valid_addresses(transaction): if transaction_type == TRANSACTION_TRUST_SCORE_CHANGE: # Only address1 is added, not address2 valid_addresses.append(transaction['specific_data']['address1']) + if transaction_type == TRANSACTION_MINER_ADDITION: + valid_addresses.append(transaction['specific_data']['wallet_address']) return valid_addresses diff --git a/app/codes/validator.py b/app/codes/validator.py index 71fe4db..43ab18e 100644 --- a/app/codes/validator.py +++ b/app/codes/validator.py @@ -11,7 +11,7 @@ from app.codes.p2p.transport import send from .blockchain import get_last_block_hash from .transactionmanager import Transactionmanager -from ..constants import MEMPOOL_PATH +from ..constants import IS_TEST, MEMPOOL_PATH from .p2p.outgoing import propogate_transaction_to_peers @@ -26,32 +26,33 @@ def validate(transaction): signatures_valid = transaction_manager.verifytransigns() valid = False if economics_valid and signatures_valid: - msg = "All well" + msg = "Transaction is valid" valid = True if not economics_valid: - msg = "Economic validation failed" + msg = "Transaction economic validation failed" if not signatures_valid: - msg = "Invalid signatures" + msg = "Transaction has invalid signatures" check = {'valid': valid, 'msg': msg} if valid: # Economics and signatures are both valid transaction_file = f"{MEMPOOL_PATH}transaction-{transaction_manager.transaction['type']}-{transaction_manager.transaction['trans_code']}.json" transaction_manager.save_transaction_to_mempool(transaction_file) - # Broadcast transaction to peers - propogate_transaction_to_peers(transaction_manager.get_transaction_complete()) - - # Broadcaset transaction via transport server - try: - payload = { - 'operation': 'send_transaction', - 'data': transaction_manager.get_transaction_complete() - } - send(payload) - except: - print('Error sending transaction to transport server') - - print(check) + if not IS_TEST: + # Broadcast transaction to peers via HTTP + propogate_transaction_to_peers(transaction_manager.get_transaction_complete()) + + # Broadcaset transaction via transport server + try: + payload = { + 'operation': 'send_transaction', + 'data': transaction_manager.get_transaction_complete() + } + send(payload) + except: + print('Error sending transaction to transport server') + + print(msg) return check diff --git a/app/constants.py b/app/constants.py index 7609edd..1a871b1 100644 --- a/app/constants.py +++ b/app/constants.py @@ -39,6 +39,7 @@ TIME_DIFF_WITH_GLOBAL = 0 MAX_ALLOWED_TIME_DIFF_SECONDS = 10 BLOCK_TIME_INTERVAL_SECONDS = 30 +TIME_MINER_BROADCAST_INTERVAL = 60 * 60 # Miner broadcast every hour MY_ADDRESS = '' ALLOWED_FEE_PAYMENT_TOKENS = [NEWRL_TOKEN_CODE, NUSD_TOKEN_CODE] \ No newline at end of file diff --git a/app/main.py b/app/main.py index 342b30e..41bcde6 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,6 @@ import logging import argparse +import os import uvicorn from fastapi.openapi.utils import get_openapi from fastapi import FastAPI @@ -9,7 +10,7 @@ from .constants import NEWRL_PORT from .codes.p2p.peers import init_bootstrap_nodes, update_my_address, update_software -from .codes.clock.global_time import start_mining_clock, update_time_difference +from .codes.clock.global_time import start_mining_clock, update_time_difference, start_miner_broadcast_clock from .routers import blockchain from .routers import p2p @@ -19,11 +20,6 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -parser = argparse.ArgumentParser() -parser.add_argument("--disablenetwork", help="run the node local only with no network connection", action="store_true") -parser.add_argument("--disableupdate", help="run the node without updating software", action="store_true") -parser.add_argument("--disablebootstrap", help="run the node without bootstrapping", action="store_true") -args = parser.parse_args() app = FastAPI( title="The Newrl APIs", @@ -44,6 +40,8 @@ app.include_router(p2p.router) app.include_router(transport.router) +args = {} + @app.on_event('startup') def app_startup(): try: @@ -55,12 +53,22 @@ def app_startup(): sync_chain_from_peers() update_time_difference() update_my_address() - # start_mining_clock() + start_miner_broadcast_clock() except Exception as e: print('Bootstrap failed') logging.critical(e, exc_info=True) +@app.on_event("shutdown") +def shutdown_event(): + print('Shutting down node') + os._exit(0) + if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--disablenetwork", help="run the node local only with no network connection", action="store_true") + parser.add_argument("--disableupdate", help="run the node without updating software", action="store_true") + parser.add_argument("--disablebootstrap", help="run the node without bootstrapping", action="store_true") + args = parser.parse_args() uvicorn.run("app.main:app", host="0.0.0.0", port=NEWRL_PORT, reload=True) diff --git a/app/migrations/init_db.py b/app/migrations/init_db.py index 00c6dce..5e84e52 100644 --- a/app/migrations/init_db.py +++ b/app/migrations/init_db.py @@ -112,12 +112,14 @@ def init_db(): contractspecs TEXT, legalparams TEXT) ''') - + cur.execute('DROP TABLE IF EXISTS miners') cur.execute(''' CREATE TABLE IF NOT EXISTS miners (id text NOT NULL PRIMARY KEY, - address text NOT NULL, wallet_address text, + network_address text NOT NULL, + last_broadcast_timestamp text, + UNIQUE (wallet_address) ) ''') con.commit() diff --git a/app/migrations/migrations/3_init_newrl_tokens.py b/app/migrations/migrations/3_init_newrl_tokens.py index 86e53f0..da3ce12 100644 --- a/app/migrations/migrations/3_init_newrl_tokens.py +++ b/app/migrations/migrations/3_init_newrl_tokens.py @@ -20,6 +20,8 @@ def init_newrl_tokens(): cur = con.cursor() create_newrl_tokens(cur, FOUNDATION_RESERVE * 2) + create_wallet(cur, FOUNDATION_WALLET, FOUNDATION_PUBLIC_KEY) + create_wallet(cur, ASQI_WALLET, ASQI_PUBLIC_KEY) credit_wallet(cur, FOUNDATION_WALLET, FOUNDATION_RESERVE) credit_wallet(cur, ASQI_WALLET, FOUNDATION_RESERVE) @@ -41,7 +43,16 @@ def create_newrl_tokens(cur, amount): (tokencode, tokenname, tokentype, amount_created, sc_flag, tokendecimal, token_attributes) VALUES (?, ?, ?, ?, ?, ?, ?)''', query_params) - + +def create_wallet(cur, wallet_address, wallet_public): + query_params = (wallet_address, + wallet_public, + '', '{}', '1', '91', '{}' + ) + cur.execute(f'''INSERT OR IGNORE INTO wallets + (wallet_address, wallet_public, custodian_wallet, kyc_docs, + owner_type, jurisdiction, specific_data) + VALUES (?, ?, ?, ?, ?, ?, ?)''', query_params) def credit_wallet(cur, wallet, amount): cur.execute(f'''INSERT OR IGNORE INTO balances diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 9bd6742..6c83dcf 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -1,6 +1,9 @@ import os import shutil -from app.codes.p2p.peers import init_peer_db + +from ..migrations.init import init_newrl +from ..codes.p2p.peers import init_peer_db +from ..migrations.migrate_db import run_migrations import pytest @@ -17,7 +20,9 @@ def setup_test_files(): if os.path.exists('data_test/.auth.json'): os.remove('data_test/.auth.json') shutil.copyfile('data_test/template/.auth.json', 'data_test/.auth.json') + init_newrl() init_peer_db() + run_migrations() os.environ['NEWRL_TEST'] = '1' diff --git a/app/tests/test_fee_rewards.py b/app/tests/test_fee_rewards.py index 8d47e59..99dd628 100644 --- a/app/tests/test_fee_rewards.py +++ b/app/tests/test_fee_rewards.py @@ -32,21 +32,21 @@ def test_mining_reward(): wallet_address = wallet['wallet_address'] assert wallet_address - check_newrl_wallet_balance(wallet_address, None) + check_newrl_wallet_balance(wallet_address, 1500000000.0) response = client.post('/run-updater') assert response.status_code == 200 - check_newrl_wallet_balance(wallet_address, 1000) + check_newrl_wallet_balance(wallet_address, 1500001000.0) time.sleep(2) response = client.post('/run-updater') assert response.status_code == 200 - check_newrl_wallet_balance(wallet_address, 1000) + check_newrl_wallet_balance(wallet_address, 1500001000.0) time.sleep(5) response = client.post('/run-updater') assert response.status_code == 200 - check_newrl_wallet_balance(wallet_address, 2000) + check_newrl_wallet_balance(wallet_address, 1500002000.0) def test_transaction_fee_payment(): diff --git a/app/tests/test_miner_committee.py b/app/tests/test_miner_committee.py new file mode 100644 index 0000000..42e464d --- /dev/null +++ b/app/tests/test_miner_committee.py @@ -0,0 +1,35 @@ +import time +import sqlite3 + +from ..codes.auth.auth import get_wallet +from ..codes.minermanager import broadcast_miner_update, get_my_miner_status +from ..codes.db_updater import update_wallet_token_balance +from fastapi.testclient import TestClient +from ..ntypes import NUSD_TOKEN_CODE +from ..constants import NEWRL_DB +from ..nvalues import TREASURY_WALLET_ADDRESS +from ..migrations.init import init_newrl + +from ..main import app + +client = TestClient(app) + + +def test_mining_reward(): + assert None == get_my_miner_status() + + broadcast_miner_update() + response = client.post('/run-updater') + assert response.status_code == 200 + + miner_info = get_my_miner_status() + assert miner_info['wallet_address'] == get_wallet()['address'] + + + +def set_balance(wallet, token, balance): + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + update_wallet_token_balance(cur, wallet, token, balance) + con.commit() + con.close() From a7eb13b9149ddb16da80feaea93718bb32169745 Mon Sep 17 00:00:00 2001 From: Kousthub Raja Date: Mon, 28 Feb 2022 14:25:59 +0530 Subject: [PATCH 4/8] miner list filtering --- app/codes/minermanager.py | 8 ++++++-- app/main.py | 15 ++++++++++----- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py index fcdb0e0..224b3db 100644 --- a/app/codes/minermanager.py +++ b/app/codes/minermanager.py @@ -4,7 +4,7 @@ from .blockchain import get_last_block_hash from .p2p.utils import get_my_address -from ..constants import COMMITTEE_SIZE, NEWRL_DB +from ..constants import COMMITTEE_SIZE, NEWRL_DB, TIME_MINER_BROADCAST_INTERVAL from .auth.auth import get_wallet from .signmanager import sign_transaction from ..ntypes import TRANSACTION_MINER_ADDITION @@ -70,8 +70,12 @@ def get_miner_list(): con = sqlite3.connect(NEWRL_DB) con.row_factory = sqlite3.Row cur = con.cursor() + cutfoff_epoch = get_time_ms() - TIME_MINER_BROADCAST_INTERVAL miner_cursor = cur.execute( - 'SELECT wallet_address, network_address, last_broadcast_timestamp FROM miners ORDER BY wallet_address ASC').fetchall() + '''SELECT wallet_address, network_address, last_broadcast_timestamp + FROM miners + WHERE last_broadcast_timestamp > ? + ORDER BY wallet_address ASC''', (cutfoff_epoch, )).fetchall() miners = [dict(m) for m in miner_cursor] return miners diff --git a/app/main.py b/app/main.py index 41bcde6..2c8c16c 100644 --- a/app/main.py +++ b/app/main.py @@ -40,15 +40,19 @@ app.include_router(p2p.router) app.include_router(transport.router) -args = {} +args = { + 'disablenetwork': False, + 'disableupdate': False, + 'disablebootstrap': False, +} @app.on_event('startup') def app_startup(): try: - if not args.disablenetwork: - if not args.disableupdate: + if not args['disablenetwork']: + if not args['disableupdate']: update_software(propogate=False) - if not args.disablebootstrap: + if not args['disablebootstrap']: init_bootstrap_nodes() sync_chain_from_peers() update_time_difference() @@ -68,7 +72,8 @@ def shutdown_event(): parser.add_argument("--disablenetwork", help="run the node local only with no network connection", action="store_true") parser.add_argument("--disableupdate", help="run the node without updating software", action="store_true") parser.add_argument("--disablebootstrap", help="run the node without bootstrapping", action="store_true") - args = parser.parse_args() + _args = parser.parse_args() + args["disablenetwork"] = _args.disablenetwork uvicorn.run("app.main:app", host="0.0.0.0", port=NEWRL_PORT, reload=True) From 1e81a612cbb619e07e22a15a3d4b414246390ad1 Mon Sep 17 00:00:00 2001 From: Kousthub Raja Date: Tue, 1 Mar 2022 09:29:54 +0530 Subject: [PATCH 5/8] more committee selection changes --- app/codes/blockchain.py | 5 +++-- app/codes/minermanager.py | 27 ++++++++++++--------------- 2 files changed, 15 insertions(+), 17 deletions(-) diff --git a/app/codes/blockchain.py b/app/codes/blockchain.py index be56df6..fd88852 100644 --- a/app/codes/blockchain.py +++ b/app/codes/blockchain.py @@ -167,7 +167,7 @@ def get_last_block_hash(): con = sqlite3.connect(NEWRL_DB) cur = con.cursor() last_block_cursor = cur.execute( - 'SELECT block_index, hash FROM blocks ORDER BY block_index DESC LIMIT 1' + 'SELECT block_index, hash, timestamp FROM blocks ORDER BY block_index DESC LIMIT 1' ) last_block = last_block_cursor.fetchone() con.close() @@ -175,7 +175,8 @@ def get_last_block_hash(): if last_block is not None: return { 'index': last_block[0], - 'hash': last_block[1] + 'hash': last_block[1], + 'timestamp': last_block[2] } else: return None diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py index 224b3db..22f7eec 100644 --- a/app/codes/minermanager.py +++ b/app/codes/minermanager.py @@ -66,33 +66,30 @@ def broadcast_miner_update(): validate(transaction) -def get_miner_list(): +def get_committee_list(): + last_block = get_last_block_hash() + if last_block: + cutfoff_epoch = last_block['timestamp'] - TIME_MINER_BROADCAST_INTERVAL + else: + cutfoff_epoch = 0 + con = sqlite3.connect(NEWRL_DB) con.row_factory = sqlite3.Row cur = con.cursor() - cutfoff_epoch = get_time_ms() - TIME_MINER_BROADCAST_INTERVAL miner_cursor = cur.execute( '''SELECT wallet_address, network_address, last_broadcast_timestamp FROM miners WHERE last_broadcast_timestamp > ? ORDER BY wallet_address ASC''', (cutfoff_epoch, )).fetchall() miners = [dict(m) for m in miner_cursor] + con.close() return miners def get_miner_for_current_block(): - last_block = get_last_block_hash() - - if not last_block: - return - - # TODO - Use hash instead of index - # random.seed takes integer arguments; hash need to convert to int - random.seed(last_block['index']) + committee_list = get_committee_list() - miners = get_miner_list() - miner = random.choice(miners) - return miner + return committee_list[0] def get_committee_for_current_block(): @@ -103,8 +100,8 @@ def get_committee_for_current_block(): random.seed(last_block['index']) - miners = get_miner_list() - committee = random.choices(miners, k=COMMITTEE_SIZE) + miners = get_committee_list() + committee = random.sample(miners, k=COMMITTEE_SIZE) return committee From 7e40bc5096898e08a6772952a1dbc1be9bd7f2b9 Mon Sep 17 00:00:00 2001 From: Kousthub Raja Date: Wed, 9 Mar 2022 15:56:11 +0530 Subject: [PATCH 6/8] block timers, empty block, miner validation --- app/codes/blockchain.py | 34 +++++++++++++++++++++ app/codes/clock/global_time.py | 30 +------------------ app/codes/clock/timers.py | 50 +++++++++++++++++++++++++++++++ app/codes/consensus/consensus.py | 12 +++++++- app/codes/minermanager.py | 33 +++++++++++++++----- app/codes/p2p/outgoing.py | 6 +++- app/codes/p2p/sync_chain.py | 4 ++- app/codes/updater.py | 23 ++++++++++++-- app/constants.py | 7 +++-- app/main.py | 3 +- app/tests/test_miner_committee.py | 43 +++++++++++++++++++++++--- app/tests/test_p2p.py | 31 ++++++++++++++++++- app/tests/test_timers.py | 45 ++++++++++++++++++++++++++++ 13 files changed, 271 insertions(+), 50 deletions(-) create mode 100644 app/codes/clock/timers.py create mode 100644 app/tests/test_timers.py diff --git a/app/codes/blockchain.py b/app/codes/blockchain.py index fd88852..e0615a1 100644 --- a/app/codes/blockchain.py +++ b/app/codes/blockchain.py @@ -109,6 +109,40 @@ def mine_block(self, cur, text, fees=0): block = self.create_block(cur, block, block_hash) return block + + def mine_empty_block(self, cur, text): + """Mine an empty block""" + print("Mining empty block") + last_block_cursor = cur.execute( + 'SELECT block_index, hash, timestamp FROM blocks ORDER BY block_index DESC LIMIT 1') + last_block = last_block_cursor.fetchone() + last_block_index = last_block[0] if last_block is not None else 0 + last_block_hash = last_block[1] if last_block is not None else 0 + last_block_timestamp = last_block[2] if last_block is not None else 0 + + EMPTY_BLOCK_NONCE = 42 + + try: + new_block_timestamp = int(last_block_timestamp) + 1 + except: + # For backward compatibility. Might need to refine. + new_block_timestamp = get_time_ms() + + block = { + 'index': last_block_index + 1, + 'timestamp': new_block_timestamp, + 'proof': EMPTY_BLOCK_NONCE, + 'text': text, + # 'creator_wallet': get_node_wallet_address(), + # 'fees': fees, + 'previous_hash': last_block_hash + } + + block_hash = self.calculate_hash(block) + print("New block hash is ", block_hash) + + block = self.create_block(cur, block, block_hash) + return block def get_latest_ts(self, cur=None): """Get the timestamp of latest block""" diff --git a/app/codes/clock/global_time.py b/app/codes/clock/global_time.py index acbbade..1a077cc 100644 --- a/app/codes/clock/global_time.py +++ b/app/codes/clock/global_time.py @@ -2,7 +2,7 @@ import time import requests import threading -from ..minermanager import broadcast_miner_update, should_i_mine +from ..minermanager import should_i_mine from ...constants import BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL, TIME_MINER_BROADCAST_INTERVAL from ..updater import run_updater @@ -20,34 +20,6 @@ def get_local_epoch(): return epoch_time -def no_receipt_timeout(): - print('No receipts received. Timing out.') - - -def mine(): - if should_i_mine(): - print('Mining block.') - run_updater() - - -def start_receipt_timeout(): - timer = threading.Timer(NO_RECEIPT_COMMITTEE_TIMEOUT, no_receipt_timeout) - timer.start() - - -def start_mining_clock(): - mine() - timer = threading.Timer(BLOCK_TIME_INTERVAL_SECONDS, start_mining_clock) - timer.start() - - -def start_miner_broadcast_clock(): - print('Broadcasting miner update') - broadcast_miner_update() - timer = threading.Timer(TIME_MINER_BROADCAST_INTERVAL, start_miner_broadcast_clock) - timer.start() - - def get_time_difference(): """Return the time difference between local and global in seconds""" global_epoch = get_global_epoch() diff --git a/app/codes/clock/timers.py b/app/codes/clock/timers.py new file mode 100644 index 0000000..0b7084b --- /dev/null +++ b/app/codes/clock/timers.py @@ -0,0 +1,50 @@ +import sys +import time +import requests +import threading +from ..minermanager import broadcast_miner_update, get_miner_for_current_block, should_i_mine +from ...constants import BLOCK_RECEIVE_TIMEOUT_SECONDS, BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL, TIME_MINER_BROADCAST_INTERVAL +from ..updater import mine_empty_block, run_updater + + +def no_receipt_timeout(): + print('No receipts received. Timing out.') + + +def mine(): + if should_i_mine(): + print('I am the miner for this block.') + run_updater() + else: + miner = get_miner_for_current_block() + print(f"Miner for current block is {miner['wallet_address']}. Waiting to receive block.") + start_block_receive_timeout_clock() + +def start_receipt_timeout(): + timer = threading.Timer(NO_RECEIPT_COMMITTEE_TIMEOUT, no_receipt_timeout) + timer.start() + + +def start_mining_clock(): + mine() + timer = threading.Timer(BLOCK_TIME_INTERVAL_SECONDS, start_mining_clock) + timer.start() + + +def block_receive_timeout(): + miner = get_miner_for_current_block() + print(f"Block receive timed out from miner {miner['wallet_address']}") + block_index = mine_empty_block()['index'] + print(f"Mined new block {block_index}") + + +def start_block_receive_timeout_clock(): + timer = threading.Timer(BLOCK_RECEIVE_TIMEOUT_SECONDS, block_receive_timeout) + timer.start() + + +def start_miner_broadcast_clock(): + print('Broadcasting miner update') + broadcast_miner_update() + timer = threading.Timer(TIME_MINER_BROADCAST_INTERVAL, start_miner_broadcast_clock) + timer.start() diff --git a/app/codes/consensus/consensus.py b/app/codes/consensus/consensus.py index ba896fc..6ceb507 100644 --- a/app/codes/consensus/consensus.py +++ b/app/codes/consensus/consensus.py @@ -6,6 +6,7 @@ from ..fs.mempool_manager import append_receipt_to_block, get_receipts_from_storage from ...constants import MINIMUM_ACCEPTANCE_RATIO, MINIMUM_ACCEPTANCE_VOTES from ..auth.auth import get_wallet +from ..minermanager import get_miner_for_current_block try: @@ -64,4 +65,13 @@ def check_community_consensus(block): # if receipt_counts['postitive_receipt_count'] >= MINIMUM_ACCEPTANCE_VOTES and receipt_counts['total_receipt_count']: # return True - return False \ No newline at end of file + return False + + +def validate_block_miner(block): + miner_address = block['signature']['address'] + + expected_miner = get_miner_for_current_block()['wallet_address'] + + if miner_address != expected_miner: + raise Exception(f"Invalid miner {miner_address} for block {block['block_index']}. Expected {expected_miner}") \ No newline at end of file diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py index 22f7eec..be5d772 100644 --- a/app/codes/minermanager.py +++ b/app/codes/minermanager.py @@ -3,8 +3,9 @@ import random from .blockchain import get_last_block_hash +# from .p2p.outgoing import propogate_transaction_to_peers from .p2p.utils import get_my_address -from ..constants import COMMITTEE_SIZE, NEWRL_DB, TIME_MINER_BROADCAST_INTERVAL +from ..constants import COMMITTEE_SIZE, IS_TEST, NEWRL_DB, TIME_MINER_BROADCAST_INTERVAL from .auth.auth import get_wallet from .signmanager import sign_transaction from ..ntypes import TRANSACTION_MINER_ADDITION @@ -13,9 +14,11 @@ from .validator import validate -def miner_addition_transaction(): - wallet = get_wallet() - my_address = get_my_address() +def miner_addition_transaction(wallet=None, my_address=None): + if wallet is None: + wallet = get_wallet() + if my_address is None: + my_address = get_my_address() timestamp = get_time_ms() transaction_data = { 'timestamp': timestamp, @@ -68,8 +71,14 @@ def broadcast_miner_update(): def get_committee_list(): last_block = get_last_block_hash() + last_block_epoch = 0 + try: + # Need try catch to support older block timestamps + last_block_epoch = int(last_block['timestamp']) + except: + pass if last_block: - cutfoff_epoch = last_block['timestamp'] - TIME_MINER_BROADCAST_INTERVAL + cutfoff_epoch = last_block_epoch - TIME_MINER_BROADCAST_INTERVAL else: cutfoff_epoch = 0 @@ -87,9 +96,18 @@ def get_committee_list(): def get_miner_for_current_block(): + last_block = get_last_block_hash() + + if not last_block: + return + + random.seed(last_block['index']) + committee_list = get_committee_list() - return committee_list[0] + return random.choice(committee_list) + + # return committee_list[0] def get_committee_for_current_block(): @@ -101,7 +119,8 @@ def get_committee_for_current_block(): random.seed(last_block['index']) miners = get_committee_list() - committee = random.sample(miners, k=COMMITTEE_SIZE) + committee_size = min(COMMITTEE_SIZE, len(miners)) + committee = random.sample(miners, k=committee_size) return committee diff --git a/app/codes/p2p/outgoing.py b/app/codes/p2p/outgoing.py index a020baa..75d0cd4 100644 --- a/app/codes/p2p/outgoing.py +++ b/app/codes/p2p/outgoing.py @@ -1,11 +1,13 @@ import requests from threading import Thread -from ...constants import NEWRL_PORT, REQUEST_TIMEOUT, TRANSPORT_SERVER +from ...constants import IS_TEST, NEWRL_PORT, REQUEST_TIMEOUT, TRANSPORT_SERVER from ..p2p.utils import get_peers from ..p2p.utils import is_my_address def propogate_transaction_to_peers(transaction): + if IS_TEST: + return peers = get_peers() for peer in peers: @@ -25,6 +27,8 @@ def send_request_in_thread(url, data): thread.start() def send_request(url, data): + if IS_TEST: + return requests.post(url, json=data, timeout=REQUEST_TIMEOUT) def send(payload): diff --git a/app/codes/p2p/sync_chain.py b/app/codes/p2p/sync_chain.py index b18edc6..996ad3f 100644 --- a/app/codes/p2p/sync_chain.py +++ b/app/codes/p2p/sync_chain.py @@ -10,7 +10,7 @@ from app.codes.validator import validate_block, validate_block_data, validate_receipt_signature from app.codes.updater import broadcast_block from app.codes.fs.temp_manager import append_receipt_to_block, append_receipt_to_block_in_storage, get_blocks_for_index_from_storage, store_block_to_temp, store_receipt_to_temp -from app.codes.consensus.consensus import check_community_consensus +from app.codes.consensus.consensus import check_community_consensus, validate_block_miner logging.basicConfig(level=logging.INFO) @@ -40,6 +40,8 @@ def receive_block(block): if block_index > get_last_block_index() + 1: sync_chain_from_peers() + validate_block_miner(block) + validate_block(block, validate_receipts=False) # if check_community_consensus(block): diff --git a/app/codes/updater.py b/app/codes/updater.py index 959b49b..441d8d0 100644 --- a/app/codes/updater.py +++ b/app/codes/updater.py @@ -18,6 +18,7 @@ from .chainscanner import get_wallet_token_balance from .db_updater import transfer_tokens_and_update_balances from .p2p.outgoing import send_request_in_thread +from .auth.auth import get_wallet MAX_BLOCK_SIZE = 10 @@ -131,9 +132,12 @@ def run_updater(): def broadcast_block(block): peers = get_peers() - private_key = _private - public_key = _public + my_wallet = get_wallet() + private_key = my_wallet['private'] + public_key = my_wallet['public'] + address = my_wallet['address'] signature = { + 'address': address, 'public': public_key, 'msgsign': sign_object(private_key, block) } @@ -190,3 +194,18 @@ def pay_fee_for_transaction(cur, transaction): fee / len(payees) ) return True + + +def mine_empty_block(): + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + + blockchain = Blockchain() + + block = blockchain.mine_empty_block(cur, {'transactions': []}) + # update_db_states(cur, block) + + con.commit() + con.close() + + return block \ No newline at end of file diff --git a/app/constants.py b/app/constants.py index 9b8bd71..2873719 100644 --- a/app/constants.py +++ b/app/constants.py @@ -22,7 +22,7 @@ BOOTSTRAP_NODES = ['testnet.newrl.net'] REQUEST_TIMEOUT = 1 -NEWRL_PORT = 8090 +NEWRL_PORT = 8182 # Devnet NEWRL_TOKEN = "newrl_token" TREASURY = "treasury_address" COINBASE_SC = "coinbase_sc_address" @@ -38,8 +38,9 @@ # Variables TIME_DIFF_WITH_GLOBAL = 0 MAX_ALLOWED_TIME_DIFF_SECONDS = 10 -BLOCK_TIME_INTERVAL_SECONDS = 30 -TIME_MINER_BROADCAST_INTERVAL = 60 * 60 # Miner broadcast every hour +BLOCK_TIME_INTERVAL_SECONDS = TIME_BETWEEN_BLOCKS_SECONDS +BLOCK_RECEIVE_TIMEOUT_SECONDS = 3 +TIME_MINER_BROADCAST_INTERVAL = 60 * 60 * 1000 # Miner broadcast every hour MY_ADDRESS = '' ALLOWED_FEE_PAYMENT_TOKENS = [NEWRL_TOKEN_CODE, NUSD_TOKEN_CODE] \ No newline at end of file diff --git a/app/main.py b/app/main.py index 2c8c16c..24e5f2c 100644 --- a/app/main.py +++ b/app/main.py @@ -10,7 +10,8 @@ from .constants import NEWRL_PORT from .codes.p2p.peers import init_bootstrap_nodes, update_my_address, update_software -from .codes.clock.global_time import start_mining_clock, update_time_difference, start_miner_broadcast_clock +from .codes.clock.global_time import update_time_difference +from .codes.clock.timers import start_mining_clock, start_miner_broadcast_clock from .routers import blockchain from .routers import p2p diff --git a/app/tests/test_miner_committee.py b/app/tests/test_miner_committee.py index 42e464d..8d20af9 100644 --- a/app/tests/test_miner_committee.py +++ b/app/tests/test_miner_committee.py @@ -1,9 +1,11 @@ import time import sqlite3 +from ..codes.blockchain import get_last_block_hash +from ..codes.utils import get_time_ms from ..codes.auth.auth import get_wallet -from ..codes.minermanager import broadcast_miner_update, get_my_miner_status -from ..codes.db_updater import update_wallet_token_balance +from ..codes.minermanager import broadcast_miner_update, get_committee_for_current_block, get_committee_list, get_miner_for_current_block, get_my_miner_status +from ..codes.db_updater import add_miner, update_wallet_token_balance from fastapi.testclient import TestClient from ..ntypes import NUSD_TOKEN_CODE from ..constants import NEWRL_DB @@ -24,12 +26,45 @@ def test_mining_reward(): miner_info = get_my_miner_status() assert miner_info['wallet_address'] == get_wallet()['address'] + + +def test_miner_selection(): + for i in range(0,20): + _add_test_miner(i) + last_block1 = get_last_block_hash() + committee = get_committee_for_current_block() + assert len(committee) == 6 + + miner1 = get_miner_for_current_block() + # Check if pseudo-random with block index as seed returns the same miner + assert miner1['wallet_address'] == get_miner_for_current_block()['wallet_address'] + # my_wallet = get_wallet() + # assert miner1['wallet_address'] == my_wallet['address'] + + time.sleep(5) + response = client.post('/run-updater') + assert response.status_code == 200 + + last_block2 = get_last_block_hash() + assert last_block2['index'] == last_block1['index'] + 1 + miner2 = get_miner_for_current_block() + + # Hoping the miner changes for the new block. Totally random though + assert miner1['wallet_address'] != miner2['wallet_address'] + + +def _add_test_miner(i): + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + add_miner(cur, f'0x0000{i}', '127.0.0.1', get_time_ms()) + con.commit() + con.close() -def set_balance(wallet, token, balance): +def clear_miner_db(): con = sqlite3.connect(NEWRL_DB) cur = con.cursor() - update_wallet_token_balance(cur, wallet, token, balance) + cur.execute('delete from miners') con.commit() con.close() diff --git a/app/tests/test_p2p.py b/app/tests/test_p2p.py index 073e302..260b188 100644 --- a/app/tests/test_p2p.py +++ b/app/tests/test_p2p.py @@ -1,5 +1,8 @@ from fastapi.testclient import TestClient +import pytest +from .test_miner_committee import _add_test_miner, clear_miner_db from ..migrations.init import init_newrl +from ..codes.minermanager import broadcast_miner_update from ..main import app @@ -53,6 +56,7 @@ def _receive_block(block_index): "previous_hash": "0000ae69c361c65f088b52af8f5372f94f5f62d84f0980ea4c2cd71551206024" }, "signature": { + "address": "0x20513a419d5b11cd510ae518dc04ac1690afbed6", "public": "4trPBhDwdxWat2I8tE4Mj+7R6tiTJ+44GWtTdf5QpXnh/Ia1i5x4ETDufrCn3mjYN8gJs/w3iiMlDEmAAs7kvg==", "msgsign": "8odtLy4zlyXNn7GFK4lpDtubGOS3bLFijmxXR1T8+TlLOl39+mA9Ajw8S4Sw3enJlGiWGorJr+0ULKdmeqf4Hw==" } @@ -72,6 +76,11 @@ def _receive_block(block_index): def test_block_receive(): + clear_miner_db() + + broadcast_miner_update() + response = client.post('/run-updater') + response = client.get('/get-last-block-index') assert response.status_code == 200 @@ -82,4 +91,24 @@ def test_block_receive(): current_block_index = int(response.text) # Block index should've increased by 1 - assert current_block_index == (previous_block_index + 1) \ No newline at end of file + assert current_block_index == (previous_block_index + 1) + + +def test_block_reject(): + """Expect block rejection from unexpected minors""" + clear_miner_db() + + _add_test_miner(1) + + response = client.get('/get-last-block-index') + assert response.status_code == 200 + + previous_block_index = int(response.text) + with pytest.raises(Exception) as e_info: + _receive_block(previous_block_index + 1) + + response = client.get('/get-last-block-index') + current_block_index = int(response.text) + + # Block index should not increase + assert current_block_index == previous_block_index \ No newline at end of file diff --git a/app/tests/test_timers.py b/app/tests/test_timers.py new file mode 100644 index 0000000..e078b30 --- /dev/null +++ b/app/tests/test_timers.py @@ -0,0 +1,45 @@ +import os +import time +import sqlite3 + +from .test_miner_committee import _add_test_miner +from ..codes.clock.timers import start_mining_clock +from ..codes.blockchain import get_last_block_hash +from ..codes.utils import get_time_ms +from ..codes.auth.auth import get_wallet +from ..codes.minermanager import broadcast_miner_update, get_committee_for_current_block, get_committee_list, get_miner_for_current_block, get_my_miner_status +from ..codes.db_updater import add_miner, update_wallet_token_balance +from fastapi.testclient import TestClient +from ..ntypes import NUSD_TOKEN_CODE +from ..constants import BLOCK_TIME_INTERVAL_SECONDS, NEWRL_DB +from ..nvalues import TREASURY_WALLET_ADDRESS +from ..migrations.init import init_newrl + +from ..main import app + +client = TestClient(app) + + +def test_mining_reward(): + broadcast_miner_update() + response = client.post('/run-updater') + assert response.status_code == 200 + + response = client.get('/get-last-block-index') + assert response.status_code == 200 + previous_block_index = int(response.text) + + start_mining_clock() + + time.sleep(BLOCK_TIME_INTERVAL_SECONDS + 2) + + response = client.get('/get-last-block-index') + assert response.status_code == 200 + block_index = int(response.text) + + assert block_index == previous_block_index + 1 + + for i in range(1, 20): + _add_test_miner(i) + # time.sleep(BLOCK_TIME_INTERVAL_SECONDS * 4) + # os._exit(1) \ No newline at end of file From cf23380f9bf607b5219cfd79616f391db9cdfc56 Mon Sep 17 00:00:00 2001 From: Kousthub Raja Date: Sun, 13 Mar 2022 09:30:53 +0530 Subject: [PATCH 7/8] block, receipt timings, committee selection --- app/codes/chainscanner.py | 1 - app/codes/clock/global_time.py | 31 ++++++++------ app/codes/clock/timers.py | 50 ---------------------- app/codes/minermanager.py | 14 +++++- app/codes/updater.py | 78 +++++++++++++++++++++++++++++++++- app/codes/utils.py | 23 ++++++++++ app/codes/validator.py | 2 +- app/constants.py | 4 +- app/main.py | 6 +-- app/tests/test_timers.py | 24 +++++++++-- 10 files changed, 156 insertions(+), 77 deletions(-) delete mode 100644 app/codes/clock/timers.py diff --git a/app/codes/chainscanner.py b/app/codes/chainscanner.py index bbabcc6..cf9d844 100644 --- a/app/codes/chainscanner.py +++ b/app/codes/chainscanner.py @@ -2,7 +2,6 @@ import sqlite3 from ..constants import NEWRL_DB -from .blockchain import Blockchain class Chainscanner(): diff --git a/app/codes/clock/global_time.py b/app/codes/clock/global_time.py index 1a077cc..335bea2 100644 --- a/app/codes/clock/global_time.py +++ b/app/codes/clock/global_time.py @@ -2,9 +2,7 @@ import time import requests import threading -from ..minermanager import should_i_mine -from ...constants import BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL, TIME_MINER_BROADCAST_INTERVAL -from ..updater import run_updater +from ...constants import BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL, TIME_DIFF_WITH_GLOBAL_FILE, TIME_MINER_BROADCAST_INTERVAL def get_global_epoch(): @@ -22,18 +20,25 @@ def get_local_epoch(): def get_time_difference(): """Return the time difference between local and global in seconds""" + try: + with open(TIME_DIFF_WITH_GLOBAL_FILE, 'r') as f: + return int(f.read()) + except: + global_epoch = get_global_epoch() + local_epoch = get_local_epoch() + diff = global_epoch - local_epoch + with open(TIME_DIFF_WITH_GLOBAL_FILE, 'w') as f: + f.write(str(diff)) + return diff + + +def sync_timer_clock_with_global(): global_epoch = get_global_epoch() local_epoch = get_local_epoch() - return global_epoch - local_epoch - - -def update_time_difference(): - TIME_DIFF_WITH_GLOBAL = get_time_difference() - print('Time difference with global is ', TIME_DIFF_WITH_GLOBAL) - if TIME_DIFF_WITH_GLOBAL > MAX_ALLOWED_TIME_DIFF_SECONDS: - print('System time is not syncronised. Time difference in seconds: ', TIME_DIFF_WITH_GLOBAL) - quit() - return True + diff = global_epoch - local_epoch + with open(TIME_DIFF_WITH_GLOBAL_FILE, 'w') as f: + f.write(str(diff)) + print('Synced clock. Time difference is', diff) if __name__ == '__main__': diff --git a/app/codes/clock/timers.py b/app/codes/clock/timers.py deleted file mode 100644 index 0b7084b..0000000 --- a/app/codes/clock/timers.py +++ /dev/null @@ -1,50 +0,0 @@ -import sys -import time -import requests -import threading -from ..minermanager import broadcast_miner_update, get_miner_for_current_block, should_i_mine -from ...constants import BLOCK_RECEIVE_TIMEOUT_SECONDS, BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL, TIME_MINER_BROADCAST_INTERVAL -from ..updater import mine_empty_block, run_updater - - -def no_receipt_timeout(): - print('No receipts received. Timing out.') - - -def mine(): - if should_i_mine(): - print('I am the miner for this block.') - run_updater() - else: - miner = get_miner_for_current_block() - print(f"Miner for current block is {miner['wallet_address']}. Waiting to receive block.") - start_block_receive_timeout_clock() - -def start_receipt_timeout(): - timer = threading.Timer(NO_RECEIPT_COMMITTEE_TIMEOUT, no_receipt_timeout) - timer.start() - - -def start_mining_clock(): - mine() - timer = threading.Timer(BLOCK_TIME_INTERVAL_SECONDS, start_mining_clock) - timer.start() - - -def block_receive_timeout(): - miner = get_miner_for_current_block() - print(f"Block receive timed out from miner {miner['wallet_address']}") - block_index = mine_empty_block()['index'] - print(f"Mined new block {block_index}") - - -def start_block_receive_timeout_clock(): - timer = threading.Timer(BLOCK_RECEIVE_TIMEOUT_SECONDS, block_receive_timeout) - timer.start() - - -def start_miner_broadcast_clock(): - print('Broadcasting miner update') - broadcast_miner_update() - timer = threading.Timer(TIME_MINER_BROADCAST_INTERVAL, start_miner_broadcast_clock) - timer.start() diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py index be5d772..de5afb4 100644 --- a/app/codes/minermanager.py +++ b/app/codes/minermanager.py @@ -2,7 +2,7 @@ import sqlite3 import random -from .blockchain import get_last_block_hash +from .utils import get_last_block_hash # from .p2p.outgoing import propogate_transaction_to_peers from .p2p.utils import get_my_address from ..constants import COMMITTEE_SIZE, IS_TEST, NEWRL_DB, TIME_MINER_BROADCAST_INTERVAL @@ -129,4 +129,14 @@ def should_i_mine(): miner = get_miner_for_current_block() if miner['wallet_address'] == my_wallet['address']: return True - return False \ No newline at end of file + return False + + +def am_i_in_current_committee(): + my_wallet_address = get_wallet()['address'] + committee = get_committee_for_current_block() + + found = list(filter(lambda w: w['wallet_address'] == my_wallet_address, committee)) + if len(found) == 0: + return False + return True \ No newline at end of file diff --git a/app/codes/updater.py b/app/codes/updater.py index 441d8d0..58edd17 100644 --- a/app/codes/updater.py +++ b/app/codes/updater.py @@ -3,10 +3,13 @@ import json import os import sqlite3 +import threading import requests +from .clock.global_time import get_time_difference +from .minermanager import am_i_in_current_committee, broadcast_miner_update, get_miner_for_current_block, should_i_mine from ..nvalues import TREASURY_WALLET_ADDRESS -from ..constants import ALLOWED_FEE_PAYMENT_TOKENS, IS_TEST, NEWRL_DB, NEWRL_PORT, REQUEST_TIMEOUT, MEMPOOL_PATH, TIME_BETWEEN_BLOCKS_SECONDS +from ..constants import ALLOWED_FEE_PAYMENT_TOKENS, BLOCK_RECEIVE_TIMEOUT_SECONDS, BLOCK_TIME_INTERVAL_SECONDS, IS_TEST, NEWRL_DB, NEWRL_PORT, NO_RECEIPT_COMMITTEE_TIMEOUT, REQUEST_TIMEOUT, MEMPOOL_PATH, TIME_BETWEEN_BLOCKS_SECONDS, TIME_MINER_BROADCAST_INTERVAL from .p2p.peers import get_peers from .p2p.utils import is_my_address from .utils import BufferedLog, get_time_ms @@ -208,4 +211,75 @@ def mine_empty_block(): con.commit() con.close() - return block \ No newline at end of file + return block + + +def no_receipt_timeout(): + print('Inadequate receipts. Timing out and sending empty block.') + + +def mine(): + if should_i_mine(): + print('I am the miner for this block.') + run_updater() + else: + miner = get_miner_for_current_block() + print(f"Miner for current block is {miner['wallet_address']}. Waiting to receive block.") + start_block_receive_timeout_clock() + +def start_receipt_timeout(): + timer = threading.Timer(NO_RECEIPT_COMMITTEE_TIMEOUT, no_receipt_timeout) + timer.start() + + +def start_mining_clock(): + mine() + timer = threading.Timer(BLOCK_TIME_INTERVAL_SECONDS, start_mining_clock) + timer.start() + + +def block_receive_timeout(): + miner = get_miner_for_current_block() + print(f"Block receive timed out from miner {miner['wallet_address']}") + block_index = mine_empty_block()['index'] + print(f"Mined new block {block_index}") + + +def start_block_receive_timeout_clock(): + timer = threading.Timer(BLOCK_RECEIVE_TIMEOUT_SECONDS, block_receive_timeout) + timer.start() + + +def start_miner_broadcast_clock(): + print('Broadcasting miner update') + broadcast_miner_update() + timer = threading.Timer(TIME_MINER_BROADCAST_INTERVAL, start_miner_broadcast_clock) + timer.start() + + +def block_receive_timeout(): + miner = get_miner_for_current_block() + print(f"Block receive timed out from miner {miner['wallet_address']}") + block_index = mine_empty_block()['index'] + print(f"Mined new block {block_index}") + + +def start_timers(block_timestamp): + print('Starting timer with timestamp ', block_timestamp) + + my_global_timestamp = get_time_ms() - get_time_difference() + propogation_delay = my_global_timestamp - block_timestamp + + # If I'm miner, start mining clock + if should_i_mine(): + wait_time = BLOCK_TIME_INTERVAL_SECONDS - propogation_delay + timer = threading.Timer(wait_time, mine) + timer.start() + elif am_i_in_current_committee(): + wait_time = NO_RECEIPT_COMMITTEE_TIMEOUT - propogation_delay + timer = threading.Timer(wait_time, block_receive_timeout) + timer.start() + else: + pass + # Not a miner and part of committee. No action to be performed + # Hoping the sentinel node will trigger an empty block start stagnant network \ No newline at end of file diff --git a/app/codes/utils.py b/app/codes/utils.py index 106063c..9e38fe3 100644 --- a/app/codes/utils.py +++ b/app/codes/utils.py @@ -1,6 +1,9 @@ import hashlib +import sqlite3 import time +from ..constants import NEWRL_DB + def save_file_and_get_path(upload_file): if upload_file is None: @@ -33,3 +36,23 @@ def get_person_id_for_wallet_address(wallet_address): hs.update(wallet_address.encode()) person_id = 'pi' + hs.hexdigest() return person_id + + +def get_last_block_hash(): + """Get last block hash from db""" + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + last_block_cursor = cur.execute( + 'SELECT block_index, hash, timestamp FROM blocks ORDER BY block_index DESC LIMIT 1' + ) + last_block = last_block_cursor.fetchone() + con.close() + + if last_block is not None: + return { + 'index': last_block[0], + 'hash': last_block[1], + 'timestamp': last_block[2] + } + else: + return None \ No newline at end of file diff --git a/app/codes/validator.py b/app/codes/validator.py index 866b31b..c6f5338 100644 --- a/app/codes/validator.py +++ b/app/codes/validator.py @@ -10,7 +10,7 @@ from app.codes.fs.mempool_manager import get_mempool_transaction from app.codes.p2p.transport import send -from .blockchain import get_last_block_hash +from .utils import get_last_block_hash from .transactionmanager import Transactionmanager from ..constants import IS_TEST, MEMPOOL_PATH from .p2p.outgoing import propogate_transaction_to_peers diff --git a/app/constants.py b/app/constants.py index 2873719..8e87b10 100644 --- a/app/constants.py +++ b/app/constants.py @@ -32,10 +32,12 @@ COMMITTEE_SIZE = 6 MINIMUM_ACCEPTANCE_VOTES = 4 MINIMUM_ACCEPTANCE_RATIO = 0.6 -NO_RECEIPT_COMMITTEE_TIMEOUT = 10 # Timeout in seconds NO_BLOCK_TIMEOUT = 5 # No block received timeout in seconds +NO_RECEIPT_COMMITTEE_TIMEOUT = 10 # Timeout in seconds +NETWORK_BLOCK_TIMEOUT = 25 # Variables +TIME_DIFF_WITH_GLOBAL_FILE = DATA_PATH + 'time_diff.txt' TIME_DIFF_WITH_GLOBAL = 0 MAX_ALLOWED_TIME_DIFF_SECONDS = 10 BLOCK_TIME_INTERVAL_SECONDS = TIME_BETWEEN_BLOCKS_SECONDS diff --git a/app/main.py b/app/main.py index 24e5f2c..e9f4cfc 100644 --- a/app/main.py +++ b/app/main.py @@ -10,8 +10,8 @@ from .constants import NEWRL_PORT from .codes.p2p.peers import init_bootstrap_nodes, update_my_address, update_software -from .codes.clock.global_time import update_time_difference -from .codes.clock.timers import start_mining_clock, start_miner_broadcast_clock +from .codes.clock.global_time import sync_timer_clock_with_global +from .codes.updater import start_miner_broadcast_clock from .routers import blockchain from .routers import p2p @@ -51,12 +51,12 @@ def app_startup(): try: if not args['disablenetwork']: + sync_timer_clock_with_global() if not args['disableupdate']: update_software(propogate=False) if not args['disablebootstrap']: init_bootstrap_nodes() sync_chain_from_peers() - update_time_difference() update_my_address() start_miner_broadcast_clock() except Exception as e: diff --git a/app/tests/test_timers.py b/app/tests/test_timers.py index e078b30..ccf0e71 100644 --- a/app/tests/test_timers.py +++ b/app/tests/test_timers.py @@ -2,8 +2,9 @@ import time import sqlite3 +from .test_p2p import _receive_block from .test_miner_committee import _add_test_miner -from ..codes.clock.timers import start_mining_clock +from ..codes.updater import start_mining_clock from ..codes.blockchain import get_last_block_hash from ..codes.utils import get_time_ms from ..codes.auth.auth import get_wallet @@ -20,7 +21,7 @@ client = TestClient(app) -def test_mining_reward(): +def test_mining_clock(): broadcast_miner_update() response = client.post('/run-updater') assert response.status_code == 200 @@ -39,7 +40,22 @@ def test_mining_reward(): assert block_index == previous_block_index + 1 - for i in range(1, 20): + for i in range(1, 3): _add_test_miner(i) # time.sleep(BLOCK_TIME_INTERVAL_SECONDS * 4) - # os._exit(1) \ No newline at end of file + # os._exit(1) + + +def test_all_timers(): + broadcast_miner_update() + response = client.post('/run-updater') + assert response.status_code == 200 + + broadcast_miner_update() + response = client.post('/run-updater') + + response = client.get('/get-last-block-index') + assert response.status_code == 200 + + previous_block_index = int(response.text) + _receive_block(previous_block_index + 1) \ No newline at end of file From 412b89461e8922be1a68adc5de99a20a3f3feae9 Mon Sep 17 00:00:00 2001 From: Kousthub Raja Date: Tue, 15 Mar 2022 14:43:11 +0530 Subject: [PATCH 8/8] review changes, fixes --- app/codes/blockchain.py | 6 +----- app/codes/chainscanner.py | 1 + app/codes/minermanager.py | 28 +++++++++++++++------------- app/codes/updater.py | 7 ------- app/main.py | 7 ++++++- app/tests/test_miner_committee.py | 7 ++----- app/tests/test_timers.py | 15 +++------------ 7 files changed, 28 insertions(+), 43 deletions(-) diff --git a/app/codes/blockchain.py b/app/codes/blockchain.py index e0615a1..824ae47 100644 --- a/app/codes/blockchain.py +++ b/app/codes/blockchain.py @@ -122,11 +122,7 @@ def mine_empty_block(self, cur, text): EMPTY_BLOCK_NONCE = 42 - try: - new_block_timestamp = int(last_block_timestamp) + 1 - except: - # For backward compatibility. Might need to refine. - new_block_timestamp = get_time_ms() + new_block_timestamp = int(last_block_timestamp) + 1 block = { 'index': last_block_index + 1, diff --git a/app/codes/chainscanner.py b/app/codes/chainscanner.py index cf9d844..bbabcc6 100644 --- a/app/codes/chainscanner.py +++ b/app/codes/chainscanner.py @@ -2,6 +2,7 @@ import sqlite3 from ..constants import NEWRL_DB +from .blockchain import Blockchain class Chainscanner(): diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py index de5afb4..c8841c1 100644 --- a/app/codes/minermanager.py +++ b/app/codes/minermanager.py @@ -69,18 +69,20 @@ def broadcast_miner_update(): validate(transaction) -def get_committee_list(): +def get_eligible_miners(): last_block = get_last_block_hash() - last_block_epoch = 0 - try: - # Need try catch to support older block timestamps - last_block_epoch = int(last_block['timestamp']) - except: - pass - if last_block: - cutfoff_epoch = last_block_epoch - TIME_MINER_BROADCAST_INTERVAL - else: - cutfoff_epoch = 0 + # last_block_epoch = 0 + # try: + # # Need try catch to support older block timestamps + # last_block_epoch = int(last_block['timestamp']) + # except: + # pass + # if last_block: + # cutfoff_epoch = last_block_epoch - TIME_MINER_BROADCAST_INTERVAL + # else: + # cutfoff_epoch = 0 + last_block_epoch = int(last_block['timestamp']) + cutfoff_epoch = last_block_epoch - TIME_MINER_BROADCAST_INTERVAL con = sqlite3.connect(NEWRL_DB) con.row_factory = sqlite3.Row @@ -103,7 +105,7 @@ def get_miner_for_current_block(): random.seed(last_block['index']) - committee_list = get_committee_list() + committee_list = get_committee_for_current_block() return random.choice(committee_list) @@ -118,7 +120,7 @@ def get_committee_for_current_block(): random.seed(last_block['index']) - miners = get_committee_list() + miners = get_eligible_miners() committee_size = min(COMMITTEE_SIZE, len(miners)) committee = random.sample(miners, k=committee_size) return committee diff --git a/app/codes/updater.py b/app/codes/updater.py index 58edd17..1bf7296 100644 --- a/app/codes/updater.py +++ b/app/codes/updater.py @@ -257,13 +257,6 @@ def start_miner_broadcast_clock(): timer.start() -def block_receive_timeout(): - miner = get_miner_for_current_block() - print(f"Block receive timed out from miner {miner['wallet_address']}") - block_index = mine_empty_block()['index'] - print(f"Mined new block {block_index}") - - def start_timers(block_timestamp): print('Starting timer with timestamp ', block_timestamp) diff --git a/app/main.py b/app/main.py index e9f4cfc..257499d 100644 --- a/app/main.py +++ b/app/main.py @@ -58,10 +58,15 @@ def app_startup(): init_bootstrap_nodes() sync_chain_from_peers() update_my_address() - start_miner_broadcast_clock() except Exception as e: print('Bootstrap failed') logging.critical(e, exc_info=True) + + try: + start_miner_broadcast_clock() + except Exception as e: + print('Miner broadcast failed') + logging.warning(e, exc_info=True) @app.on_event("shutdown") def shutdown_event(): diff --git a/app/tests/test_miner_committee.py b/app/tests/test_miner_committee.py index 8d20af9..e51844a 100644 --- a/app/tests/test_miner_committee.py +++ b/app/tests/test_miner_committee.py @@ -4,13 +4,10 @@ from ..codes.blockchain import get_last_block_hash from ..codes.utils import get_time_ms from ..codes.auth.auth import get_wallet -from ..codes.minermanager import broadcast_miner_update, get_committee_for_current_block, get_committee_list, get_miner_for_current_block, get_my_miner_status -from ..codes.db_updater import add_miner, update_wallet_token_balance +from ..codes.minermanager import broadcast_miner_update, get_committee_for_current_block, get_miner_for_current_block, get_my_miner_status +from ..codes.db_updater import add_miner from fastapi.testclient import TestClient -from ..ntypes import NUSD_TOKEN_CODE from ..constants import NEWRL_DB -from ..nvalues import TREASURY_WALLET_ADDRESS -from ..migrations.init import init_newrl from ..main import app diff --git a/app/tests/test_timers.py b/app/tests/test_timers.py index ccf0e71..6aa777e 100644 --- a/app/tests/test_timers.py +++ b/app/tests/test_timers.py @@ -1,20 +1,11 @@ -import os import time -import sqlite3 from .test_p2p import _receive_block from .test_miner_committee import _add_test_miner from ..codes.updater import start_mining_clock -from ..codes.blockchain import get_last_block_hash -from ..codes.utils import get_time_ms -from ..codes.auth.auth import get_wallet -from ..codes.minermanager import broadcast_miner_update, get_committee_for_current_block, get_committee_list, get_miner_for_current_block, get_my_miner_status -from ..codes.db_updater import add_miner, update_wallet_token_balance +from ..codes.minermanager import broadcast_miner_update from fastapi.testclient import TestClient -from ..ntypes import NUSD_TOKEN_CODE -from ..constants import BLOCK_TIME_INTERVAL_SECONDS, NEWRL_DB -from ..nvalues import TREASURY_WALLET_ADDRESS -from ..migrations.init import init_newrl +from ..constants import BLOCK_TIME_INTERVAL_SECONDS from ..main import app @@ -38,7 +29,7 @@ def test_mining_clock(): assert response.status_code == 200 block_index = int(response.text) - assert block_index == previous_block_index + 1 + assert block_index == previous_block_index + 2 for i in range(1, 3): _add_test_miner(i)