diff --git a/app/codes/auth/auth.py b/app/codes/auth/auth.py index fe16f93..53f6186 100644 --- a/app/codes/auth/auth.py +++ b/app/codes/auth/auth.py @@ -36,7 +36,6 @@ def get_auth(): 'public': wallet['public'], } auth_data['signature'] = sign_object(private_key, auth_data) - print('auth', auth_data) return auth_data except: auth_data = {} diff --git a/app/codes/blockchain.py b/app/codes/blockchain.py index be56df6..824ae47 100644 --- a/app/codes/blockchain.py +++ b/app/codes/blockchain.py @@ -109,6 +109,36 @@ def mine_block(self, cur, text, fees=0): block = self.create_block(cur, block, block_hash) return block + + def mine_empty_block(self, cur, text): + """Mine an empty block""" + print("Mining empty block") + last_block_cursor = cur.execute( + 'SELECT block_index, hash, timestamp FROM blocks ORDER BY block_index DESC LIMIT 1') + last_block = last_block_cursor.fetchone() + last_block_index = last_block[0] if last_block is not None else 0 + last_block_hash = last_block[1] if last_block is not None else 0 + last_block_timestamp = last_block[2] if last_block is not None else 0 + + EMPTY_BLOCK_NONCE = 42 + + new_block_timestamp = int(last_block_timestamp) + 1 + + block = { + 'index': last_block_index + 1, + 'timestamp': new_block_timestamp, + 'proof': EMPTY_BLOCK_NONCE, + 'text': text, + # 'creator_wallet': get_node_wallet_address(), + # 'fees': fees, + 'previous_hash': last_block_hash + } + + block_hash = self.calculate_hash(block) + print("New block hash is ", block_hash) + + block = self.create_block(cur, block, block_hash) + return block def get_latest_ts(self, cur=None): """Get the timestamp of latest block""" @@ -167,7 +197,7 @@ def get_last_block_hash(): con = sqlite3.connect(NEWRL_DB) cur = con.cursor() last_block_cursor = cur.execute( - 'SELECT block_index, hash FROM blocks ORDER BY block_index DESC LIMIT 1' + 'SELECT block_index, hash, timestamp FROM blocks ORDER BY block_index DESC LIMIT 1' ) last_block = last_block_cursor.fetchone() con.close() @@ -175,7 +205,8 @@ def get_last_block_hash(): if last_block is not None: return { 'index': last_block[0], - 'hash': last_block[1] + 'hash': last_block[1], + 'timestamp': last_block[2] } else: return None diff --git a/app/codes/clock/global_time.py b/app/codes/clock/global_time.py index 008afec..335bea2 100644 --- a/app/codes/clock/global_time.py +++ b/app/codes/clock/global_time.py @@ -2,7 +2,7 @@ import time import requests import threading -from ...constants import BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL +from ...constants import BLOCK_TIME_INTERVAL_SECONDS, MAX_ALLOWED_TIME_DIFF_SECONDS, NO_RECEIPT_COMMITTEE_TIMEOUT, TIME_DIFF_WITH_GLOBAL, TIME_DIFF_WITH_GLOBAL_FILE, TIME_MINER_BROADCAST_INTERVAL def get_global_epoch(): @@ -18,39 +18,27 @@ def get_local_epoch(): return epoch_time -def no_receipt_timeout(): - print('No receipts received. Timing out.') - - -def mine(): - print('Mining block.') - - -def start_receipt_timeout(): - timer = threading.Timer(NO_RECEIPT_COMMITTEE_TIMEOUT, no_receipt_timeout) - timer.start() - - -def start_mining_clock(): - mine() - timer = threading.Timer(BLOCK_TIME_INTERVAL_SECONDS, start_mining_clock) - timer.start() - - def get_time_difference(): """Return the time difference between local and global in seconds""" + try: + with open(TIME_DIFF_WITH_GLOBAL_FILE, 'r') as f: + return int(f.read()) + except: + global_epoch = get_global_epoch() + local_epoch = get_local_epoch() + diff = global_epoch - local_epoch + with open(TIME_DIFF_WITH_GLOBAL_FILE, 'w') as f: + f.write(str(diff)) + return diff + + +def sync_timer_clock_with_global(): global_epoch = get_global_epoch() local_epoch = get_local_epoch() - return global_epoch - local_epoch - - -def update_time_difference(): - TIME_DIFF_WITH_GLOBAL = get_time_difference() - print('Time difference with global is ', TIME_DIFF_WITH_GLOBAL) - if TIME_DIFF_WITH_GLOBAL > MAX_ALLOWED_TIME_DIFF_SECONDS: - print('System time is not syncronised. Time difference in seconds: ', TIME_DIFF_WITH_GLOBAL) - quit() - return True + diff = global_epoch - local_epoch + with open(TIME_DIFF_WITH_GLOBAL_FILE, 'w') as f: + f.write(str(diff)) + print('Synced clock. Time difference is', diff) if __name__ == '__main__': diff --git a/app/codes/consensus/consensus.py b/app/codes/consensus/consensus.py index ba896fc..6ceb507 100644 --- a/app/codes/consensus/consensus.py +++ b/app/codes/consensus/consensus.py @@ -6,6 +6,7 @@ from ..fs.mempool_manager import append_receipt_to_block, get_receipts_from_storage from ...constants import MINIMUM_ACCEPTANCE_RATIO, MINIMUM_ACCEPTANCE_VOTES from ..auth.auth import get_wallet +from ..minermanager import get_miner_for_current_block try: @@ -64,4 +65,13 @@ def check_community_consensus(block): # if receipt_counts['postitive_receipt_count'] >= MINIMUM_ACCEPTANCE_VOTES and receipt_counts['total_receipt_count']: # return True - return False \ No newline at end of file + return False + + +def validate_block_miner(block): + miner_address = block['signature']['address'] + + expected_miner = get_miner_for_current_block()['wallet_address'] + + if miner_address != expected_miner: + raise Exception(f"Invalid miner {miner_address} for block {block['block_index']}. Expected {expected_miner}") \ No newline at end of file diff --git a/app/codes/db_updater.py b/app/codes/db_updater.py index c494f2c..71d6869 100644 --- a/app/codes/db_updater.py +++ b/app/codes/db_updater.py @@ -269,3 +269,10 @@ def input_to_dict(ipval): else: callparams = ipval return callparams + + +def add_miner(cur, wallet_address, network_address, broadcast_timestamp): + cur.execute('''INSERT OR REPLACE INTO miners + (id, wallet_address, network_address, last_broadcast_timestamp) + VALUES (?, ?, ?, ?)''', + (wallet_address, wallet_address, network_address, broadcast_timestamp)) diff --git a/app/codes/kycwallet.py b/app/codes/kycwallet.py index 86540da..6b8df8a 100644 --- a/app/codes/kycwallet.py +++ b/app/codes/kycwallet.py @@ -8,6 +8,7 @@ import base64 import sqlite3 +from .utils import get_time_ms from ..constants import TMP_PATH, NEWRL_DB from .transactionmanager import Transactionmanager @@ -113,7 +114,7 @@ def generate_wallet(kyccustodian, kycdocs, ownertype, jurisd, wallet_specific_da def create_add_wallet_transaction(wallet): transaction_data = { - 'timestamp': str(datetime.datetime.now()), + 'timestamp': get_time_ms(), 'type': 1, 'currency': 'INR', 'fee': 0.0, diff --git a/app/codes/minermanager.py b/app/codes/minermanager.py new file mode 100644 index 0000000..c8841c1 --- /dev/null +++ b/app/codes/minermanager.py @@ -0,0 +1,144 @@ +"""Miner update functions""" +import sqlite3 +import random + +from .utils import get_last_block_hash +# from .p2p.outgoing import propogate_transaction_to_peers +from .p2p.utils import get_my_address +from ..constants import COMMITTEE_SIZE, IS_TEST, NEWRL_DB, TIME_MINER_BROADCAST_INTERVAL +from .auth.auth import get_wallet +from .signmanager import sign_transaction +from ..ntypes import TRANSACTION_MINER_ADDITION +from .utils import get_time_ms +from .transactionmanager import Transactionmanager +from .validator import validate + + +def miner_addition_transaction(wallet=None, my_address=None): + if wallet is None: + wallet = get_wallet() + if my_address is None: + my_address = get_my_address() + timestamp = get_time_ms() + transaction_data = { + 'timestamp': timestamp, + 'type': TRANSACTION_MINER_ADDITION, + 'currency': "NWRL", + 'fee': 0.0, + 'descr': "Miner addition", + 'valid': 1, + 'block_index': 0, + 'specific_data': { + 'wallet_address': wallet['address'], + 'network_address': my_address, + 'broadcast_timestamp': timestamp + } + } + + transaction_manager = Transactionmanager() + transaction_data = {'transaction': transaction_data, 'signatures': []} + transaction_manager.transactioncreator(transaction_data) + transaction = transaction_manager.get_transaction_complete() + signed_transaction = sign_transaction(wallet, transaction) + return signed_transaction + + +def get_miner_status(wallet_address): + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + miner_cursor = cur.execute( + 'SELECT wallet_address, network_address, last_broadcast_timestamp FROM miners WHERE wallet_address=?', (wallet_address, )).fetchone() + if miner_cursor is None: + return None + miner_info = { + 'wallet_address': miner_cursor[0], + 'network_address': miner_cursor[1], + 'broadcast_timestamp': miner_cursor[2] + } + return miner_info + + +def get_my_miner_status(): + wallet = get_wallet() + my_status = get_miner_status(wallet['address']) + return my_status + + +def broadcast_miner_update(): + transaction = miner_addition_transaction() + validate(transaction) + + +def get_eligible_miners(): + last_block = get_last_block_hash() + # last_block_epoch = 0 + # try: + # # Need try catch to support older block timestamps + # last_block_epoch = int(last_block['timestamp']) + # except: + # pass + # if last_block: + # cutfoff_epoch = last_block_epoch - TIME_MINER_BROADCAST_INTERVAL + # else: + # cutfoff_epoch = 0 + last_block_epoch = int(last_block['timestamp']) + cutfoff_epoch = last_block_epoch - TIME_MINER_BROADCAST_INTERVAL + + con = sqlite3.connect(NEWRL_DB) + con.row_factory = sqlite3.Row + cur = con.cursor() + miner_cursor = cur.execute( + '''SELECT wallet_address, network_address, last_broadcast_timestamp + FROM miners + WHERE last_broadcast_timestamp > ? + ORDER BY wallet_address ASC''', (cutfoff_epoch, )).fetchall() + miners = [dict(m) for m in miner_cursor] + con.close() + return miners + + +def get_miner_for_current_block(): + last_block = get_last_block_hash() + + if not last_block: + return + + random.seed(last_block['index']) + + committee_list = get_committee_for_current_block() + + return random.choice(committee_list) + + # return committee_list[0] + + +def get_committee_for_current_block(): + last_block = get_last_block_hash() + + if not last_block: + return + + random.seed(last_block['index']) + + miners = get_eligible_miners() + committee_size = min(COMMITTEE_SIZE, len(miners)) + committee = random.sample(miners, k=committee_size) + return committee + + +def should_i_mine(): + my_wallet = get_wallet() + miner = get_miner_for_current_block() + if miner['wallet_address'] == my_wallet['address']: + return True + return False + + +def am_i_in_current_committee(): + my_wallet_address = get_wallet()['address'] + committee = get_committee_for_current_block() + + found = list(filter(lambda w: w['wallet_address'] == my_wallet_address, committee)) + if len(found) == 0: + return False + return True \ No newline at end of file diff --git a/app/codes/p2p/outgoing.py b/app/codes/p2p/outgoing.py index a020baa..75d0cd4 100644 --- a/app/codes/p2p/outgoing.py +++ b/app/codes/p2p/outgoing.py @@ -1,11 +1,13 @@ import requests from threading import Thread -from ...constants import NEWRL_PORT, REQUEST_TIMEOUT, TRANSPORT_SERVER +from ...constants import IS_TEST, NEWRL_PORT, REQUEST_TIMEOUT, TRANSPORT_SERVER from ..p2p.utils import get_peers from ..p2p.utils import is_my_address def propogate_transaction_to_peers(transaction): + if IS_TEST: + return peers = get_peers() for peer in peers: @@ -25,6 +27,8 @@ def send_request_in_thread(url, data): thread.start() def send_request(url, data): + if IS_TEST: + return requests.post(url, json=data, timeout=REQUEST_TIMEOUT) def send(payload): diff --git a/app/codes/p2p/sync_chain.py b/app/codes/p2p/sync_chain.py index b18edc6..996ad3f 100644 --- a/app/codes/p2p/sync_chain.py +++ b/app/codes/p2p/sync_chain.py @@ -10,7 +10,7 @@ from app.codes.validator import validate_block, validate_block_data, validate_receipt_signature from app.codes.updater import broadcast_block from app.codes.fs.temp_manager import append_receipt_to_block, append_receipt_to_block_in_storage, get_blocks_for_index_from_storage, store_block_to_temp, store_receipt_to_temp -from app.codes.consensus.consensus import check_community_consensus +from app.codes.consensus.consensus import check_community_consensus, validate_block_miner logging.basicConfig(level=logging.INFO) @@ -40,6 +40,8 @@ def receive_block(block): if block_index > get_last_block_index() + 1: sync_chain_from_peers() + validate_block_miner(block) + validate_block(block, validate_receipts=False) # if check_community_consensus(block): diff --git a/app/codes/p2p/utils.py b/app/codes/p2p/utils.py index 3fb6201..108e163 100644 --- a/app/codes/p2p/utils.py +++ b/app/codes/p2p/utils.py @@ -1,6 +1,7 @@ import sqlite3 import requests import socket + from ...constants import NEWRL_P2P_DB @@ -22,4 +23,4 @@ def is_my_address(address): my_address = get_my_address() if socket.gethostbyname(address) == my_address: return True - return False \ No newline at end of file + return False diff --git a/app/codes/state_updater.py b/app/codes/state_updater.py index 4527a17..7545c4b 100644 --- a/app/codes/state_updater.py +++ b/app/codes/state_updater.py @@ -6,7 +6,7 @@ from ..constants import NEWRL_DB from .db_updater import * -from ..ntypes import NEWRL_TOKEN_CODE, NEWRL_TOKEN_NAME +from ..ntypes import NEWRL_TOKEN_CODE, NEWRL_TOKEN_NAME, TRANSACTION_MINER_ADDITION, TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TOKEN_CREATION, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION def update_db_states(cur, block): @@ -42,13 +42,13 @@ def update_db_states(cur, block): def update_state_from_transaction(cur, transaction_type, transaction_data, transaction_code, transaction_timestamp): - if transaction_type == 1: # this is a wallet creation transaction + if transaction_type == TRANSACTION_WALLET_CREATION: # this is a wallet creation transaction add_wallet_pid(cur, transaction_data) - if transaction_type == 2: # this is a token creation or addition transaction + if transaction_type == TRANSACTION_TOKEN_CREATION: # this is a token creation or addition transaction add_token(cur, transaction_data, transaction_code) - if transaction_type == 4 or transaction_type == 5: # this is a transfer tx + if transaction_type == TRANSACTION_TWO_WAY_TRANSFER or transaction_type == TRANSACTION_ONE_WAY_TRANSFER: # this is a transfer tx sender1 = transaction_data['wallet1'] sender2 = transaction_data['wallet2'] @@ -62,14 +62,14 @@ def update_state_from_transaction(cur, transaction_type, transaction_data, trans transfer_tokens_and_update_balances( cur, sender2, sender1, tokencode2, amount2) - if transaction_type == 6: # score update transaction + if transaction_type == TRANSACTION_TRUST_SCORE_CHANGE: # score update transaction personid1 = get_pid_from_wallet(cur, transaction_data['address1']) personid2 = get_pid_from_wallet(cur, transaction_data['address2']) new_score = transaction_data['new_score'] tstamp = transaction_timestamp update_trust_score(cur, personid1, personid2, new_score, tstamp) - if transaction_type == 3: # smart contract transaction + if transaction_type == TRANSACTION_SMART_CONTRACT: # smart contract transaction funct = transaction_data['function'] if funct == "setup": # sc is being set up contract = dict(transaction_data['params']) @@ -84,6 +84,14 @@ def update_state_from_transaction(cur, transaction_type, transaction_data, trans # sc_instance = nusd1(transaction['specific_data']['address']) funct = getattr(sc_instance, funct) funct(cur, transaction_data['params']) + + if transaction_type == TRANSACTION_MINER_ADDITION: + add_miner( + cur, + transaction_data['wallet_address'], + transaction_data['network_address'], + transaction_data['broadcast_timestamp'], + ) def add_block_reward(cur, creator, blockindex): diff --git a/app/codes/tokenmanager.py b/app/codes/tokenmanager.py index d33419e..422f927 100644 --- a/app/codes/tokenmanager.py +++ b/app/codes/tokenmanager.py @@ -1,13 +1,12 @@ # Python programm to create object that enables addition of a block -import datetime - +from .utils import get_time_ms from .transactionmanager import Transactionmanager def create_token_transaction(token_data): transaction = { - 'timestamp': str(datetime.datetime.now()), + 'timestamp': get_time_ms(), 'type': 2, 'currency': "INR", 'fee': 0.0, diff --git a/app/codes/transactionmanager.py b/app/codes/transactionmanager.py index 01063f6..3e7a11a 100644 --- a/app/codes/transactionmanager.py +++ b/app/codes/transactionmanager.py @@ -8,7 +8,9 @@ import base64 import sqlite3 -from ..ntypes import TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION, TRANSCATION_TOKEN_CREATION + +from ..ntypes import TRANSACTION_MINER_ADDITION, TRANSACTION_ONE_WAY_TRANSFER, TRANSACTION_SMART_CONTRACT, TRANSACTION_TRUST_SCORE_CHANGE, TRANSACTION_TWO_WAY_TRANSFER, TRANSACTION_WALLET_CREATION, TRANSACTION_TOKEN_CREATION + from .chainscanner import get_wallet_token_balance from ..constants import ALLOWED_CUSTODIANS_FILE, MEMPOOL_PATH, NEWRL_DB from .utils import get_time_ms @@ -199,7 +201,7 @@ def econvalidator(self): # from mempool only include transactions that reduce balance and not those that increase # check if the sender has enough balance to spend self.validity = 0 - if self.transaction['type'] == 1: + if self.transaction['type'] == TRANSACTION_WALLET_CREATION: custodian = self.transaction['specific_data']['custodian_wallet'] walletaddress = self.transaction['specific_data']['wallet_address'] if not is_wallet_valid(custodian): @@ -244,7 +246,7 @@ def econvalidator(self): self.validity = 0 # self.validity=0 - if self.transaction['type'] == 2: # token addition transaction + if self.transaction['type'] == TRANSACTION_TOKEN_CREATION: # token addition transaction firstowner = self.transaction['specific_data']['first_owner'] custodian = self.transaction['specific_data']['custodian'] fovalidity = False @@ -298,7 +300,7 @@ def econvalidator(self): "Tokencode provided does not exist. Will append as new one.") self.validity = 1 # tokencode is provided by user - if self.transaction['type'] == 3: + if self.transaction['type'] == TRANSACTION_SMART_CONTRACT: self.validity = 1 for wallet in self.transaction['specific_data']['signers']: if not is_wallet_valid(wallet): @@ -309,7 +311,7 @@ def econvalidator(self): self.validity = 0 # self.validity=0 - if self.transaction['type'] == 4 or self.transaction['type'] == 5: + if self.transaction['type'] == TRANSACTION_TWO_WAY_TRANSFER or self.transaction['type'] == TRANSACTION_ONE_WAY_TRANSFER: ttype = self.transaction['type'] startingbalance1 = 0 startingbalance2 = 0 @@ -400,7 +402,7 @@ def econvalidator(self): # self.transaction['valid']=1; self.validity = 1 - if self.transaction['type'] == 6: # score change transaction + if self.transaction['type'] == TRANSACTION_TRUST_SCORE_CHANGE: # score change transaction ttype = self.transaction['type'] # personid1 = self.transaction['specific_data']['personid1'] # personid2 = self.transaction['specific_data']['personid2'] @@ -426,6 +428,14 @@ def econvalidator(self): self.validity = 0 else: self.validity = 1 + + if self.transaction['type'] == TRANSACTION_MINER_ADDITION: + # No checks for fee in the beginning + if not is_wallet_valid(self.transaction['specific_data']['wallet_address']): + print("Miner wallet not in chain") + self.validity = 0 + else: + self.validity = 1 if self.validity == 1: return True @@ -540,7 +550,7 @@ def get_valid_addresses(transaction): if transaction_type == TRANSACTION_WALLET_CREATION: # Custodian needs to sign valid_addresses.append( transaction['specific_data']['custodian_wallet']) - if transaction_type == TRANSCATION_TOKEN_CREATION: # Custodian needs to sign + if transaction_type == TRANSACTION_TOKEN_CREATION: # Custodian needs to sign valid_addresses.append(transaction['specific_data']['custodian']) if transaction_type == TRANSACTION_SMART_CONTRACT: valid_addresses = get_sc_validadds(transaction) @@ -552,4 +562,6 @@ def get_valid_addresses(transaction): if transaction_type == TRANSACTION_TRUST_SCORE_CHANGE: # Only address1 is added, not address2 valid_addresses.append(transaction['specific_data']['address1']) + if transaction_type == TRANSACTION_MINER_ADDITION: + valid_addresses.append(transaction['specific_data']['wallet_address']) return valid_addresses diff --git a/app/codes/updater.py b/app/codes/updater.py index 959b49b..1bf7296 100644 --- a/app/codes/updater.py +++ b/app/codes/updater.py @@ -3,10 +3,13 @@ import json import os import sqlite3 +import threading import requests +from .clock.global_time import get_time_difference +from .minermanager import am_i_in_current_committee, broadcast_miner_update, get_miner_for_current_block, should_i_mine from ..nvalues import TREASURY_WALLET_ADDRESS -from ..constants import ALLOWED_FEE_PAYMENT_TOKENS, IS_TEST, NEWRL_DB, NEWRL_PORT, REQUEST_TIMEOUT, MEMPOOL_PATH, TIME_BETWEEN_BLOCKS_SECONDS +from ..constants import ALLOWED_FEE_PAYMENT_TOKENS, BLOCK_RECEIVE_TIMEOUT_SECONDS, BLOCK_TIME_INTERVAL_SECONDS, IS_TEST, NEWRL_DB, NEWRL_PORT, NO_RECEIPT_COMMITTEE_TIMEOUT, REQUEST_TIMEOUT, MEMPOOL_PATH, TIME_BETWEEN_BLOCKS_SECONDS, TIME_MINER_BROADCAST_INTERVAL from .p2p.peers import get_peers from .p2p.utils import is_my_address from .utils import BufferedLog, get_time_ms @@ -18,6 +21,7 @@ from .chainscanner import get_wallet_token_balance from .db_updater import transfer_tokens_and_update_balances from .p2p.outgoing import send_request_in_thread +from .auth.auth import get_wallet MAX_BLOCK_SIZE = 10 @@ -131,9 +135,12 @@ def run_updater(): def broadcast_block(block): peers = get_peers() - private_key = _private - public_key = _public + my_wallet = get_wallet() + private_key = my_wallet['private'] + public_key = my_wallet['public'] + address = my_wallet['address'] signature = { + 'address': address, 'public': public_key, 'msgsign': sign_object(private_key, block) } @@ -190,3 +197,82 @@ def pay_fee_for_transaction(cur, transaction): fee / len(payees) ) return True + + +def mine_empty_block(): + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + + blockchain = Blockchain() + + block = blockchain.mine_empty_block(cur, {'transactions': []}) + # update_db_states(cur, block) + + con.commit() + con.close() + + return block + + +def no_receipt_timeout(): + print('Inadequate receipts. Timing out and sending empty block.') + + +def mine(): + if should_i_mine(): + print('I am the miner for this block.') + run_updater() + else: + miner = get_miner_for_current_block() + print(f"Miner for current block is {miner['wallet_address']}. Waiting to receive block.") + start_block_receive_timeout_clock() + +def start_receipt_timeout(): + timer = threading.Timer(NO_RECEIPT_COMMITTEE_TIMEOUT, no_receipt_timeout) + timer.start() + + +def start_mining_clock(): + mine() + timer = threading.Timer(BLOCK_TIME_INTERVAL_SECONDS, start_mining_clock) + timer.start() + + +def block_receive_timeout(): + miner = get_miner_for_current_block() + print(f"Block receive timed out from miner {miner['wallet_address']}") + block_index = mine_empty_block()['index'] + print(f"Mined new block {block_index}") + + +def start_block_receive_timeout_clock(): + timer = threading.Timer(BLOCK_RECEIVE_TIMEOUT_SECONDS, block_receive_timeout) + timer.start() + + +def start_miner_broadcast_clock(): + print('Broadcasting miner update') + broadcast_miner_update() + timer = threading.Timer(TIME_MINER_BROADCAST_INTERVAL, start_miner_broadcast_clock) + timer.start() + + +def start_timers(block_timestamp): + print('Starting timer with timestamp ', block_timestamp) + + my_global_timestamp = get_time_ms() - get_time_difference() + propogation_delay = my_global_timestamp - block_timestamp + + # If I'm miner, start mining clock + if should_i_mine(): + wait_time = BLOCK_TIME_INTERVAL_SECONDS - propogation_delay + timer = threading.Timer(wait_time, mine) + timer.start() + elif am_i_in_current_committee(): + wait_time = NO_RECEIPT_COMMITTEE_TIMEOUT - propogation_delay + timer = threading.Timer(wait_time, block_receive_timeout) + timer.start() + else: + pass + # Not a miner and part of committee. No action to be performed + # Hoping the sentinel node will trigger an empty block start stagnant network \ No newline at end of file diff --git a/app/codes/utils.py b/app/codes/utils.py index 106063c..9e38fe3 100644 --- a/app/codes/utils.py +++ b/app/codes/utils.py @@ -1,6 +1,9 @@ import hashlib +import sqlite3 import time +from ..constants import NEWRL_DB + def save_file_and_get_path(upload_file): if upload_file is None: @@ -33,3 +36,23 @@ def get_person_id_for_wallet_address(wallet_address): hs.update(wallet_address.encode()) person_id = 'pi' + hs.hexdigest() return person_id + + +def get_last_block_hash(): + """Get last block hash from db""" + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + last_block_cursor = cur.execute( + 'SELECT block_index, hash, timestamp FROM blocks ORDER BY block_index DESC LIMIT 1' + ) + last_block = last_block_cursor.fetchone() + con.close() + + if last_block is not None: + return { + 'index': last_block[0], + 'hash': last_block[1], + 'timestamp': last_block[2] + } + else: + return None \ No newline at end of file diff --git a/app/codes/validator.py b/app/codes/validator.py index 5aacd8d..c6f5338 100644 --- a/app/codes/validator.py +++ b/app/codes/validator.py @@ -10,9 +10,9 @@ from app.codes.fs.mempool_manager import get_mempool_transaction from app.codes.p2p.transport import send -from .blockchain import get_last_block_hash +from .utils import get_last_block_hash from .transactionmanager import Transactionmanager -from ..constants import MEMPOOL_PATH +from ..constants import IS_TEST, MEMPOOL_PATH from .p2p.outgoing import propogate_transaction_to_peers @@ -31,32 +31,33 @@ def validate(transaction): signatures_valid = transaction_manager.verifytransigns() valid = False if economics_valid and signatures_valid: - msg = "All well" + msg = "Transaction is valid" valid = True if not economics_valid: - msg = "Economic validation failed" + msg = "Transaction economic validation failed" if not signatures_valid: - msg = "Invalid signatures" + msg = "Transaction has invalid signatures" check = {'valid': valid, 'msg': msg} if valid: # Economics and signatures are both valid transaction_file = f"{MEMPOOL_PATH}transaction-{transaction_manager.transaction['type']}-{transaction_manager.transaction['trans_code']}.json" transaction_manager.save_transaction_to_mempool(transaction_file) - # Broadcast transaction to peers - propogate_transaction_to_peers(transaction_manager.get_transaction_complete()) - - # Broadcaset transaction via transport server - try: - payload = { - 'operation': 'send_transaction', - 'data': transaction_manager.get_transaction_complete() - } - send(payload) - except: - print('Error sending transaction to transport server') - - print(check) + if not IS_TEST: + # Broadcast transaction to peers via HTTP + propogate_transaction_to_peers(transaction_manager.get_transaction_complete()) + + # Broadcaset transaction via transport server + try: + payload = { + 'operation': 'send_transaction', + 'data': transaction_manager.get_transaction_complete() + } + send(payload) + except: + print('Error sending transaction to transport server') + + print(msg) return check diff --git a/app/constants.py b/app/constants.py index 0f04fc2..8e87b10 100644 --- a/app/constants.py +++ b/app/constants.py @@ -22,7 +22,7 @@ BOOTSTRAP_NODES = ['testnet.newrl.net'] REQUEST_TIMEOUT = 1 -NEWRL_PORT = 8090 +NEWRL_PORT = 8182 # Devnet NEWRL_TOKEN = "newrl_token" TREASURY = "treasury_address" COINBASE_SC = "coinbase_sc_address" @@ -32,13 +32,17 @@ COMMITTEE_SIZE = 6 MINIMUM_ACCEPTANCE_VOTES = 4 MINIMUM_ACCEPTANCE_RATIO = 0.6 -NO_RECEIPT_COMMITTEE_TIMEOUT = 10 # Timeout in seconds NO_BLOCK_TIMEOUT = 5 # No block received timeout in seconds +NO_RECEIPT_COMMITTEE_TIMEOUT = 10 # Timeout in seconds +NETWORK_BLOCK_TIMEOUT = 25 # Variables +TIME_DIFF_WITH_GLOBAL_FILE = DATA_PATH + 'time_diff.txt' TIME_DIFF_WITH_GLOBAL = 0 MAX_ALLOWED_TIME_DIFF_SECONDS = 10 -BLOCK_TIME_INTERVAL_SECONDS = 30 +BLOCK_TIME_INTERVAL_SECONDS = TIME_BETWEEN_BLOCKS_SECONDS +BLOCK_RECEIVE_TIMEOUT_SECONDS = 3 +TIME_MINER_BROADCAST_INTERVAL = 60 * 60 * 1000 # Miner broadcast every hour MY_ADDRESS = '' ALLOWED_FEE_PAYMENT_TOKENS = [NEWRL_TOKEN_CODE, NUSD_TOKEN_CODE] \ No newline at end of file diff --git a/app/main.py b/app/main.py index 342b30e..257499d 100644 --- a/app/main.py +++ b/app/main.py @@ -1,5 +1,6 @@ import logging import argparse +import os import uvicorn from fastapi.openapi.utils import get_openapi from fastapi import FastAPI @@ -9,7 +10,8 @@ from .constants import NEWRL_PORT from .codes.p2p.peers import init_bootstrap_nodes, update_my_address, update_software -from .codes.clock.global_time import start_mining_clock, update_time_difference +from .codes.clock.global_time import sync_timer_clock_with_global +from .codes.updater import start_miner_broadcast_clock from .routers import blockchain from .routers import p2p @@ -19,11 +21,6 @@ logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) -parser = argparse.ArgumentParser() -parser.add_argument("--disablenetwork", help="run the node local only with no network connection", action="store_true") -parser.add_argument("--disableupdate", help="run the node without updating software", action="store_true") -parser.add_argument("--disablebootstrap", help="run the node without bootstrapping", action="store_true") -args = parser.parse_args() app = FastAPI( title="The Newrl APIs", @@ -44,23 +41,45 @@ app.include_router(p2p.router) app.include_router(transport.router) +args = { + 'disablenetwork': False, + 'disableupdate': False, + 'disablebootstrap': False, +} + @app.on_event('startup') def app_startup(): try: - if not args.disablenetwork: - if not args.disableupdate: + if not args['disablenetwork']: + sync_timer_clock_with_global() + if not args['disableupdate']: update_software(propogate=False) - if not args.disablebootstrap: + if not args['disablebootstrap']: init_bootstrap_nodes() sync_chain_from_peers() - update_time_difference() update_my_address() - # start_mining_clock() except Exception as e: print('Bootstrap failed') logging.critical(e, exc_info=True) + + try: + start_miner_broadcast_clock() + except Exception as e: + print('Miner broadcast failed') + logging.warning(e, exc_info=True) + +@app.on_event("shutdown") +def shutdown_event(): + print('Shutting down node') + os._exit(0) if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--disablenetwork", help="run the node local only with no network connection", action="store_true") + parser.add_argument("--disableupdate", help="run the node without updating software", action="store_true") + parser.add_argument("--disablebootstrap", help="run the node without bootstrapping", action="store_true") + _args = parser.parse_args() + args["disablenetwork"] = _args.disablenetwork uvicorn.run("app.main:app", host="0.0.0.0", port=NEWRL_PORT, reload=True) diff --git a/app/migrations/init_db.py b/app/migrations/init_db.py index 27f9795..15da8b2 100644 --- a/app/migrations/init_db.py +++ b/app/migrations/init_db.py @@ -16,6 +16,7 @@ def clear_db(): cur.execute('DROP TABLE IF EXISTS transactions') cur.execute('DROP TABLE IF EXISTS transfers') cur.execute('DROP TABLE IF EXISTS contracts') + cur.execute('DROP TABLE IF EXISTS miners') con.commit() con.close() @@ -111,7 +112,16 @@ def init_db(): contractspecs TEXT, legalparams TEXT) ''') - + cur.execute('DROP TABLE IF EXISTS miners') + cur.execute(''' + CREATE TABLE IF NOT EXISTS miners + (id text NOT NULL PRIMARY KEY, + wallet_address text, + network_address text NOT NULL, + last_broadcast_timestamp text, + UNIQUE (wallet_address) + ) + ''') con.commit() con.close() diff --git a/app/migrations/migrations/3_init_newrl_tokens.py b/app/migrations/migrations/3_init_newrl_tokens.py index 86e53f0..da3ce12 100644 --- a/app/migrations/migrations/3_init_newrl_tokens.py +++ b/app/migrations/migrations/3_init_newrl_tokens.py @@ -20,6 +20,8 @@ def init_newrl_tokens(): cur = con.cursor() create_newrl_tokens(cur, FOUNDATION_RESERVE * 2) + create_wallet(cur, FOUNDATION_WALLET, FOUNDATION_PUBLIC_KEY) + create_wallet(cur, ASQI_WALLET, ASQI_PUBLIC_KEY) credit_wallet(cur, FOUNDATION_WALLET, FOUNDATION_RESERVE) credit_wallet(cur, ASQI_WALLET, FOUNDATION_RESERVE) @@ -41,7 +43,16 @@ def create_newrl_tokens(cur, amount): (tokencode, tokenname, tokentype, amount_created, sc_flag, tokendecimal, token_attributes) VALUES (?, ?, ?, ?, ?, ?, ?)''', query_params) - + +def create_wallet(cur, wallet_address, wallet_public): + query_params = (wallet_address, + wallet_public, + '', '{}', '1', '91', '{}' + ) + cur.execute(f'''INSERT OR IGNORE INTO wallets + (wallet_address, wallet_public, custodian_wallet, kyc_docs, + owner_type, jurisdiction, specific_data) + VALUES (?, ?, ?, ?, ?, ?, ?)''', query_params) def credit_wallet(cur, wallet, amount): cur.execute(f'''INSERT OR IGNORE INTO balances diff --git a/app/ntypes.py b/app/ntypes.py index 6870c77..d60a913 100644 --- a/app/ntypes.py +++ b/app/ntypes.py @@ -1,12 +1,13 @@ """Type declarations""" TRANSACTION_WALLET_CREATION = 1 -TRANSCATION_TOKEN_CREATION = 2 +TRANSACTION_TOKEN_CREATION = 2 TRANSACTION_SMART_CONTRACT = 3 TRANSACTION_TWO_WAY_TRANSFER = 4 TRANSACTION_ONE_WAY_TRANSFER = 5 TRANSACTION_TRUST_SCORE_CHANGE = 6 +TRANSACTION_MINER_ADDITION = 7 NEWRL_TOKEN_CODE = 'NWRL' NUSD_TOKEN_CODE = 'NUSD' -NEWRL_TOKEN_NAME = 'Newrl' \ No newline at end of file +NEWRL_TOKEN_NAME = 'Newrl' diff --git a/app/tests/conftest.py b/app/tests/conftest.py index 4bbd771..afc9671 100644 --- a/app/tests/conftest.py +++ b/app/tests/conftest.py @@ -1,6 +1,11 @@ import os import shutil + +from ..migrations.init import init_newrl +from ..codes.p2p.peers import init_peer_db +from ..migrations.migrate_db import run_migrations from ..migrations.init_db import init_peer_db + import pytest @@ -17,7 +22,9 @@ def setup_test_files(): if os.path.exists('data_test/.auth.json'): os.remove('data_test/.auth.json') shutil.copyfile('data_test/template/.auth.json', 'data_test/.auth.json') + init_newrl() init_peer_db() + run_migrations() os.environ['NEWRL_TEST'] = '1' diff --git a/app/tests/test_fee_rewards.py b/app/tests/test_fee_rewards.py index 8d47e59..99dd628 100644 --- a/app/tests/test_fee_rewards.py +++ b/app/tests/test_fee_rewards.py @@ -32,21 +32,21 @@ def test_mining_reward(): wallet_address = wallet['wallet_address'] assert wallet_address - check_newrl_wallet_balance(wallet_address, None) + check_newrl_wallet_balance(wallet_address, 1500000000.0) response = client.post('/run-updater') assert response.status_code == 200 - check_newrl_wallet_balance(wallet_address, 1000) + check_newrl_wallet_balance(wallet_address, 1500001000.0) time.sleep(2) response = client.post('/run-updater') assert response.status_code == 200 - check_newrl_wallet_balance(wallet_address, 1000) + check_newrl_wallet_balance(wallet_address, 1500001000.0) time.sleep(5) response = client.post('/run-updater') assert response.status_code == 200 - check_newrl_wallet_balance(wallet_address, 2000) + check_newrl_wallet_balance(wallet_address, 1500002000.0) def test_transaction_fee_payment(): diff --git a/app/tests/test_miner_committee.py b/app/tests/test_miner_committee.py new file mode 100644 index 0000000..e51844a --- /dev/null +++ b/app/tests/test_miner_committee.py @@ -0,0 +1,67 @@ +import time +import sqlite3 + +from ..codes.blockchain import get_last_block_hash +from ..codes.utils import get_time_ms +from ..codes.auth.auth import get_wallet +from ..codes.minermanager import broadcast_miner_update, get_committee_for_current_block, get_miner_for_current_block, get_my_miner_status +from ..codes.db_updater import add_miner +from fastapi.testclient import TestClient +from ..constants import NEWRL_DB + +from ..main import app + +client = TestClient(app) + + +def test_mining_reward(): + assert None == get_my_miner_status() + + broadcast_miner_update() + response = client.post('/run-updater') + assert response.status_code == 200 + + miner_info = get_my_miner_status() + assert miner_info['wallet_address'] == get_wallet()['address'] + + +def test_miner_selection(): + for i in range(0,20): + _add_test_miner(i) + + last_block1 = get_last_block_hash() + committee = get_committee_for_current_block() + assert len(committee) == 6 + + miner1 = get_miner_for_current_block() + # Check if pseudo-random with block index as seed returns the same miner + assert miner1['wallet_address'] == get_miner_for_current_block()['wallet_address'] + # my_wallet = get_wallet() + # assert miner1['wallet_address'] == my_wallet['address'] + + time.sleep(5) + response = client.post('/run-updater') + assert response.status_code == 200 + + last_block2 = get_last_block_hash() + + assert last_block2['index'] == last_block1['index'] + 1 + miner2 = get_miner_for_current_block() + + # Hoping the miner changes for the new block. Totally random though + assert miner1['wallet_address'] != miner2['wallet_address'] + + +def _add_test_miner(i): + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + add_miner(cur, f'0x0000{i}', '127.0.0.1', get_time_ms()) + con.commit() + con.close() + +def clear_miner_db(): + con = sqlite3.connect(NEWRL_DB) + cur = con.cursor() + cur.execute('delete from miners') + con.commit() + con.close() diff --git a/app/tests/test_p2p.py b/app/tests/test_p2p.py index 073e302..260b188 100644 --- a/app/tests/test_p2p.py +++ b/app/tests/test_p2p.py @@ -1,5 +1,8 @@ from fastapi.testclient import TestClient +import pytest +from .test_miner_committee import _add_test_miner, clear_miner_db from ..migrations.init import init_newrl +from ..codes.minermanager import broadcast_miner_update from ..main import app @@ -53,6 +56,7 @@ def _receive_block(block_index): "previous_hash": "0000ae69c361c65f088b52af8f5372f94f5f62d84f0980ea4c2cd71551206024" }, "signature": { + "address": "0x20513a419d5b11cd510ae518dc04ac1690afbed6", "public": "4trPBhDwdxWat2I8tE4Mj+7R6tiTJ+44GWtTdf5QpXnh/Ia1i5x4ETDufrCn3mjYN8gJs/w3iiMlDEmAAs7kvg==", "msgsign": "8odtLy4zlyXNn7GFK4lpDtubGOS3bLFijmxXR1T8+TlLOl39+mA9Ajw8S4Sw3enJlGiWGorJr+0ULKdmeqf4Hw==" } @@ -72,6 +76,11 @@ def _receive_block(block_index): def test_block_receive(): + clear_miner_db() + + broadcast_miner_update() + response = client.post('/run-updater') + response = client.get('/get-last-block-index') assert response.status_code == 200 @@ -82,4 +91,24 @@ def test_block_receive(): current_block_index = int(response.text) # Block index should've increased by 1 - assert current_block_index == (previous_block_index + 1) \ No newline at end of file + assert current_block_index == (previous_block_index + 1) + + +def test_block_reject(): + """Expect block rejection from unexpected minors""" + clear_miner_db() + + _add_test_miner(1) + + response = client.get('/get-last-block-index') + assert response.status_code == 200 + + previous_block_index = int(response.text) + with pytest.raises(Exception) as e_info: + _receive_block(previous_block_index + 1) + + response = client.get('/get-last-block-index') + current_block_index = int(response.text) + + # Block index should not increase + assert current_block_index == previous_block_index \ No newline at end of file diff --git a/app/tests/test_timers.py b/app/tests/test_timers.py new file mode 100644 index 0000000..6aa777e --- /dev/null +++ b/app/tests/test_timers.py @@ -0,0 +1,52 @@ +import time + +from .test_p2p import _receive_block +from .test_miner_committee import _add_test_miner +from ..codes.updater import start_mining_clock +from ..codes.minermanager import broadcast_miner_update +from fastapi.testclient import TestClient +from ..constants import BLOCK_TIME_INTERVAL_SECONDS + +from ..main import app + +client = TestClient(app) + + +def test_mining_clock(): + broadcast_miner_update() + response = client.post('/run-updater') + assert response.status_code == 200 + + response = client.get('/get-last-block-index') + assert response.status_code == 200 + previous_block_index = int(response.text) + + start_mining_clock() + + time.sleep(BLOCK_TIME_INTERVAL_SECONDS + 2) + + response = client.get('/get-last-block-index') + assert response.status_code == 200 + block_index = int(response.text) + + assert block_index == previous_block_index + 2 + + for i in range(1, 3): + _add_test_miner(i) + # time.sleep(BLOCK_TIME_INTERVAL_SECONDS * 4) + # os._exit(1) + + +def test_all_timers(): + broadcast_miner_update() + response = client.post('/run-updater') + assert response.status_code == 200 + + broadcast_miner_update() + response = client.post('/run-updater') + + response = client.get('/get-last-block-index') + assert response.status_code == 200 + + previous_block_index = int(response.text) + _receive_block(previous_block_index + 1) \ No newline at end of file