From 752f17d9a2294dec1499b426f0a13f04bef4c707 Mon Sep 17 00:00:00 2001 From: Cristina Date: Thu, 3 Oct 2019 16:48:40 +0300 Subject: [PATCH 01/14] Publish milestone together with virtual transactions #188 --- .../helix/pendulum/TransactionValidator.java | 26 +++-- .../controllers/TransactionViewModel.java | 36 ++++++ .../net/helix/pendulum/crypto/Merkle.java | 67 +++++++++-- .../network/TransactionRequester.java | 3 +- .../java/net/helix/pendulum/service/API.java | 109 +++++++++++++++--- .../pendulum/utils/bundle/BundleUtils.java | 20 +++- .../net/helix/pendulum/model/HashTest.java | 26 +++++ 7 files changed, 249 insertions(+), 38 deletions(-) diff --git a/src/main/java/net/helix/pendulum/TransactionValidator.java b/src/main/java/net/helix/pendulum/TransactionValidator.java index fda7aabf..5f098800 100644 --- a/src/main/java/net/helix/pendulum/TransactionValidator.java +++ b/src/main/java/net/helix/pendulum/TransactionValidator.java @@ -144,9 +144,6 @@ public int getMinWeightMagnitude() { */ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { // ignore invalid timestamps for transactions that were requested by our node while solidifying a milestone - if(transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) { - return false; - } if (transactionViewModel.getAttachmentTimestamp() == 0) { return transactionViewModel.getTimestamp() < snapshotProvider.getInitialSnapshot().getTimestamp() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(transactionViewModel.getHash()) || transactionViewModel.getTimestamp() > (System.currentTimeMillis() / 1000) + MAX_TIMESTAMP_FUTURE; @@ -155,6 +152,14 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { || transactionViewModel.getAttachmentTimestamp() > System.currentTimeMillis() + MAX_TIMESTAMP_FUTURE_MS; } + private boolean isTransactionRequested(TransactionViewModel transactionViewModel) { + if(transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) { + //todo if is virtual compute it locally + return true; + } + return false; + } + /** * Runs the following validation checks on a transaction: *
    @@ -174,15 +179,17 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { public void runValidation(TransactionViewModel transactionViewModel, final int minWeightMagnitude) { transactionViewModel.setMetadata(); transactionViewModel.setAttachmentData(); + + if (!transactionViewModel.isVirtual() && isTransactionRequested(transactionViewModel)) { + log.error("Waiting for transaction... " + transactionViewModel.getHash()); + throw new IllegalStateException("Transaction is requested"); + } if(hasInvalidTimestamp(transactionViewModel)) { throw new StaleTimestampException("Invalid transaction timestamp."); } - for (int i = VALUE_OFFSET + VALUE_USABLE_SIZE; i < VALUE_OFFSET + VALUE_SIZE; i++) { // todo always false. - if (transactionViewModel.getBytes()[i] != 0) { - throw new IllegalStateException("Invalid transaction value"); - } + if(transactionViewModel.isVirtual()){ + return; } - int weightMagnitude = transactionViewModel.weightMagnitude; if((weightMagnitude < minWeightMagnitude)) { throw new IllegalStateException("Invalid transaction hash"); @@ -203,7 +210,8 @@ public void runValidation(TransactionViewModel transactionViewModel, final int m * @throws RuntimeException if validation fails */ public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagnitude) { - TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, TransactionHash.calculate(bytes, 0, bytes.length, SpongeFactory.create(SpongeFactory.Mode.S256))); + TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, SpongeFactory.Mode.S256); + runValidation(transactionViewModel, minWeightMagnitude); return transactionViewModel; } diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index de847bdb..3a0861f9 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -1,5 +1,6 @@ package net.helix.pendulum.controllers; +import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.model.*; import net.helix.pendulum.model.persistables.*; import net.helix.pendulum.service.milestone.MilestoneTracker; @@ -148,6 +149,28 @@ public TransactionViewModel(final byte[] bytes, Hash hash) throws RuntimeExcepti transaction.type = FILLED_SLOT; } + /** + * Constructor with transaction bytes and transaction hash. + * @param bytes transaction bytes + * @param mode mode of the hash function used to compute transaction hash + */ + public TransactionViewModel(final byte[] bytes, SpongeFactory.Mode mode) throws RuntimeException { + transaction = new Transaction(); + transaction.bytes = new byte[SIZE]; + System.arraycopy(bytes, 0, transaction.bytes, 0, SIZE); + if (isVirtual()) { + byte[] buffer = new byte[2 * Hash.SIZE_IN_BYTES]; + System.arraycopy(bytes, BRANCH_TRANSACTION_OFFSET, buffer, 0, Hash.SIZE_IN_BYTES); + System.arraycopy(bytes, TRUNK_TRANSACTION_OFFSET, buffer, Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); + this.hash = TransactionHash.calculate(buffer, 0, buffer.length, SpongeFactory.create(mode)); + } else { + this.hash = TransactionHash.calculate(bytes, 0, bytes.length, SpongeFactory.create(mode)); + } + + weightMagnitude = this.hash.leadingZeros(); + transaction.type = FILLED_SLOT; + } + /** * Get the number of transactins in the database. * @param tangle @@ -675,6 +698,19 @@ public boolean isMilestone() { return transaction.milestone; } + /** + * The {@link Transaction#milestone} flag indicates if the {@link Transaction} is a virtual transaction + * @return true if the {@link Transaction} is virtual and false otherwise + */ + public boolean isVirtual() { + byte[] bundleNonceForVirtual = new byte[Hash.SIZE_IN_BYTES]; + Arrays.fill(bundleNonceForVirtual, (byte) 0xff); + if (getBundleNonceHash().equals(HashFactory.BUNDLENONCE.create(bundleNonceForVirtual)) && Arrays.equals(getNonce(), new byte[NONCE_SIZE])) { + return true; + } + return false; + } + public TransactionViewModel isMilestoneBundle(Tangle tangle) throws Exception{ for (Hash bundleTx : BundleViewModel.load(tangle, getBundleHash()).getHashes()){ TransactionViewModel tx = TransactionViewModel.fromHash(tangle, bundleTx); diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java index ef55826c..19a849a0 100644 --- a/src/main/java/net/helix/pendulum/crypto/Merkle.java +++ b/src/main/java/net/helix/pendulum/crypto/Merkle.java @@ -4,7 +4,10 @@ import net.helix.pendulum.controllers.TransactionViewModel; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; +import net.helix.pendulum.utils.bundle.BundleUtils; import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.*; import java.nio.ByteBuffer; @@ -14,6 +17,8 @@ public class Merkle { + private static final Logger log = LoggerFactory.getLogger(Merkle.class); + public static byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { int index = indexIn; final Sponge sha3 = SpongeFactory.create(mode); @@ -55,6 +60,44 @@ public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, } + public static List buildMerkleTransactionTree(List leaves, Hash milestoneHash, Hash address){ + if (leaves.isEmpty()) { + leaves.add(Hash.NULL_HASH); + } + byte[] buffer; + Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); + int depth = (int) Math.ceil(Math.sqrt(leaves.size())); + int row = 1; + List virtualTransactionList = new ArrayList(); + while (leaves.size() > 1) { + List nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]); + for (int i = 0; i < nextKeys.size(); i++) { + if (areLeavesNull(leaves, i)) continue; + sha3.reset(); + Hash k1 = leaves.get(i * 2); + Hash k2 = leaves.get(i * 2 + 1); + buffer = computeParentHash(sha3, k1, k2); + Hash parentHash = HashFactory.TRANSACTION.create(buffer); + nextKeys.set(i, parentHash); + // if( i == 0) add only one virtual transaction for each level + virtualTransactionList.add(createVirtualTransaction(k1, k2, row, milestoneHash, address, parentHash)); + } + leaves = nextKeys; + row++; + } + return virtualTransactionList; + } + + private static byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { + byte[] buffer; + buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32); + sha3.absorb(buffer, 0, buffer.length); + buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32); + sha3.absorb(buffer, 0, buffer.length); + sha3.squeeze(buffer, 0, buffer.length); + return buffer; + } + public static List> buildMerkleTree(List leaves){ if (leaves.isEmpty()) { leaves.add(Hash.NULL_HASH); @@ -70,18 +113,11 @@ public static List> buildMerkleTree(List leaves){ // Take two following keys (i=0: (k0,k1), i=1: (k2,k3), ...) and get one crypto of them List nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]); for (int i = 0; i < nextKeys.size(); i++) { - if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { - // leave the combined key null as well - continue; - } + if (areLeavesNull(leaves, i)) continue; sha3.reset(); Hash k1 = leaves.get(i * 2); Hash k2 = leaves.get(i * 2 + 1); - buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32); - sha3.absorb(buffer, 0, buffer.length); - buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32); - sha3.absorb(buffer, 0, buffer.length); - sha3.squeeze(buffer, 0, buffer.length); + buffer = computeParentHash(sha3, k1, k2); nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); } leaves = nextKeys; @@ -90,6 +126,19 @@ public static List> buildMerkleTree(List leaves){ return merkleTree; } + private static boolean areLeavesNull(List leaves, int i) { + if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { + // leave the combined key null as well + return true; + } + return false; + } + + private static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, Hash milestoneHash, Hash address, Hash transactionHash) { + log.debug("New virtual transaction + " + transactionHash + " for milestone: " + milestoneHash + " with merkle index: " + merkleIndex + " [" + branchHash + ", " + trunkHash + " ]"); + return BundleUtils.createVirtualTransaction(branchHash, trunkHash, merkleIndex, milestoneHash, address); + } + public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) { //System.out.println("Validate Merkle Signature"); diff --git a/src/main/java/net/helix/pendulum/network/TransactionRequester.java b/src/main/java/net/helix/pendulum/network/TransactionRequester.java index de522101..5c56a2e5 100644 --- a/src/main/java/net/helix/pendulum/network/TransactionRequester.java +++ b/src/main/java/net/helix/pendulum/network/TransactionRequester.java @@ -90,7 +90,8 @@ public void requestTransaction(Hash hash, boolean milestone) throws Exception { protected void popEldestTransactionToRequest() { Iterator iterator = transactionsToRequest.iterator(); if (iterator.hasNext()) { - iterator.next(); + Hash transactionHash = iterator.next(); + log.debug("Transaction: " + transactionHash + " has been removed from requestList!"); iterator.remove(); } } diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 8251fdbd..4bfe26f6 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -13,11 +13,11 @@ import net.helix.pendulum.crypto.*; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; +import net.helix.pendulum.model.TransactionHash; import net.helix.pendulum.model.persistables.Transaction; import net.helix.pendulum.network.Neighbor; import net.helix.pendulum.network.Node; import net.helix.pendulum.network.TransactionRequester; -import net.helix.pendulum.service.validatomanager.CandidateTracker; import net.helix.pendulum.service.dto.*; import net.helix.pendulum.service.ledger.LedgerService; import net.helix.pendulum.service.milestone.MilestoneTracker; @@ -26,6 +26,8 @@ import net.helix.pendulum.service.spentaddresses.SpentAddressesService; import net.helix.pendulum.service.tipselection.TipSelector; import net.helix.pendulum.service.tipselection.impl.WalkValidatorImpl; +import net.helix.pendulum.service.utils.RoundIndexUtil; +import net.helix.pendulum.service.validatomanager.CandidateTracker; import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Serializer; import net.helix.pendulum.utils.bundle.BundleTypes; @@ -115,6 +117,7 @@ public class API { private final String[] features; + //endregion //////////////////////////////////////////////////////////////////////////////////////////////////////// private final Gson gson = new GsonBuilder().create(); @@ -177,7 +180,6 @@ public API(ApiArgs args) { commandRoute.put(ApiCommand.GET_MISSING_TRANSACTIONS, getMissingTransactions()); commandRoute.put(ApiCommand.CHECK_CONSISTENCY, checkConsistency()); commandRoute.put(ApiCommand.WERE_ADDRESSES_SPENT_FROM, wereAddressesSpentFrom()); - // commandRoute.put(ApiCommand.GET_MILESTONES, wereAddressesSpentFrom()); } /** @@ -682,7 +684,7 @@ private AbstractResponse getNodeAPIConfigurationStatement() { * @return {@link net.helix.pendulum.service.dto.GetInclusionStatesResponse} * @throws Exception When a transaction cannot be loaded from hash **/ - private AbstractResponse getInclusionStatesStatement(final List transactions, final List tips) throws Exception { + public AbstractResponse getInclusionStatesStatement(final List transactions, final List tips) throws Exception { final List trans = transactions.stream() .map(HashFactory.TRANSACTION::create) @@ -1177,12 +1179,7 @@ public synchronized List attachToTangleStatement(final Hash trunkTransac if(IntStream.range(TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_OFFSET + TransactionViewModel.TAG_SIZE).allMatch(idx -> txBytes[idx] == ((byte) 0))) { System.arraycopy(txBytes, TransactionViewModel.BUNDLE_NONCE_OFFSET, txBytes, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); } - System.arraycopy(Serializer.serialize(timestamp),0, txBytes,TransactionViewModel.ATTACHMENT_TIMESTAMP_OFFSET, - TransactionViewModel.ATTACHMENT_TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize(0L),0,txBytes, TransactionViewModel.ATTACHMENT_TIMESTAMP_LOWER_BOUND_OFFSET, - TransactionViewModel.ATTACHMENT_TIMESTAMP_LOWER_BOUND_SIZE); - System.arraycopy(Serializer.serialize(MAX_TIMESTAMP_VALUE),0,txBytes,TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_OFFSET, - TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_SIZE); + fillAttachmentTransactionFields(txBytes, timestamp); if (!miner.mine(txBytes, minWeightMagnitude, 4)) { transactionViewModels.clear(); @@ -1215,6 +1212,15 @@ public synchronized List attachToTangleStatement(final Hash trunkTransac return elements; } + private void fillAttachmentTransactionFields(byte[] txBytes, long timestamp) { + System.arraycopy(Serializer.serialize(timestamp),0, txBytes, TransactionViewModel.ATTACHMENT_TIMESTAMP_OFFSET, + TransactionViewModel.ATTACHMENT_TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize(0L),0,txBytes, TransactionViewModel.ATTACHMENT_TIMESTAMP_LOWER_BOUND_OFFSET, + TransactionViewModel.ATTACHMENT_TIMESTAMP_LOWER_BOUND_SIZE); + System.arraycopy(Serializer.serialize(MAX_TIMESTAMP_VALUE),0,txBytes,TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_OFFSET, + TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_SIZE); + } + /** * Can be 0 or more, and is set to 0 every 100 requests. * Each increase indicates another 2 tips send. @@ -1450,10 +1456,10 @@ private void attachStoreAndBroadcast(final String address, final String message, * @param txs transactions list * @throws Exception if storing fails */ - private void storeAndBroadcast(Hash tip1, Hash tip2, int mwm, List txs) throws Exception{ + private List attachAndStore(Hash tip1, Hash tip2, int mwm, List txs) throws Exception{ List powResult = attachToTangleStatement(tip1, tip2, mwm, txs); storeTransactionsStatement(powResult); - broadcastTransactionsStatement(powResult); + return powResult; } // @@ -1498,10 +1504,74 @@ public void publish(BundleTypes type, final String address, final int minWeightM * @param txToApprove transactions to approve * @throws Exception if storing fails */ - private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, String keyfile, int security) throws Exception { + private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, String keyfile, int security, boolean createVirtualTransactions) throws Exception { BundleUtils bundle = new BundleUtils(sndAddr, rcvAddr); bundle.create(data, tag, sign, keyIdx, maxKeyIdx, keyfile, security); - storeAndBroadcast(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); + List transactionStrings = attachAndStore(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); + + if (createVirtualTransactions) { + createAndBroadcastVirtualTransactions(data, transactionStrings); + } + broadcastTransactionsStatement(transactionStrings); + } + + /** + * Creates and broadcast virtual transactions + * + * @param tipsBytes + * @param milestoneBundle + */ + private void createAndBroadcastVirtualTransactions(byte[] tipsBytes, List milestoneBundle) { + TransactionViewModel milestone = getFirstTransactionFromBundle(milestoneBundle); + if (milestone != null) { + List tips = extractTipsFromData(tipsBytes); + if (tips.size() == 0) { + return; + } + List virtualTransactions = Merkle.buildMerkleTransactionTree(tips, + TransactionHash.calculate(SpongeFactory.Mode.S256, milestone.getBytes()), milestone.getAddressHash()). + stream().map(t -> { + fillAttachmentTransactionFields(t, RoundIndexUtil.getCurrentTime()); + return Hex.toHexString(t); + }).collect(Collectors.toList()); + broadcastTransactionsStatement(virtualTransactions); + } + } + + /** + * Get transaction with index = 0 from a transaction list + * + * @param bundle + * @return TransactionViewModel with current index = 0, otherwise null + */ + private TransactionViewModel getFirstTransactionFromBundle(List bundle) { + for (String transactionString : bundle) { + final TransactionViewModel transactionViewModel = transactionValidator.validateBytes(Hex.decode(transactionString), + transactionValidator.getMinWeightMagnitude()); + if (transactionViewModel.getCurrentIndex() == 0) { + return transactionViewModel; + } + } + return null; + } + + /** + * Extract tips list from data byte array + * + * @param data + * @return + */ + private List extractTipsFromData(byte[] data) { + List tips = new ArrayList<>(); + int i = 0; + while (i < data.length / Hash.SIZE_IN_BYTES) { + Hash tip = HashFactory.TRANSACTION.create(data, i++ * Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); + if (tip.equals(Hash.NULL_HASH)) { + break; + } + tips.add(tip); + } + return tips; } // refactoring WIP (the following methods will be moved from API) @@ -1521,7 +1591,8 @@ public void publishMilestone(final String address, final int minWeightMagnitude, byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); - storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity()); + + storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity(), true); } public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { @@ -1530,7 +1601,7 @@ public void publishRegistration(final String address, final int minWeightMagnitu List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getValidatorManagerAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity()); + storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getValidatorManagerAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity(), false); } public void publishKeyChange(final String oldAddress, final Hash newAddress, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1539,7 +1610,7 @@ public void publishKeyChange(final String oldAddress, final Hash newAddress, fin List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - storeCustomBundle(HashFactory.ADDRESS.create(oldAddress), configuration.getValidatorManagerAddress(), txToApprove, data, 0L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity()); + storeCustomBundle(HashFactory.ADDRESS.create(oldAddress), configuration.getValidatorManagerAddress(), txToApprove, data, 0L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity(), false); } public void publishValidator(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1551,7 +1622,7 @@ public void publishValidator(int startRoundDelay, final int minWeightMagnitude, // get branch and trunk List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - storeCustomBundle(configuration.getValidatorManagerAddress(), Hash.NULL_HASH, txToApprove, validatorBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorManagerKeyfile(), configuration.getValidatorManagerSecurity()); + storeCustomBundle(configuration.getValidatorManagerAddress(), Hash.NULL_HASH, txToApprove, validatorBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorManagerKeyfile(), configuration.getValidatorManagerSecurity(), false); } @@ -1574,7 +1645,7 @@ private List getConfirmedTips() throws Exception { BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { if (walkValidator.isValid(transaction)) { confirmedTips.add(transaction); - } else { + } else if(txVM.isSolid()){ log.warn("Inconsistent transaction has been removed from tips: " + transaction.toString()); tipsViewModel.removeTipHash(transaction); } @@ -1607,6 +1678,8 @@ private List addMilestoneReferences(List confirmedTips, int roundInd //branch List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips + + log.debug("Milestone future branch transaction hash: " + merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); } return txToApprove; } diff --git a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java index df8134fa..b6934e29 100644 --- a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java @@ -6,6 +6,7 @@ import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; import net.helix.pendulum.model.Hash; +import net.helix.pendulum.service.utils.RoundIndexUtil; import net.helix.pendulum.utils.Serializer; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -38,6 +39,9 @@ public BundleUtils(Hash senderAddress, Hash receiverAddress) { this.receiverAddress = receiverAddress.toString(); } + public BundleUtils() { + } + /** * @return transactions */ @@ -116,7 +120,7 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey * @param tag tag * @return transaction */ - private byte[] initTransaction(String address, int currentIndex, int lastIndex, long timestamp, long tag) { + public byte[] initTransaction(String address, int currentIndex, int lastIndex, long timestamp, long tag) { byte[] transaction = new byte[TransactionViewModel.SIZE]; System.arraycopy(Hex.decode(address), 0, transaction, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); System.arraycopy(Serializer.serialize((long) currentIndex), 0, transaction, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); @@ -180,4 +184,18 @@ private void signBundle(String filepath, byte[] merkleTransaction, List System.arraycopy(signature, 0, senderTransactions.get(i), TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); } } + + public static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, Hash milestoneHash, Hash address) { + BundleUtils bundleUtils = new BundleUtils(); + byte[] transaction = bundleUtils.initTransaction(Hex.toHexString(address.bytes()), 0, 0, RoundIndexUtil.getCurrentTime(), merkleIndex); + System.arraycopy(milestoneHash.bytes(), 0, transaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, milestoneHash.bytes().length); + + // mark this transaction as virtual (all bundle nonce bits are one) and nonce is zero. + byte[] bundleNonce = new byte[TransactionViewModel.BUNDLE_NONCE_SIZE]; + Arrays.fill(bundleNonce, (byte) 0xff); + System.arraycopy(bundleNonce, 0, transaction, TransactionViewModel.BUNDLE_NONCE_OFFSET, TransactionViewModel.BUNDLE_NONCE_SIZE); + System.arraycopy(branchHash.bytes(), 0, transaction, TransactionViewModel.BRANCH_TRANSACTION_OFFSET, TransactionViewModel.BRANCH_TRANSACTION_SIZE); + System.arraycopy(trunkHash.bytes(), 0, transaction, TransactionViewModel.TRUNK_TRANSACTION_OFFSET, TransactionViewModel.TRUNK_TRANSACTION_SIZE); + return transaction; + } } diff --git a/src/test/java/net/helix/pendulum/model/HashTest.java b/src/test/java/net/helix/pendulum/model/HashTest.java index 8e26a449..4cc65f1b 100644 --- a/src/test/java/net/helix/pendulum/model/HashTest.java +++ b/src/test/java/net/helix/pendulum/model/HashTest.java @@ -1,6 +1,8 @@ package net.helix.pendulum.model; +import net.helix.pendulum.TransactionTestUtils; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import org.bouncycastle.util.encoders.Hex; import org.junit.Assert; @@ -110,4 +112,28 @@ public void compareToTest() throws Exception { Assert.assertEquals(hash.compareTo(Hash.NULL_HASH), -Hash.NULL_HASH.compareTo(hash)); } + @Test + public void testHashAggregation() { + TransactionHash hash1 = TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes()); + TransactionHash hash2 = TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes()); + + Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); + byte[] result = new byte[Hash.SIZE_IN_BYTES]; + sha3.reset(); + sha3.absorb(hash1.bytes(), 0, hash1.getByteSize()); + sha3.absorb(hash2.bytes(), 0, hash2.getByteSize()); + sha3.squeeze(result, 0, Hash.SIZE_IN_BYTES); + TransactionHash resultHash = (TransactionHash) HashFactory.TRANSACTION.create(result, 0, Hash.SIZE_IN_BYTES); + + byte[] buffer = new byte[2 * Hash.SIZE_IN_BYTES]; + System.arraycopy(hash1.bytes(), 0, buffer, 0, hash1.getByteSize()); + System.arraycopy(hash2.bytes(), 0, buffer, Hash.SIZE_IN_BYTES, hash2.getByteSize()); + + TransactionHash hash = TransactionHash.calculate(SpongeFactory.Mode.S256, buffer); + + Assert.assertFalse(hash.equals(hash1)); + Assert.assertTrue(hash.equals(resultHash)); + Assert.assertFalse(hash.equals(Hash.NULL_HASH)); + Assert.assertFalse(resultHash.equals(Hash.NULL_HASH)); + } } From 39fb050a26a1e6f754b966900250321fd41ab351 Mon Sep 17 00:00:00 2001 From: Cristina Date: Thu, 10 Oct 2019 16:10:48 +0300 Subject: [PATCH 02/14] Fix code review issues (change tree depth, extract constant) --- .../net/helix/pendulum/crypto/Merkle.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java index 19a849a0..42e39db2 100644 --- a/src/main/java/net/helix/pendulum/crypto/Merkle.java +++ b/src/main/java/net/helix/pendulum/crypto/Merkle.java @@ -66,7 +66,6 @@ public static List buildMerkleTransactionTree(List leaves, Hash mi } byte[] buffer; Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); - int depth = (int) Math.ceil(Math.sqrt(leaves.size())); int row = 1; List virtualTransactionList = new ArrayList(); while (leaves.size() > 1) { @@ -90,9 +89,9 @@ public static List buildMerkleTransactionTree(List leaves, Hash mi private static byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { byte[] buffer; - buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32); + buffer = copyHash(k1); sha3.absorb(buffer, 0, buffer.length); - buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32); + buffer = copyHash(k2); sha3.absorb(buffer, 0, buffer.length); sha3.squeeze(buffer, 0, buffer.length); return buffer; @@ -104,7 +103,7 @@ public static List> buildMerkleTree(List leaves){ } byte[] buffer; Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); - int depth = (int) Math.ceil(Math.sqrt(leaves.size())); + int depth = getTreeDepth(leaves.size()); List> merkleTree = new ArrayList<>(depth + 1); merkleTree.add(0, leaves); int row = 1; @@ -134,20 +133,24 @@ private static boolean areLeavesNull(List leaves, int i) { return false; } + private static int getTreeDepth(int leavesNumber) { + return (int) Math.ceil((float)(leavesNumber / Math.log(2))); + } + private static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, Hash milestoneHash, Hash address, Hash transactionHash) { log.debug("New virtual transaction + " + transactionHash + " for milestone: " + milestoneHash + " with merkle index: " + merkleIndex + " [" + branchHash + ", " + trunkHash + " ]"); return BundleUtils.createVirtualTransaction(branchHash, trunkHash, merkleIndex, milestoneHash, address); } + private static byte[] copyHash(Hash k2) { + return Arrays.copyOfRange(k2 == null ? Hex.decode(Hash.NULL_HASH.bytes()) : k2.bytes(), 0, Hash.SIZE_IN_BYTES); + } + public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) { - //System.out.println("Validate Merkle Signature"); final TransactionViewModel merkleTx = bundleTransactionViewModels.get(securityLevel); int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex - //System.out.println("Address: " + validationAddress); - //System.out.println("Keyindex: " + keyIndex); - //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) //TODO: check if its okay here to use bundle hash instead of tx hash byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); @@ -164,15 +167,9 @@ public static boolean validateMerkleSignature(List bundleT byte[] digests = bb.array(); byte[] address = Winternitz.address(mode, digests); - //System.out.println("Public Key: " + Hex.toHexString(address)); - //validate Merkle path - //System.out.println("Merkle Path: " + Hex.toHexString(merkleTx.getSignature())); byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, merkleTx.getSignature(), 0, keyIndex, depth); - - //System.out.println("Recalculated Address: " + HashFactory.ADDRESS.create(merkleRoot)); - return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); } From bc91d2872c31e9a2a82aefe017d8b717915ae512 Mon Sep 17 00:00:00 2001 From: Cristina Date: Tue, 15 Oct 2019 18:31:11 +0300 Subject: [PATCH 03/14] Reconstruct virtual transactions locally #189 --- .../java/net/helix/pendulum/Pendulum.java | 12 +- .../helix/pendulum/TransactionValidator.java | 2 +- .../controllers/BundleNonceViewModel.java | 141 +++++++++++ .../controllers/TransactionViewModel.java | 34 ++- .../net/helix/pendulum/crypto/Merkle.java | 47 ++-- .../crypto/merkle/impl/MerkleTreeImpl.java | 4 + .../java/net/helix/pendulum/network/Node.java | 14 +- .../java/net/helix/pendulum/service/API.java | 17 +- .../milestone/VirtualTransactionService.java | 15 ++ .../impl/VirtualTransactionServiceImpl.java | 233 ++++++++++++++++++ .../pendulum/utils/bundle/BundleUtils.java | 36 ++- .../helix/pendulum/crypto/MerkleTreeTest.java | 47 ++++ .../net/helix/pendulum/network/NodeTest.java | 4 +- 13 files changed, 555 insertions(+), 51 deletions(-) create mode 100644 src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java create mode 100644 src/main/java/net/helix/pendulum/service/milestone/VirtualTransactionService.java create mode 100644 src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java create mode 100644 src/test/java/net/helix/pendulum/crypto/MerkleTreeTest.java diff --git a/src/main/java/net/helix/pendulum/Pendulum.java b/src/main/java/net/helix/pendulum/Pendulum.java index a32991d8..1734dd8a 100644 --- a/src/main/java/net/helix/pendulum/Pendulum.java +++ b/src/main/java/net/helix/pendulum/Pendulum.java @@ -11,11 +11,7 @@ import net.helix.pendulum.network.replicator.Replicator; import net.helix.pendulum.service.TipsSolidifier; import net.helix.pendulum.service.ledger.impl.LedgerServiceImpl; -import net.helix.pendulum.service.milestone.impl.LatestSolidMilestoneTrackerImpl; -import net.helix.pendulum.service.milestone.impl.MilestoneServiceImpl; -import net.helix.pendulum.service.milestone.impl.MilestoneSolidifierImpl; -import net.helix.pendulum.service.milestone.impl.MilestoneTrackerImpl; -import net.helix.pendulum.service.milestone.impl.SeenMilestonesRetrieverImpl; +import net.helix.pendulum.service.milestone.impl.*; import net.helix.pendulum.service.snapshot.SnapshotException; import net.helix.pendulum.service.snapshot.impl.LocalSnapshotManagerImpl; import net.helix.pendulum.service.snapshot.impl.SnapshotProviderImpl; @@ -106,6 +102,7 @@ public class Pendulum { public final MilestoneSolidifierImpl milestoneSolidifier; public final CandidateSolidifierImpl candidateSolidifier; public final TransactionRequesterWorkerImpl transactionRequesterWorker; + public final VirtualTransactionServiceImpl virtualTransactionService; public final Tangle tangle; public final TransactionValidator transactionValidator; @@ -153,7 +150,7 @@ public Pendulum(PendulumConfig configuration) throws TransactionPruningException ? new AsyncTransactionPruner() : null; transactionRequesterWorker = new TransactionRequesterWorkerImpl(); - + virtualTransactionService = new VirtualTransactionServiceImpl(); // legacy code bundleValidator = new BundleValidator(); tangle = new Tangle(); @@ -161,7 +158,7 @@ public Pendulum(PendulumConfig configuration) throws TransactionPruningException transactionRequester = new TransactionRequester(tangle, snapshotProvider); transactionValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration); node = new Node(tangle, snapshotProvider, transactionValidator, transactionRequester, tipsViewModel, - latestMilestoneTracker, configuration); + latestMilestoneTracker, virtualTransactionService, configuration); replicator = new Replicator(node, configuration); udpReceiver = new UDPReceiver(node, configuration); tipsSolidifier = new TipsSolidifier(tangle, transactionValidator, tipsViewModel, configuration); @@ -235,6 +232,7 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration); } milestoneService.init(tangle, snapshotProvider, snapshotService, transactionValidator, configuration); + virtualTransactionService.init(tangle, snapshotProvider, transactionValidator); validatorManagerService.init(tangle, snapshotProvider, snapshotService, configuration); candidateTracker.init(tangle, snapshotProvider, validatorManagerService, candidateSolidifier, configuration); latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, candidateTracker, configuration); diff --git a/src/main/java/net/helix/pendulum/TransactionValidator.java b/src/main/java/net/helix/pendulum/TransactionValidator.java index 5f098800..c097afcb 100644 --- a/src/main/java/net/helix/pendulum/TransactionValidator.java +++ b/src/main/java/net/helix/pendulum/TransactionValidator.java @@ -182,7 +182,7 @@ public void runValidation(TransactionViewModel transactionViewModel, final int m if (!transactionViewModel.isVirtual() && isTransactionRequested(transactionViewModel)) { log.error("Waiting for transaction... " + transactionViewModel.getHash()); - throw new IllegalStateException("Transaction is requested"); + throw new IllegalStateException("Transaction is requested {} " + transactionViewModel.getHash()); } if(hasInvalidTimestamp(transactionViewModel)) { throw new StaleTimestampException("Invalid transaction timestamp."); diff --git a/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java b/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java new file mode 100644 index 00000000..78393117 --- /dev/null +++ b/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java @@ -0,0 +1,141 @@ +package net.helix.pendulum.controllers; + +import net.helix.pendulum.model.BundleHash; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.persistables.Bundle; +import net.helix.pendulum.model.persistables.BundleNonce; +import net.helix.pendulum.storage.Indexable; +import net.helix.pendulum.storage.Persistable; +import net.helix.pendulum.storage.Tangle; +import net.helix.pendulum.utils.Pair; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** +* The BundleViewModel class is an implementation of the HashesViewModel interface. +* It consists of an Indexable bundle hash and the Bundle model, +* which contains the set of transaction hashes that are part of the bundle. +*/ +public class BundleNonceViewModel implements HashesViewModel { + private BundleNonce self; + private Indexable hash; + + /** + * Constructor with bundle hash + * @param hash bundle hash + */ + public BundleNonceViewModel(Hash hash) { + this.hash = hash; + } + + /** + * Constructor with bundle hash and related bundle model + * @param hashes bundle model + * @param hash transaction hash + */ + private BundleNonceViewModel(BundleNonce hashes, Indexable hash) { + self = hashes == null || hashes.set == null ? new BundleNonce(): hashes; + this.hash = hash; + } + + /** + * Get the BundleNonceViewModel of a given bundle hash from the database. + * @param tangle + * @param hash bundle hash + * @return BundleNoncefViewModel + */ + public static BundleNonceViewModel load(Tangle tangle, Indexable hash) throws Exception { + return new BundleNonceViewModel((BundleNonce) tangle.load(BundleNonce.class, hash), hash); + } + + /** + * Convert a transaction set hash into the bundle model. + * @param hash transaction hash + * @param hashToMerge mergable set hash + * @return Map.Entry map entry of bundle hash and Bundle model + */ + public static Map.Entry getEntry(Hash hash, Hash hashToMerge) throws Exception { + BundleNonce hashes = new BundleNonce(); + hashes.set.add(hashToMerge); + return new HashMap.SimpleEntry<>(hash, hashes); + } + + /** + * Store the bundle hash (key) + belonging transaction hashes (value) in the database. + * @param tangle + * @return boolean success + */ + public boolean store(Tangle tangle) throws Exception { + return tangle.save(self, hash); + } + + /** + * Get the number of transactions belonging to the bundle. + * @return int number + */ + public int size() { + return self.set.size(); + } + + /** + * Add a transaction hash to the bundle. + * @param theHash transaction hash + * @return boolean success + */ + public boolean addHash(Hash theHash) { + return getHashes().add(theHash); + } + + /** + * Get the bundle hash / index. + * @return Indexable bundle hash + */ + public Indexable getIndex() { + return hash; + } + + /** + * Get the set of transaction hashes belonging to the bundle. + * @return Set transaction hashes + */ + public Set getHashes() { + return self.set; + } + + /** + * Delete the bundle from the database. + * @param tangle + */ + @Override + public void delete(Tangle tangle) throws Exception { + tangle.delete(BundleNonce.class,hash); + } + + /** + * Get the first bundle from the database. + * @param tangle + * @return bundleNonceViewModel + */ + public static BundleNonceViewModel first(Tangle tangle) throws Exception { + Pair bundlePair = tangle.getFirst(Bundle.class, BundleHash.class); + if(bundlePair != null && bundlePair.hi != null) { + return new BundleNonceViewModel((BundleNonce) bundlePair.hi, bundlePair.low); + } + return null; + } + + /** + * Get the next bundle from the database. + * @param tangle + * @return bundleNonceViewModel + */ + public BundleNonceViewModel next(Tangle tangle) throws Exception { + Pair bundlePair = tangle.next(Bundle.class, hash); + if(bundlePair != null && bundlePair.hi != null) { + return new BundleNonceViewModel((BundleNonce) bundlePair.hi, bundlePair.low); + } + return null; + } +} diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index 3a0861f9..b6bba59b 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -1,5 +1,6 @@ package net.helix.pendulum.controllers; +import net.helix.pendulum.TransactionValidator; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.model.*; import net.helix.pendulum.model.persistables.*; @@ -10,6 +11,7 @@ import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Converter; import net.helix.pendulum.utils.Pair; +import net.helix.pendulum.utils.Serializer; import java.util.*; @@ -180,6 +182,7 @@ public static int getNumberOfStoredTransactions(Tangle tangle) throws Exception return tangle.getCount(Transaction.class).intValue(); } + /** * This method updates the metadata contained in the {@link Transaction} object, and updates the object in the * database. First, all the most recent {@link Hash} identifiers are fetched to make sure the object's metadata is @@ -337,6 +340,19 @@ public boolean store(Tangle tangle, Snapshot initialSnapshot) throws Exception { return tangle.saveBatch(batch); } + + public void storeTransactionLocal(Tangle tangle, Snapshot initialSnapshot, TransactionValidator transactionValidator) throws Exception { + if(store(tangle, initialSnapshot)) { + setArrivalTime(System.currentTimeMillis() / 1000L); + if (isMilestoneBundle(tangle) == null) { + transactionValidator.updateStatus(this); + } + updateSender("local"); + update(tangle, initialSnapshot, "sender"); + } + } + + /** * Creates a copy of the underlying {@link Transaction} object. * @@ -510,6 +526,15 @@ public Hash getTagValue() { return transaction.tag; } + /** + * Gets the {@link Long} identifier of a {@link Transaction}. + * + * @return The {@link Long} identifier. + */ + public long getTagLongValue() { + return Serializer.getLong(getBytes(), TransactionViewModel.TAG_OFFSET); + } + /** * Gets the {@link Transaction#attachmentTimestamp}. The Attachment Timestapm is used to show when a * transaction has been attached to the database. @@ -703,12 +728,9 @@ public boolean isMilestone() { * @return true if the {@link Transaction} is virtual and false otherwise */ public boolean isVirtual() { - byte[] bundleNonceForVirtual = new byte[Hash.SIZE_IN_BYTES]; - Arrays.fill(bundleNonceForVirtual, (byte) 0xff); - if (getBundleNonceHash().equals(HashFactory.BUNDLENONCE.create(bundleNonceForVirtual)) && Arrays.equals(getNonce(), new byte[NONCE_SIZE])) { - return true; - } - return false; + byte[] nonceForVirtual = new byte[NONCE_SIZE]; + Arrays.fill(nonceForVirtual, (byte) 0xff); + return Arrays.equals(getNonce(),nonceForVirtual); } public TransactionViewModel isMilestoneBundle(Tangle tangle) throws Exception{ diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java index 42e39db2..d30795d6 100644 --- a/src/main/java/net/helix/pendulum/crypto/Merkle.java +++ b/src/main/java/net/helix/pendulum/crypto/Merkle.java @@ -34,15 +34,15 @@ public static byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] sha3.squeeze(hash, 0, hash.length); index >>= 1; } - if(index != 0) { + if (index != 0) { return Hash.NULL_HASH.bytes(); } return hash; } - public static List getMerklePath(List> merkleTree, int keyIndex){ - List merklePath = new ArrayList<>((merkleTree.size()-1) * 32); - for (int i = 0; i < merkleTree.size()-1; i++) { + public static List getMerklePath(List> merkleTree, int keyIndex) { + List merklePath = new ArrayList<>((merkleTree.size() - 1) * 32); + for (int i = 0; i < merkleTree.size() - 1; i++) { Hash subkey = merkleTree.get(i).get(keyIndex ^ 1); merklePath.add(subkey == null ? Hash.NULL_HASH : subkey); keyIndex /= 2; @@ -50,7 +50,7 @@ public static List getMerklePath(List> merkleTree, int keyIndex return merklePath; } - public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security){ + public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security) { List keys = new ArrayList<>(1 << pubkeyDepth); for (int i = 0; i < pubkeyCount; i++) { int idx = firstIndex + i; @@ -60,7 +60,7 @@ public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, } - public static List buildMerkleTransactionTree(List leaves, Hash milestoneHash, Hash address){ + public static List buildMerkleTransactionTree(List leaves, Hash milestoneHash, Hash address) { if (leaves.isEmpty()) { leaves.add(Hash.NULL_HASH); } @@ -68,18 +68,18 @@ public static List buildMerkleTransactionTree(List leaves, Hash mi Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); int row = 1; List virtualTransactionList = new ArrayList(); + int depth = getTreeDepth(leaves.size()); while (leaves.size() > 1) { - List nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]); + List nextKeys = Arrays.asList(new Hash[(leaves.size() % 2 == 0 ? (leaves.size() / 2) : (leaves.size() / 2 + 1))]); for (int i = 0; i < nextKeys.size(); i++) { if (areLeavesNull(leaves, i)) continue; sha3.reset(); - Hash k1 = leaves.get(i * 2); - Hash k2 = leaves.get(i * 2 + 1); + Hash k1 = getLeaves(leaves, i * 2); + Hash k2 = getLeaves(leaves, i * 2 + 1); buffer = computeParentHash(sha3, k1, k2); Hash parentHash = HashFactory.TRANSACTION.create(buffer); nextKeys.set(i, parentHash); - // if( i == 0) add only one virtual transaction for each level - virtualTransactionList.add(createVirtualTransaction(k1, k2, row, milestoneHash, address, parentHash)); + virtualTransactionList.add(createVirtualTransaction(k1, k2, getParentMerkleIndex(row, depth, i * 2), milestoneHash, address, parentHash)); } leaves = nextKeys; row++; @@ -87,6 +87,18 @@ public static List buildMerkleTransactionTree(List leaves, Hash mi return virtualTransactionList; } + private static Hash getLeaves(List leaves, int index) { + return index < leaves.size() ? leaves.get(index) : Hash.NULL_HASH; + } + + private static long getParentMerkleIndex(int row, int depth, int i) { + if (row == depth) { + return 0; // root + } + long index = depth - row; + return (long) Math.pow(2, index) + i / 2 - 1; + } + private static byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { byte[] buffer; buffer = copyHash(k1); @@ -97,7 +109,7 @@ private static byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { return buffer; } - public static List> buildMerkleTree(List leaves){ + public static List> buildMerkleTree(List leaves) { if (leaves.isEmpty()) { leaves.add(Hash.NULL_HASH); } @@ -127,19 +139,18 @@ public static List> buildMerkleTree(List leaves){ private static boolean areLeavesNull(List leaves, int i) { if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { - // leave the combined key null as well return true; } return false; } private static int getTreeDepth(int leavesNumber) { - return (int) Math.ceil((float)(leavesNumber / Math.log(2))); + return (int) Math.ceil((float) (Math.log(leavesNumber) / Math.log(2))); } private static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, Hash milestoneHash, Hash address, Hash transactionHash) { log.debug("New virtual transaction + " + transactionHash + " for milestone: " + milestoneHash + " with merkle index: " + merkleIndex + " [" + branchHash + ", " + trunkHash + " ]"); - return BundleUtils.createVirtualTransaction(branchHash, trunkHash, merkleIndex, milestoneHash, address); + return BundleUtils.createVirtualTransaction(branchHash, trunkHash, merkleIndex, milestoneHash.bytes(), address); } private static byte[] copyHash(Hash k2) { @@ -159,7 +170,7 @@ public static boolean validateMerkleSignature(List bundleT ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * securityLevel); for (int i = 0; i < securityLevel; i++) { - byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i+1)); + byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i + 1)); byte[] digest = Winternitz.digest(mode, bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); bb.put(digest); } @@ -186,7 +197,7 @@ public static List> readKeyfile(File keyfile) throws IOException { row.add(Hash.NULL_HASH); } for (int j = 0; j < fields[1].length() / 64; j++) { - row.add(HashFactory.TRANSACTION.create(fields[1].substring(j * 64, (j+1) * 64))); + row.add(HashFactory.TRANSACTION.create(fields[1].substring(j * 64, (j + 1) * 64))); } result.add(row); } @@ -232,7 +243,7 @@ private static void writeKeys(BufferedWriter bw, List keys) throws IOExcep bw.newLine(); } - public static byte[] padding(byte[] input, int length){ + public static byte[] padding(byte[] input, int length) { if (input.length < length) { byte[] output = new byte[length]; System.arraycopy(input, 0, output, length - input.length, input.length); diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java new file mode 100644 index 00000000..9a9396e1 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java @@ -0,0 +1,4 @@ +package net.helix.pendulum.crypto.merkle; + +public class MerkleTreeImpl { +} diff --git a/src/main/java/net/helix/pendulum/network/Node.java b/src/main/java/net/helix/pendulum/network/Node.java index fe45302a..fa0c156c 100644 --- a/src/main/java/net/helix/pendulum/network/Node.java +++ b/src/main/java/net/helix/pendulum/network/Node.java @@ -13,6 +13,7 @@ import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.TransactionHash; import net.helix.pendulum.service.milestone.MilestoneTracker; +import net.helix.pendulum.service.milestone.VirtualTransactionService; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.storage.Tangle; import org.apache.commons.lang3.StringUtils; @@ -75,6 +76,7 @@ public class Node { private final TipsViewModel tipsViewModel; private final TransactionValidator transactionValidator; private final TransactionRequester transactionRequester; + private final VirtualTransactionService virtualTransactionService; private static final SecureRandom rnd = new SecureRandom(); @@ -109,7 +111,8 @@ public class Node { * @param configuration Contains all the config. * */ - public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final MilestoneTracker milestoneTracker, final NodeConfig configuration + public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final MilestoneTracker milestoneTracker, + final VirtualTransactionService virtualTransactionService, final NodeConfig configuration ) { this.configuration = configuration; this.tangle = tangle; @@ -121,7 +124,7 @@ public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final Transa int packetSize = configuration.getTransactionPacketSize(); this.sendingPacket = new DatagramPacket(new byte[packetSize], packetSize); this.tipRequestingPacket = new DatagramPacket(new byte[packetSize], packetSize); - + this.virtualTransactionService = virtualTransactionService; } /** @@ -313,6 +316,7 @@ public void preProcessReceivedData(byte[] receivedData, SocketAddress senderAddr } //if valid - add to receive queue (receivedTransactionViewModel, neighbor) + buildVirtualTransaction(receivedTransactionViewModel); addReceivedDataToReceiveQueue(receivedTransactionViewModel, neighbor); } @@ -400,6 +404,12 @@ public void preProcessReceivedData(byte[] receivedData, SocketAddress senderAddr } } + public void buildVirtualTransaction(TransactionViewModel model){ + if(model.isVirtual()){ + virtualTransactionService.rebuildVirtualTransactionsIfPossible(model); + } + } + /** * Adds incoming transactions to the {@link Node#receiveQueue} to be processed later. */ diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 4bfe26f6..01961bac 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -11,6 +11,7 @@ import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.*; import net.helix.pendulum.crypto.*; +import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.TransactionHash; @@ -592,15 +593,7 @@ private synchronized AbstractResponse getTipsStatement() throws Exception { public void storeTransactionsStatement(final List txString) throws Exception { final List elements = addValidTxvmToList(txString); for (final TransactionViewModel transactionViewModel : elements) { - //store transactions - if(transactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot())) { - transactionViewModel.setArrivalTime(System.currentTimeMillis() / 1000L); - if (transactionViewModel.isMilestoneBundle(tangle) == null) { - transactionValidator.updateStatus(transactionViewModel); - } - transactionViewModel.updateSender("local"); - transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "sender"); - } + transactionViewModel.storeTransactionLocal(tangle, snapshotProvider.getInitialSnapshot(), transactionValidator); } } @@ -1531,7 +1524,13 @@ private void createAndBroadcastVirtualTransactions(byte[] tipsBytes, List virtualTransactions = Merkle.buildMerkleTransactionTree(tips, TransactionHash.calculate(SpongeFactory.Mode.S256, milestone.getBytes()), milestone.getAddressHash()). stream().map(t -> { + try { fillAttachmentTransactionFields(t, RoundIndexUtil.getCurrentTime()); + TransactionViewModel virtualTransaction = new TransactionViewModel(t, SpongeFactory.Mode.S256); + virtualTransaction.storeTransactionLocal(tangle,snapshotProvider.getInitialSnapshot(),transactionValidator); + } catch (Exception e) { + e.printStackTrace(); + } return Hex.toHexString(t); }).collect(Collectors.toList()); broadcastTransactionsStatement(virtualTransactions); diff --git a/src/main/java/net/helix/pendulum/service/milestone/VirtualTransactionService.java b/src/main/java/net/helix/pendulum/service/milestone/VirtualTransactionService.java new file mode 100644 index 00000000..699d3811 --- /dev/null +++ b/src/main/java/net/helix/pendulum/service/milestone/VirtualTransactionService.java @@ -0,0 +1,15 @@ +package net.helix.pendulum.service.milestone; + +import net.helix.pendulum.controllers.TransactionViewModel; + + +public interface VirtualTransactionService { + + /** + * It tries to rebuild the virtual parent transaction for given trasaction model, parent transaction could be build if the sibling transaction already exists, + * then it will try to rebuild the next parent transaction an so on. + * @param transactionViewModel + * @return number of the built transactions. + */ + int rebuildVirtualTransactionsIfPossible(TransactionViewModel transactionViewModel); +} diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java new file mode 100644 index 00000000..fab27f33 --- /dev/null +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java @@ -0,0 +1,233 @@ +package net.helix.pendulum.service.milestone.impl; + +import net.helix.pendulum.TransactionValidator; +import net.helix.pendulum.controllers.BundleNonceViewModel; +import net.helix.pendulum.controllers.RoundViewModel; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.service.milestone.VirtualTransactionService; +import net.helix.pendulum.service.snapshot.SnapshotProvider; +import net.helix.pendulum.storage.Tangle; +import net.helix.pendulum.utils.bundle.BundleUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +import static net.helix.pendulum.controllers.TransactionViewModel.PREFILLED_SLOT; +import static net.helix.pendulum.controllers.TransactionViewModel.fromHash; + +/** + * Creates a service instance that allows us to perform virtual transaction operations.
    + *
    + */ +public class VirtualTransactionServiceImpl implements VirtualTransactionService { + + private static final Logger log = LoggerFactory.getLogger(VirtualTransactionServiceImpl.class); + + /** + * Holds the tangle object which acts as a database interface.
    + */ + private Tangle tangle; + + /** + * Holds the snapshot provider which gives us access to the relevant snapshots.
    + */ + private SnapshotProvider snapshotProvider; + + /** + * Holds transaction validator.
    + */ + private TransactionValidator transactionValidator; + + + /** + * This method initializes the instance and registers its dependencies.
    + *
    + * It simply stores the passed in values in their corresponding private properties.
    + *
    + * Note: Instead of handing over the dependencies in the constructor, we register them lazy. This allows us to have + * circular dependencies because the instantiation is separated from the dependency injection. To reduce the + * amount of code that is necessary to correctly instantiate this class, we return the instance itself which + * allows us to still instantiate, initialize and assign in one line - see Example:
    + *
    + * {@code milestoneService = new VirtualTransactionServiceImpl().init(...);} + * + * @param tangle Tangle object which acts as a database interface + * @param snapshotProvider snapshot provider which gives us access to the relevant snapshots + * @return the initialized instance itself to allow chaining + */ + public VirtualTransactionServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, TransactionValidator transactionValidator) { + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.transactionValidator = transactionValidator; + return this; + } + + /** + * Rebuilds a new virtual transaction which is based on current virtual transaction and its sibling (it this transaction exists) + * Attached the new built transaction in local tangle + * + * @param transactionViewModel transaction view model of the child + * @return true if the new transaction is created, otherwise false. + */ + @Override + public int rebuildVirtualTransactionsIfPossible(TransactionViewModel transactionViewModel) { + if (!transactionViewModel.isVirtual()) { + return 0; + } + int noOfRebuitTransactions = 0; + TransactionViewModel nextTransaction = transactionViewModel; + while (nextTransaction != null) { + nextTransaction = rebuildViirtualParentTransactionIfPossible(nextTransaction); + if (nextTransaction != null) { + noOfRebuitTransactions++; + } + } + return noOfRebuitTransactions; + } + + private TransactionViewModel rebuildViirtualParentTransactionIfPossible(TransactionViewModel tvm) { + long currentIndex = tvm.getTagLongValue(); + long siblingIndex = getSiblingIndex(currentIndex); + long parentIndex = getParentIndex(currentIndex); + + Hash milestoneHash = tvm.getBundleNonceHash(); + Map transactions = findByTagAndMilestone(milestoneHash, Arrays.asList(siblingIndex, parentIndex)); + + if (!transactions.containsKey(parentIndex) && transactions.containsKey(siblingIndex)) { + try { + TransactionViewModel siblingTransaction = fromHash(tangle, transactions.get(siblingIndex)); + TransactionViewModel newTransaction = reconstructParent(tvm, siblingTransaction); + log.debug("Rebuild transaction virtual parent transaction {} for: {} ", newTransaction.getHash(), tvm.getHash()); + return newTransaction; + } catch (Exception e) { + log.error("Error during generation of virtual transaction parent!"); + } + } + return null; + } + + /** + * Reconstruct virtual parent based on 2 transactions which are consider the direct children of the new created transaction. + * Order of the t1 and t2 doesn't matter, they will be sorted according with their merkle index. + * + * @param t1 first child transaction + * @param t2 second child transactiopn + * @return + * @throws Exception + */ + private TransactionViewModel reconstructParent(TransactionViewModel t1, TransactionViewModel t2) throws Exception { + if (validVirtualTransaction(Arrays.asList(t1, t2))) { + log.warn("Transactions: {} {} are not valid virtual transaction!", t1.getHash(), t2.getHash()); + return null; + } + Hash trunk = RoundViewModel.getRoundIndex(t1) > RoundViewModel.getRoundIndex(t2) ? t2.getHash() : t1.getHash(); + Hash branch = RoundViewModel.getRoundIndex(t1) < RoundViewModel.getRoundIndex(t2) ? t2.getHash() : t1.getHash(); + + byte[] newTransaction = BundleUtils.createVirtualTransaction(branch, trunk, getParentIndex(RoundViewModel.getRoundIndex(t1)), + t1.getSignature(), t1.getAddressHash()); + + TransactionViewModel newTransactionViewModel = new TransactionViewModel(newTransaction, SpongeFactory.Mode.S256); + newTransactionViewModel.storeTransactionLocal(tangle, snapshotProvider.getInitialSnapshot(), transactionValidator); + return newTransactionViewModel; + } + + /** + * Get parent index of the current virtual transaction + * + * @param index merkle index of the current transaction + * @return merkle index of the parent transaction + */ + private long getParentIndex(long index) { + return index % 2 == 0 ? index / 2 - 1 : (index - 1) / 2L; + } + + /** + * Get sibling index, based on current transaction index. + * + * @param index of the current transaction + * @return the index of the other child (from merkle tree) + */ + private long getSiblingIndex(long index) { + return index % 2 == 0 ? index - 1 : index + 1; + } + + + /** + * Virtual transactions validator. Call areVirtualTransaction and areSiblingVirtualTransactions validators. + * + * @param transactions, list of transactions that will be validated + * @return true if all transaction from povided list are valid virtual transactions. + */ + private boolean validVirtualTransaction(List transactions) { + return areVirtualTransaction(transactions) && + areSiblingVirtualTransactions(transactions); + } + + /** + * Virtual transactions validator. Checks if all transactions are virtual. + * + * @param transactions, list of transactions that will be validated + * @return true if all transaction from povided list are valid virtual transactions. + */ + private boolean areVirtualTransaction(List transactions) { + if (CollectionUtils.isEmpty(transactions)) { + return false; + } + return transactions.stream().filter(t -> !t.isVirtual()).collect(Collectors.toList()).size() == 0; + } + + /** + * Virtual transactions validator. Checks if all transactions are siblings + * + * @param transactions, list of transactions that will be validated + * @return true if all transaction from povided list are valid virtual transactions. + */ + private boolean areSiblingVirtualTransactions(List transactions) { + if (CollectionUtils.isEmpty(transactions)) { + return false; + } + Hash firstTag = transactions.get(0).getTagValue(); + if (firstTag == null) { + return false; + } + return transactions.stream().filter(t -> firstTag.equals(t.getTagValue())).collect(Collectors.toList()).size() == 0; + } + + + /** + * Find all transaction which belongs to milestone and have tags from parameter. + * + * @param milestoneHash milestone hash + * @param tags List of longs which represents the index in merkle tree + * @return map of tag (long value) and transaction hashes + */ + public Map findByTagAndMilestone(Hash milestoneHash, List tags) { + Map tagTransactionHash = new HashMap(); + try { + Set transactionHashes = BundleNonceViewModel.load(tangle, milestoneHash) + .getHashes(); + for (Hash t : transactionHashes) { + TransactionViewModel tvm = fromHash(tangle, t); + if (tvm.getType() == PREFILLED_SLOT) { + continue; + } + tags.stream().filter(tag -> !tagTransactionHash.keySet().contains(tag)).forEach(tag -> { + if (tag == tvm.getTagLongValue()) { + tagTransactionHash.put(tag, t); + } + }); + if (tagTransactionHash.size() == tags.size()) { + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return tagTransactionHash; + } +} diff --git a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java index b6934e29..929737d7 100644 --- a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java @@ -21,7 +21,7 @@ public class BundleUtils { - private final Logger log = LoggerFactory.getLogger(BundleUtils.class); + private static final Logger log = LoggerFactory.getLogger(BundleUtils.class); private List senderTransactions = new ArrayList<>(); private byte[] merkleTransaction; @@ -185,15 +185,39 @@ private void signBundle(String filepath, byte[] merkleTransaction, List } } - public static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, Hash milestoneHash, Hash address) { + /** + * Create a virtual transaction, the following fields are filled: + *
  1. Branch and trunk transactions with information given in input parameters, should be the children of the current node in Merkle tree
    2. + *
  2. Tag with merkle index from input parameters
    3. + *
  3. Address with address from input parameters, should be the address of the validator
    4. + *
  4. SignatureMessage with bundleNonce from input parameters, should be the hash of the milestone
    5. + *
  5. BundleNonce with a default value for virtual transaction TransasctionViewModel.DEFAULT_VIRTUAL_BUNDLE_NONCE
    6. + *
  6. Timestamp with current timestamp
    7. + *
  7. Current index and last index are set as 0
    8. + * + * @param branchHash - branch transaction hash + * @param trunkHash - trunk transaction hash + * @param merkleIndex - index in merkle tree + * @param bundleNonce - list of bytes should contain the milestone hash + * @param address - current address of the validator + * @return + */ + public static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, byte[] bundleNonce, Hash address) { + + if(bundleNonce.length > TransactionViewModel.BUNDLE_NONCE_SIZE){ + log.error("Tried to create a virtual trasaction with bundleNonce greater than {}, for branch{}, and trunk {}.", + TransactionViewModel.BUNDLE_NONCE_SIZE, branchHash.toString(), trunkHash.toString()); + return null; + } + BundleUtils bundleUtils = new BundleUtils(); byte[] transaction = bundleUtils.initTransaction(Hex.toHexString(address.bytes()), 0, 0, RoundIndexUtil.getCurrentTime(), merkleIndex); - System.arraycopy(milestoneHash.bytes(), 0, transaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, milestoneHash.bytes().length); + System.arraycopy(bundleNonce, 0, transaction, TransactionViewModel.BUNDLE_NONCE_OFFSET, TransactionViewModel.BUNDLE_NONCE_SIZE); // mark this transaction as virtual (all bundle nonce bits are one) and nonce is zero. - byte[] bundleNonce = new byte[TransactionViewModel.BUNDLE_NONCE_SIZE]; - Arrays.fill(bundleNonce, (byte) 0xff); - System.arraycopy(bundleNonce, 0, transaction, TransactionViewModel.BUNDLE_NONCE_OFFSET, TransactionViewModel.BUNDLE_NONCE_SIZE); + byte[] nonce = new byte[TransactionViewModel.NONCE_SIZE]; + Arrays.fill(nonce, (byte) 0xff); + System.arraycopy(nonce, 0, transaction, TransactionViewModel.NONCE_OFFSET, TransactionViewModel.NONCE_SIZE); System.arraycopy(branchHash.bytes(), 0, transaction, TransactionViewModel.BRANCH_TRANSACTION_OFFSET, TransactionViewModel.BRANCH_TRANSACTION_SIZE); System.arraycopy(trunkHash.bytes(), 0, transaction, TransactionViewModel.TRUNK_TRANSACTION_OFFSET, TransactionViewModel.TRUNK_TRANSACTION_SIZE); return transaction; diff --git a/src/test/java/net/helix/pendulum/crypto/MerkleTreeTest.java b/src/test/java/net/helix/pendulum/crypto/MerkleTreeTest.java new file mode 100644 index 00000000..e47b5e7f --- /dev/null +++ b/src/test/java/net/helix/pendulum/crypto/MerkleTreeTest.java @@ -0,0 +1,47 @@ +package net.helix.pendulum.crypto; + +import net.helix.pendulum.TransactionTestUtils; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.TransactionHash; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + + +public class MerkleTest { + + private static final Logger log = LoggerFactory.getLogger(MerkleTest.class); + + @Test + public void testTreeTransactionGeneration() { + // 6 index is missing because null leaves are not added into merkle tree + checkMerkleVirtualIndexes(10, new HashSet(Arrays.asList(0L, 1L, 2L, 3L, 4L, 5L, 7L, 8L, 9L, 10L, 11L))); + } + + private void checkMerkleVirtualIndexes(int noOfLeaves, Set expectedIndexes) { + TransactionHash milestoneHash = TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes()); + List transactions = new ArrayList(); + for (int i = 0; i < noOfLeaves; i++) { + transactions.add(TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes())); + } + + List virtualTransactions = Merkle.buildMerkleTransactionTree(transactions, milestoneHash, Hash.NULL_HASH); + Assert.assertTrue(virtualTransactions.size() > 0); + Set merkleIndexes = new HashSet(); + virtualTransactions.forEach(t -> { + try { + TransactionViewModel virtualTransaction = new TransactionViewModel(t, SpongeFactory.Mode.S256); + Assert.assertTrue(virtualTransaction.isVirtual()); + merkleIndexes.add(virtualTransaction.getTagLongValue()); + + } catch (Exception e) { + e.printStackTrace(); + } + }); + Assert.assertEquals(expectedIndexes, merkleIndexes); + } +} diff --git a/src/test/java/net/helix/pendulum/network/NodeTest.java b/src/test/java/net/helix/pendulum/network/NodeTest.java index d2cc9cd7..d4ed4f52 100644 --- a/src/test/java/net/helix/pendulum/network/NodeTest.java +++ b/src/test/java/net/helix/pendulum/network/NodeTest.java @@ -41,7 +41,7 @@ public void setUp() { // set up class under test nodeConfig = mock(NodeConfig.class); - classUnderTest = new Node(null, null, null, null, null, null, nodeConfig); + classUnderTest = new Node(null, null, null, null, null, null, null, nodeConfig); // verify config calls in Node constructor verify(nodeConfig).getRequestHashSize(); @@ -71,7 +71,7 @@ public void spawnNeighborDNSRefresherThreadTest() { @Test public void whenProcessReceivedDataSetArrivalTimeToCurrentMillis() throws Exception { - Node node = new Node(mock(Tangle.class), mock(SnapshotProvider.class), mock(TransactionValidator.class), null, null, null, mock(NodeConfig.class)); + Node node = new Node(mock(Tangle.class), mock(SnapshotProvider.class), mock(TransactionValidator.class), null, null, null, null, mock(NodeConfig.class)); TransactionViewModel transaction = mock(TransactionViewModel.class); // It is important to stub the getHash method here because processReceivedData will broadcast the transaction. // This might sometimes (concurrency issue) lead to a NPE in the process receiver thread. From 6290a45344b5a56d1e659f8671920d4730c5926b Mon Sep 17 00:00:00 2001 From: Cristina Date: Wed, 16 Oct 2019 17:57:55 +0300 Subject: [PATCH 04/14] Refactor Merkle --- .../java/net/helix/pendulum/SignedFiles.java | 8 +- .../pendulum/controllers/RoundViewModel.java | 22 +- .../controllers/TransactionViewModel.java | 7 +- .../net/helix/pendulum/crypto/Merkle.java | 260 ------------------ .../pendulum/crypto/merkle/MerkleNode.java | 9 + .../pendulum/crypto/merkle/MerkleOptions.java | 87 ++++++ .../pendulum/crypto/merkle/MerkleTree.java | 78 ++++++ .../merkle/impl/AbstractMerkleTree.java | 199 ++++++++++++++ .../crypto/merkle/impl/MerkleTreeImpl.java | 23 +- .../impl/TransactionMerkleTreeImpl.java | 35 +++ .../java/net/helix/pendulum/model/Hash.java | 3 +- .../java/net/helix/pendulum/service/API.java | 29 +- .../milestone/impl/MilestonePublisher.java | 16 +- .../milestone/impl/MilestoneServiceImpl.java | 5 +- .../impl/ValidatorManagerServiceImpl.java | 6 +- .../validator/impl/ValidatorServiceImpl.java | 5 +- .../net/helix/pendulum/utils/KeyfileUtil.java | 98 +++++++ .../pendulum/utils/bundle/BundleUtils.java | 11 +- .../crypto/merkle/MerkleTreeTest.java | 58 ++++ .../TransactionMerkleTreeTest.java} | 16 +- 20 files changed, 669 insertions(+), 306 deletions(-) delete mode 100644 src/main/java/net/helix/pendulum/crypto/Merkle.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleNode.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/impl/TransactionMerkleTreeImpl.java create mode 100644 src/main/java/net/helix/pendulum/utils/KeyfileUtil.java create mode 100644 src/test/java/net/helix/pendulum/crypto/merkle/MerkleTreeTest.java rename src/test/java/net/helix/pendulum/crypto/{MerkleTreeTest.java => merkle/TransactionMerkleTreeTest.java} (68%) diff --git a/src/main/java/net/helix/pendulum/SignedFiles.java b/src/main/java/net/helix/pendulum/SignedFiles.java index 415ba60f..692d7e4f 100644 --- a/src/main/java/net/helix/pendulum/SignedFiles.java +++ b/src/main/java/net/helix/pendulum/SignedFiles.java @@ -5,6 +5,8 @@ import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import org.apache.commons.lang3.ArrayUtils; import org.bouncycastle.util.encoders.Hex; @@ -31,7 +33,7 @@ private static boolean validateSignature(String signatureFilename, String public byte[] bundle = Winternitz.normalizedBundle(digest); byte[] root; int i; - + MerkleTree merkle = new MerkleTreeImpl(); try (InputStream inputStream = SignedFiles.class.getResourceAsStream(signatureFilename); BufferedReader reader = new BufferedReader((inputStream == null) ? new FileReader(signatureFilename) : new InputStreamReader(inputStream))) { @@ -46,7 +48,7 @@ private static boolean validateSignature(String signatureFilename, String public if ((line = reader.readLine()) != null) { byte[] lineBytes = Hex.decode(line); - root = Merkle.getMerkleRoot(mode, Winternitz.address(mode, digests), lineBytes, 0, index, depth); + root = merkle.getMerkleRoot(mode, Winternitz.address(mode, digests), lineBytes, 0, index, depth); } else { root = Winternitz.address(mode, digests); @@ -75,7 +77,7 @@ private static byte[] digestFile(String filename, Sponge sha3) throws IOExceptio messageBytes = Hash.NULL_HASH.bytes(); } int requiredLength = (int) Math.ceil(messageBytes.length / 32.0) * 32; - byte[] finalizedMessage = Merkle.padding(messageBytes, requiredLength); + byte[] finalizedMessage = MerkleTree.padding(messageBytes, requiredLength); // crypto snapshot message sha3.absorb(finalizedMessage, 0, finalizedMessage.length); byte[] signature = new byte[Sha3.HASH_LENGTH]; diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 6f90b594..beafce77 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -2,6 +2,9 @@ import net.helix.pendulum.TransactionValidator; import net.helix.pendulum.crypto.Merkle; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.IntegerIndex; @@ -256,8 +259,9 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + MerkleTree merkle = new MerkleTreeImpl(); + byte[] merkleTreeRoot = merkle.getMerkleRoot(new ArrayList<>(prevMilestones), MerkleOptions.getDefault()); + if (transaction.getTrunkTransactionHash().equals(HashFactory.TRANSACTION.create(merkleTreeRoot))) { //System.out.println("trunk (prev. milestones): "); if (prevMilestones.isEmpty()) { trunk.add(Hash.NULL_HASH); @@ -286,10 +290,11 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); + MerkleTree merkle = new MerkleTreeImpl(); + byte[] root = merkle.getMerkleRoot(new ArrayList<>(confirmedTips), MerkleOptions.getDefault()); //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { + if (transaction.getBranchTransactionHash().equals(HashFactory.TRANSACTION.create(root))) { //System.out.println("branch (tips): "); if (confirmedTips.isEmpty()){ branch.add(Hash.NULL_HASH); @@ -309,8 +314,8 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + byte[] merkleTreeRoot = new MerkleTreeImpl().getMerkleRoot(new ArrayList<>(prevMilestones), MerkleOptions.getDefault()); + if (transaction.getBranchTransactionHash().equals(HashFactory.TRANSACTION.create(merkleTreeRoot))) { //System.out.println("branch (prev. milestones): "); if (prevMilestones.isEmpty()) { branch.add(Hash.NULL_HASH); @@ -433,9 +438,8 @@ public Integer index() { } public Hash getMerkleRoot() { - List> merkleTree = Merkle.buildMerkleTree(new LinkedList<>(getHashes())); - Hash root = merkleTree.get(merkleTree.size()-1).get(0); - return root; + MerkleTree merkle = new MerkleTreeImpl(); + return HashFactory.TRANSACTION.create(merkle.getMerkleRoot(new LinkedList<>(getHashes()), MerkleOptions.getDefault())); } /** diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index b6bba59b..f84c5a04 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -2,6 +2,7 @@ import net.helix.pendulum.TransactionValidator; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleNode; import net.helix.pendulum.model.*; import net.helix.pendulum.model.persistables.*; import net.helix.pendulum.service.milestone.MilestoneTracker; @@ -21,7 +22,7 @@ * of trunk and branch and the hash of a transaction. * The size and offset of the transaction attributes and the supply are also defined here. */ -public class TransactionViewModel { +public class TransactionViewModel implements MerkleNode { private final Transaction transaction; @@ -425,6 +426,10 @@ public byte[] getBytes() { return transaction.bytes; } + public byte[] bytes(){ + return getBytes(); + } + public Hash getHash() { return hash; } diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java deleted file mode 100644 index d30795d6..00000000 --- a/src/main/java/net/helix/pendulum/crypto/Merkle.java +++ /dev/null @@ -1,260 +0,0 @@ -package net.helix.pendulum.crypto; - -import net.helix.pendulum.controllers.RoundViewModel; -import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.model.Hash; -import net.helix.pendulum.model.HashFactory; -import net.helix.pendulum.utils.bundle.BundleUtils; -import org.bouncycastle.util.encoders.Hex; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.*; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class Merkle { - - private static final Logger log = LoggerFactory.getLogger(Merkle.class); - - public static byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { - int index = indexIn; - final Sponge sha3 = SpongeFactory.create(mode); - for (int i = 0; i < size; i++) { - sha3.reset(); - if ((index & 1) == 0) { - sha3.absorb(hash, 0, hash.length); - sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); - } else { - sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); - sha3.absorb(hash, 0, hash.length); - } - sha3.squeeze(hash, 0, hash.length); - index >>= 1; - } - if (index != 0) { - return Hash.NULL_HASH.bytes(); - } - return hash; - } - - public static List getMerklePath(List> merkleTree, int keyIndex) { - List merklePath = new ArrayList<>((merkleTree.size() - 1) * 32); - for (int i = 0; i < merkleTree.size() - 1; i++) { - Hash subkey = merkleTree.get(i).get(keyIndex ^ 1); - merklePath.add(subkey == null ? Hash.NULL_HASH : subkey); - keyIndex /= 2; - } - return merklePath; - } - - public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security) { - List keys = new ArrayList<>(1 << pubkeyDepth); - for (int i = 0; i < pubkeyCount; i++) { - int idx = firstIndex + i; - keys.add(HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security))); - } - return buildMerkleTree(keys); - } - - - public static List buildMerkleTransactionTree(List leaves, Hash milestoneHash, Hash address) { - if (leaves.isEmpty()) { - leaves.add(Hash.NULL_HASH); - } - byte[] buffer; - Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); - int row = 1; - List virtualTransactionList = new ArrayList(); - int depth = getTreeDepth(leaves.size()); - while (leaves.size() > 1) { - List nextKeys = Arrays.asList(new Hash[(leaves.size() % 2 == 0 ? (leaves.size() / 2) : (leaves.size() / 2 + 1))]); - for (int i = 0; i < nextKeys.size(); i++) { - if (areLeavesNull(leaves, i)) continue; - sha3.reset(); - Hash k1 = getLeaves(leaves, i * 2); - Hash k2 = getLeaves(leaves, i * 2 + 1); - buffer = computeParentHash(sha3, k1, k2); - Hash parentHash = HashFactory.TRANSACTION.create(buffer); - nextKeys.set(i, parentHash); - virtualTransactionList.add(createVirtualTransaction(k1, k2, getParentMerkleIndex(row, depth, i * 2), milestoneHash, address, parentHash)); - } - leaves = nextKeys; - row++; - } - return virtualTransactionList; - } - - private static Hash getLeaves(List leaves, int index) { - return index < leaves.size() ? leaves.get(index) : Hash.NULL_HASH; - } - - private static long getParentMerkleIndex(int row, int depth, int i) { - if (row == depth) { - return 0; // root - } - long index = depth - row; - return (long) Math.pow(2, index) + i / 2 - 1; - } - - private static byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { - byte[] buffer; - buffer = copyHash(k1); - sha3.absorb(buffer, 0, buffer.length); - buffer = copyHash(k2); - sha3.absorb(buffer, 0, buffer.length); - sha3.squeeze(buffer, 0, buffer.length); - return buffer; - } - - public static List> buildMerkleTree(List leaves) { - if (leaves.isEmpty()) { - leaves.add(Hash.NULL_HASH); - } - byte[] buffer; - Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); - int depth = getTreeDepth(leaves.size()); - List> merkleTree = new ArrayList<>(depth + 1); - merkleTree.add(0, leaves); - int row = 1; - // hash two following keys together until only one is left -> merkle tree - while (leaves.size() > 1) { - // Take two following keys (i=0: (k0,k1), i=1: (k2,k3), ...) and get one crypto of them - List nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]); - for (int i = 0; i < nextKeys.size(); i++) { - if (areLeavesNull(leaves, i)) continue; - sha3.reset(); - Hash k1 = leaves.get(i * 2); - Hash k2 = leaves.get(i * 2 + 1); - buffer = computeParentHash(sha3, k1, k2); - nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); - } - leaves = nextKeys; - merkleTree.add(row++, leaves); - } - return merkleTree; - } - - private static boolean areLeavesNull(List leaves, int i) { - if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { - return true; - } - return false; - } - - private static int getTreeDepth(int leavesNumber) { - return (int) Math.ceil((float) (Math.log(leavesNumber) / Math.log(2))); - } - - private static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, Hash milestoneHash, Hash address, Hash transactionHash) { - log.debug("New virtual transaction + " + transactionHash + " for milestone: " + milestoneHash + " with merkle index: " + merkleIndex + " [" + branchHash + ", " + trunkHash + " ]"); - return BundleUtils.createVirtualTransaction(branchHash, trunkHash, merkleIndex, milestoneHash.bytes(), address); - } - - private static byte[] copyHash(Hash k2) { - return Arrays.copyOfRange(k2 == null ? Hex.decode(Hash.NULL_HASH.bytes()) : k2.bytes(), 0, Hash.SIZE_IN_BYTES); - } - - public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) { - - final TransactionViewModel merkleTx = bundleTransactionViewModels.get(securityLevel); - int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex - - //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) - //TODO: check if its okay here to use bundle hash instead of tx hash - byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); - - //validate leaf signature - ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * securityLevel); - - for (int i = 0; i < securityLevel; i++) { - byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i + 1)); - byte[] digest = Winternitz.digest(mode, bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); - bb.put(digest); - } - - byte[] digests = bb.array(); - byte[] address = Winternitz.address(mode, digests); - - //validate Merkle path - byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, - merkleTx.getSignature(), 0, keyIndex, depth); - return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); - } - - public static List> readKeyfile(File keyfile) throws IOException { - try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { - String[] fields = br.readLine().split(" "); - int depth = Integer.parseInt(fields[0]); - List> result = new ArrayList<>(depth + 1); - for (int i = 0; i <= depth; i++) { - fields = br.readLine().split(" "); - int leadingNulls = Integer.parseInt(fields[0]); - List row = new ArrayList<>(); - for (int j = 0; j < leadingNulls; j++) { - row.add(Hash.NULL_HASH); - } - for (int j = 0; j < fields[1].length() / 64; j++) { - row.add(HashFactory.TRANSACTION.create(fields[1].substring(j * 64, (j + 1) * 64))); - } - result.add(row); - } - return result; - } - } - - public static String getSeed(File keyfile) throws IOException { - StringBuilder seedBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { - String[] fields = br.readLine().split(" "); - seedBuilder.append(fields[1]); - } - return seedBuilder.toString(); - } - - - public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { - // fill buffer - try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { - // write pubkey depth and seed into buffer - bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); - bw.newLine(); - writeKeys(bw, merkleTree.get(0)); - for (int i = 1; i < merkleTree.size(); i++) { - writeKeys(bw, merkleTree.get(i)); - } - } - } - - private static void writeKeys(BufferedWriter bw, List keys) throws IOException { - int leadingNulls = 0; - while (keys.get(leadingNulls) == null) { - leadingNulls++; - } - bw.write(leadingNulls + " "); - for (int i = leadingNulls; i < keys.size(); i++) { - if (keys.get(i) == null) { - break; - } - bw.write(keys.get(i).toString()); - } - bw.newLine(); - } - - public static byte[] padding(byte[] input, int length) { - if (input.length < length) { - byte[] output = new byte[length]; - System.arraycopy(input, 0, output, length - input.length, input.length); - return output; - } else { - if (input.length > length) { - return Arrays.copyOfRange(input, 0, length); - } else { - return Arrays.copyOfRange(input, 0, input.length); - } - - } - } -} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleNode.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleNode.java new file mode 100644 index 00000000..1d6bd4fb --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleNode.java @@ -0,0 +1,9 @@ +package net.helix.pendulum.crypto.merkle; + +/** + * Used as node in merkle tree + */ +public interface MerkleNode { + + byte[] bytes(); +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java new file mode 100644 index 00000000..54e330a7 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java @@ -0,0 +1,87 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.model.Hash; + +public class MerkleOptions { + + private boolean includeLeavesInTree; + private Hash milestoneHash; + private Hash address; + private SpongeFactory.Mode mode; + private int securityLevel; + private int depth; + + /** + * Parameters needed fro validation + * @param mode + * @param address + * @param securityLevel + * @param depth + */ + public MerkleOptions(SpongeFactory.Mode mode, Hash address, int securityLevel, int depth) { + this.address = address; + this.mode = mode; + this.securityLevel = securityLevel; + this.depth = depth; + this.includeLeavesInTree = true; + } + + public MerkleOptions() { + } + + public static MerkleOptions getDefault() { + MerkleOptions op = new MerkleOptions(); + op.includeLeavesInTree = true; + op.mode = SpongeFactory.Mode.S256; + return op; + } + + public boolean isIncludeLeavesInTree() { + return includeLeavesInTree; + } + + public void setIncludeLeavesInTree(boolean includeLeavesInTree) { + this.includeLeavesInTree = includeLeavesInTree; + } + + public Hash getMilestoneHash() { + return milestoneHash; + } + + public void setMilestoneHash(Hash milestoneHash) { + this.milestoneHash = milestoneHash; + } + + public Hash getAddress() { + return address; + } + + public void setAddress(Hash address) { + this.address = address; + } + + public SpongeFactory.Mode getMode() { + return mode; + } + + public void setMode(SpongeFactory.Mode mode) { + this.mode = mode; + } + + public int getSecurityLevel() { + return securityLevel; + } + + public void setSecurityLevel(int securityLevel) { + this.securityLevel = securityLevel; + } + + public int getDepth() { + return depth; + } + + public void setDepth(int depth) { + this.depth = depth; + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java new file mode 100644 index 00000000..239a1842 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java @@ -0,0 +1,78 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.model.Hash; + +import java.util.Arrays; +import java.util.List; + +public interface MerkleTree { + + /** + * Gets merkle path from the specified key index (Used in merkle transaction message + * @param merkleTree + * @param keyIndex + * @return + */ + List getMerklePath(List> merkleTree, int keyIndex); + + /** + * Build flat merkle tree + * @param leaves hash leaves + * @param options + * @return + */ + List buildMerkle(List leaves, MerkleOptions options); + + /** + * Build merkle tree as lists of lists + * @param leaves + * @param options + * @return + */ + List> buildMerkleTree(List leaves, MerkleOptions options); + + /** + * Gets merkle tree root for given leaves + * @param leaves + * @param options + * @return + */ + byte[] getMerkleRoot(List leaves, MerkleOptions options); + + /** + * Get merkle tree root, based on merkle path + * @param mode + * @param hash + * @param bytes merkle path bytes + * @param offset + * @param indexIn + * @param size of elements in path + * @return + */ + byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, int indexIn, int size); + + /** + * Validate bundles which contains merkle transaction + * First it validates sender transactions signature + * @param bundleTransactionViewModels + * @param options + * @return + */ + boolean validateMerkleSignature(List bundleTransactionViewModels, MerkleOptions options); + + static byte[] padding(byte[] input, int length) { + if (input.length < length) { + byte[] output = new byte[length]; + System.arraycopy(input, 0, output, length - input.length, input.length); + return output; + } else { + if (input.length > length) { + return Arrays.copyOfRange(input, 0, length); + } else { + return Arrays.copyOfRange(input, 0, input.length); + } + } + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java new file mode 100644 index 00000000..13478984 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java @@ -0,0 +1,199 @@ +package net.helix.pendulum.crypto.merkle.impl; + +import net.helix.pendulum.controllers.RoundViewModel; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Sha3; +import net.helix.pendulum.crypto.Sponge; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public abstract class AbstractMerkleTree implements MerkleTree { + + protected abstract MerkleNode createMerkleNode(Hash hashParent, Hash h1, Hash h2, int row, long index, MerkleOptions options); + + protected abstract List createMerkleNodes(List leaves, MerkleOptions options); + + @Override + public byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { + int index = indexIn; + final Sponge sha3 = SpongeFactory.create(mode); + for (int i = 0; i < size; i++) { + sha3.reset(); + if ((index & 1) == 0) { + sha3.absorb(hash, 0, hash.length); + sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); + } else { + sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); + sha3.absorb(hash, 0, hash.length); + } + sha3.squeeze(hash, 0, hash.length); + index >>= 1; + } + if (index != 0) { + return getDefaultMerkleHash().bytes(); + } + return hash; + } + + @Override + public List getMerklePath(List> merkleTree, int keyIndex) { + List merklePath = new ArrayList<>((merkleTree.size() - 1) * Hash.SIZE_IN_BYTES); + for (int i = 0; i < merkleTree.size() - 1; i++) { + MerkleNode subkey = merkleTree.get(i).get(keyIndex ^ 1); + merklePath.add(subkey == null ? getDefaultMerkleHash() : subkey); + keyIndex /= 2; + } + return merklePath; + } + + @Override + public byte[] getMerkleRoot(List leaves, MerkleOptions options) { + List> merkleTree = buildMerkleTree(leaves, options); + return (merkleTree.get(merkleTree.size() - 1).get(0)).bytes(); + } + + @Override + public List buildMerkle(List leaves, MerkleOptions options) { + if (leaves.isEmpty()) { + leaves.add(getDefaultMerkleHash()); + } + byte[] buffer; + Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); + int row = 1; + List merkleNodes = new ArrayList<>(); + int depth = getTreeDepth(leaves.size()); + while (leaves.size() > 1) { + List nextKeys = Arrays.asList(new Hash[getParentNodesSize(leaves)]); + for (int i = 0; i < nextKeys.size(); i++) { + if (areLeavesNull(leaves, i)) continue; + sha3.reset(); + Hash k1 = getLeaves(leaves, i * 2); + Hash k2 = getLeaves(leaves, i * 2 + 1); + buffer = computeParentHash(sha3, k1, k2); + Hash parentHash = HashFactory.TRANSACTION.create(buffer); + nextKeys.set(i, parentHash); + merkleNodes.add(createMerkleNode(parentHash, k1, k2, row, getParentMerkleIndex(row, depth, i * 2), options)); + } + leaves = nextKeys; + row++; + } + return merkleNodes; + } + + @Override + public List> buildMerkleTree(List leaves, MerkleOptions options) { + if (leaves.isEmpty()) { + leaves.add(getDefaultMerkleHash()); + } + byte[] buffer; + Sponge sha3 = SpongeFactory.create(options.getMode()); + int depth = getTreeDepth(leaves.size()); + List> merkleTree = new ArrayList<>(); + + if (options.isIncludeLeavesInTree()) { + merkleTree.add(0, createMerkleNodes(leaves, options)); + } + int row = 1; + while (leaves.size() > 1) { + List nextKeys = Arrays.asList(new Hash[getParentNodesSize(leaves)]); + for (int i = 0; i < nextKeys.size(); i++) { + if (areLeavesNull(leaves, i)) continue; + sha3.reset(); + Hash k1 = getLeaves(leaves, i * 2); + Hash k2 = getLeaves(leaves, i * 2 + 1); + buffer = computeParentHash(sha3, k1, k2); + nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); + } + leaves = nextKeys; + merkleTree.add(row++, createMerkleNodes(leaves, options)); + } + return merkleTree; + } + + public boolean validateMerkleSignature(List bundleTransactionViewModels, MerkleOptions options) { + + final TransactionViewModel merkleTx = bundleTransactionViewModels.get(options.getSecurityLevel()); + int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex + + //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) + //TODO: check if its okay here to use bundle hash instead of tx hash + byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); + + //validate leaf signature + ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * options.getSecurityLevel()); + + for (int i = 0; i < options.getSecurityLevel(); i++) { + byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i + 1)); + byte[] digest = Winternitz.digest(options.getMode(), bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); + bb.put(digest); + } + + byte[] digests = bb.array(); + byte[] address = Winternitz.address(options.getMode(), digests); + + return validateMerklePath(merkleTx.getSignature(), keyIndex, address, options); + } + + private boolean validateMerklePath(byte[] path, int keyIndex, byte[] address, MerkleOptions options) { + byte[] merkleRoot = getMerkleRoot(options.getMode(), address, + path, 0, keyIndex, options.getDepth()); + return HashFactory.ADDRESS.create(merkleRoot).equals(options.getAddress()); + } + + protected int getTreeDepth(int leavesNumber) { + return (int) Math.ceil((float) (Math.log(leavesNumber) / Math.log(2))); + } + + protected Hash getLeaves(List leaves, int index) { + return index < leaves.size() ? leaves.get(index) : getDefaultMerkleHash(); + } + + protected static long getParentMerkleIndex(int row, int depth, int i) { + if (row == depth) { + return 0; + } + long index = depth - row; + return (long) Math.pow(2, index) + i / 2 - 1; + } + + protected int getParentNodesSize(List leaves) { + return leaves.size() % 2 == 0 ? (leaves.size() / 2) : (leaves.size() / 2 + 1); + } + + private boolean areLeavesNull(List leaves, int i) { + if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { + return true; + } + return false; + } + + private byte[] computeParentHash(Sponge sha3, byte[] k1, byte[] k2) { + byte[] buffer = new byte[Hash.SIZE_IN_BYTES]; + sha3.absorb(k1, 0, k1.length); + sha3.absorb(k2, 0, k2.length); + sha3.squeeze(buffer, 0, buffer.length); + return buffer; + } + + private byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { + return computeParentHash(sha3, copyHash(k1), copyHash(k2)); + } + + private byte[] copyHash(Hash k2) { + return Arrays.copyOfRange(k2 == null ? getDefaultMerkleHash().bytes() : k2.bytes(), 0, Hash.SIZE_IN_BYTES); + } + + private Hash getDefaultMerkleHash() { + return Hash.NULL_HASH; + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java index 9a9396e1..abb9c96b 100644 --- a/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java @@ -1,4 +1,23 @@ -package net.helix.pendulum.crypto.merkle; +package net.helix.pendulum.crypto.merkle.impl; -public class MerkleTreeImpl { +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.model.Hash; + +import java.util.ArrayList; +import java.util.List; + +public class MerkleTreeImpl extends AbstractMerkleTree { + + @Override + protected MerkleNode createMerkleNode(Hash hashParent, Hash h1, Hash h2, int row, long index, MerkleOptions options) { + return hashParent; + } + + @Override + protected List createMerkleNodes(List leaves, MerkleOptions options) { + List result = new ArrayList<>(); + result.addAll(leaves); + return result; + } } diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/TransactionMerkleTreeImpl.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/TransactionMerkleTreeImpl.java new file mode 100644 index 00000000..d39246d4 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/TransactionMerkleTreeImpl.java @@ -0,0 +1,35 @@ +package net.helix.pendulum.crypto.merkle.impl; + +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.utils.bundle.BundleUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class TransactionMerkleTreeImpl extends AbstractMerkleTree { + + private static final Logger log = LoggerFactory.getLogger(TransactionMerkleTreeImpl.class); + + @Override + protected MerkleNode createMerkleNode(Hash parentHash, Hash h1, Hash h2, int row, long index, MerkleOptions options) { + log.debug("New virtual transaction + " + parentHash + " for milestone: " + options.getMilestoneHash() + " with merkle index: " + index + " [" + h1 + ", " + h2 + " ]"); + byte[] virtualTransaction = BundleUtils.createVirtualTransaction(h1, h2, index, options.getMilestoneHash().bytes(), options.getAddress()); + return new TransactionViewModel(virtualTransaction, options.getMode()); + } + + @Override + protected List createMerkleNodes(List leaves, MerkleOptions options) { + + int depth = getTreeDepth(leaves.size()); + List result = new ArrayList<>(); + for (int i = 0; i < leaves.size(); i++) { + result.add(createMerkleNode(leaves.get(i), Hash.NULL_HASH, Hash.NULL_HASH, 0, getParentMerkleIndex(0, depth, i), options)); + } + return result; + } +} diff --git a/src/main/java/net/helix/pendulum/model/Hash.java b/src/main/java/net/helix/pendulum/model/Hash.java index 4fa078dc..e40de872 100644 --- a/src/main/java/net/helix/pendulum/model/Hash.java +++ b/src/main/java/net/helix/pendulum/model/Hash.java @@ -1,6 +1,7 @@ package net.helix.pendulum.model; import net.helix.pendulum.crypto.Sha3; +import net.helix.pendulum.crypto.merkle.MerkleNode; import net.helix.pendulum.storage.Indexable; @@ -9,7 +10,7 @@ * The model class contains a hash Hash , the size of the hash, lock * and the inner classes ByteSafe */ -public interface Hash extends Indexable, HashId { +public interface Hash extends Indexable, HashId, MerkleNode { /** * Creates a null transaction hash with from a byte array of length {@value Sha3#HASH_LENGTH}. diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 01961bac..6e8893ee 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -12,6 +12,10 @@ import net.helix.pendulum.controllers.*; import net.helix.pendulum.crypto.*; import net.helix.pendulum.crypto.Merkle; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; +import net.helix.pendulum.crypto.merkle.impl.TransactionMerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.TransactionHash; @@ -1521,17 +1525,22 @@ private void createAndBroadcastVirtualTransactions(byte[] tipsBytes, List virtualTransactions = Merkle.buildMerkleTransactionTree(tips, - TransactionHash.calculate(SpongeFactory.Mode.S256, milestone.getBytes()), milestone.getAddressHash()). - stream().map(t -> { + MerkleOptions options = MerkleOptions.getDefault(); + options.setMilestoneHash(milestone.getHash()); + options.setAddress(milestone.getAddressHash()); + + List merkleTransactions = new TransactionMerkleTreeImpl().buildMerkle(tips, options); + + List virtualTransactions = merkleTransactions.stream().map(t -> { try { - fillAttachmentTransactionFields(t, RoundIndexUtil.getCurrentTime()); - TransactionViewModel virtualTransaction = new TransactionViewModel(t, SpongeFactory.Mode.S256); - virtualTransaction.storeTransactionLocal(tangle,snapshotProvider.getInitialSnapshot(),transactionValidator); + TransactionViewModel tvm = (TransactionViewModel)t; + fillAttachmentTransactionFields(tvm.getBytes(), RoundIndexUtil.getCurrentTime()); + tvm.storeTransactionLocal(tangle,snapshotProvider.getInitialSnapshot(),transactionValidator); + return Hex.toHexString(tvm.getBytes()); } catch (Exception e) { e.printStackTrace(); } - return Hex.toHexString(t); + return null; }).collect(Collectors.toList()); broadcastTransactionsStatement(virtualTransactions); } @@ -1675,10 +1684,10 @@ private List addMilestoneReferences(List confirmedTips, int roundInd txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones } //branch - List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); - txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips + Hash merkleRoot = HashFactory.TRANSACTION.create(new MerkleTreeImpl().getMerkleRoot(confirmedTips, MerkleOptions.getDefault())); + txToApprove.add(merkleRoot); - log.debug("Milestone future branch transaction hash: " + merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); + log.debug("Milestone future branch transaction hash: " + merkleRoot); } return txToApprove; } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java index 45cb0c26..9763e551 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java @@ -2,11 +2,13 @@ import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.crypto.Merkle; +import net.helix.pendulum.crypto.merkle.MerkleNode; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.service.API; import net.helix.pendulum.service.utils.RoundIndexUtil; import net.helix.pendulum.service.validatomanager.CandidateTracker; +import net.helix.pendulum.utils.KeyfileUtil; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,8 +75,8 @@ private void initSeed(PendulumConfig configuration) { } private void writeKeyIndex() throws IOException { - List> merkleTree = Merkle.readKeyfile(new File(keyfile)); - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); + List> merkleTree = KeyfileUtil.readKeyfile(new File(keyfile)); + KeyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); } private void readKeyfileMetadata() throws IOException { @@ -87,7 +89,7 @@ private void readKeyfileMetadata() throws IOException { seed = fields[1]; } } - List> merkleTree = Merkle.readKeyfile(new File(keyfile)); + List> merkleTree = KeyfileUtil.readKeyfile(new File(keyfile)); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size() - 1).get(0).bytes()); } @@ -106,7 +108,7 @@ private void doKeyChange() throws Exception { // generate new keyfile int newKeyfileIndex = keyfileIndex + 1; log.debug("Generating Keyfile (idx: " + newKeyfileIndex + ")"); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getValidatorSecurity()); + List> merkleTree = KeyfileUtil.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getValidatorSecurity()); Hash newAddress = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); // send keyChange bundle to register new address api.publishKeyChange(address.toString(), newAddress, mwm, sign, currentKeyIndex, maxKeyIndex); @@ -114,13 +116,13 @@ private void doKeyChange() throws Exception { keyfileIndex = newKeyfileIndex; address = newAddress; currentKeyIndex = maxKeyIndex * keyfileIndex; - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + KeyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); } private void generateKeyfile(String seed) throws Exception { log.debug("Generating Keyfile (idx: " + keyfileIndex + ")"); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getValidatorSecurity()); - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + List> merkleTree = KeyfileUtil.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getValidatorSecurity()); + KeyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java index b3ffd128..0c8ba294 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java @@ -8,6 +8,8 @@ import net.helix.pendulum.controllers.TransactionViewModel; import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.IntegerIndex; import net.helix.pendulum.model.StateDiff; @@ -183,7 +185,8 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); + boolean validSignature = new MerkleTreeImpl().validateMerkleSignature(bundleTransactionViewModels, + new MerkleOptions(mode, senderAddress, securityLevel, config.getMilestoneKeyDepth())); //System.out.println("valid signature: " + validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || diff --git a/src/main/java/net/helix/pendulum/service/validatomanager/impl/ValidatorManagerServiceImpl.java b/src/main/java/net/helix/pendulum/service/validatomanager/impl/ValidatorManagerServiceImpl.java index 78362663..d479b8b6 100644 --- a/src/main/java/net/helix/pendulum/service/validatomanager/impl/ValidatorManagerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatomanager/impl/ValidatorManagerServiceImpl.java @@ -5,6 +5,8 @@ import net.helix.pendulum.controllers.TransactionViewModel; import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; @@ -74,7 +76,9 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM if (tail.getHash().equals(transactionViewModel.getHash()) && isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); + + boolean validSignature = new MerkleTreeImpl().validateMerkleSignature(bundleTransactionViewModels, + new MerkleOptions(mode, senderAddress, securityLevel, config.getMilestoneKeyDepth())); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validator.contains(senderAddress)) && validSignature) { return VALID; diff --git a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java index 36d5fa66..3f17ff92 100644 --- a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java @@ -5,6 +5,8 @@ import net.helix.pendulum.controllers.TransactionViewModel; import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; @@ -58,7 +60,8 @@ public ValidatorValidity validateValidators(TransactionViewModel transactionView // validate signature Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getValidatorManagerKeyDepth()); + boolean validSignature = new MerkleTreeImpl().validateMerkleSignature(bundleTransactionViewModels, + new MerkleOptions(mode, senderAddress, securityLevel, config.getMilestoneKeyDepth())); //System.out.println("valid signature (validator): " + validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getValidatorManagerAddress().equals(senderAddress) && validSignature)) { diff --git a/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java b/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java new file mode 100644 index 00000000..4ced4645 --- /dev/null +++ b/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java @@ -0,0 +1,98 @@ +package net.helix.pendulum.utils; + +import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; +import org.bouncycastle.util.encoders.Hex; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +/** + * It contains the required methods to manage key files (read, write, build) + */ +public class KeyfileUtil { + + public static String getSeed(File keyfile) throws IOException { + StringBuilder seedBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + seedBuilder.append(fields[1]); + } + return seedBuilder.toString(); + } + + public static byte[] getKeyTreeRoot(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security) { + List> tree = buildMerkleKeyTree(seed, pubkeyDepth,firstIndex,pubkeyCount,security); + return tree.get(tree.size()-1).get(0).bytes(); + + } + public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security) { + List keys = new ArrayList<>(1 << pubkeyDepth); + for (int i = 0; i < pubkeyCount; i++) { + int idx = firstIndex + i; + keys.add(createKeyHash(seed, security, idx)); + } + return (new MerkleTreeImpl()).buildMerkleTree(keys, MerkleOptions.getDefault()); + } + + public static void writeKeys(BufferedWriter bw, List keys) throws IOException { + int leadingNulls = 0; + while (keys.get(leadingNulls) == null) { + leadingNulls++; + } + bw.write(leadingNulls + " "); + for (int i = leadingNulls; i < keys.size(); i++) { + if (keys.get(i) == null) { + break; + } + bw.write(writeMerkleNode(keys.get(i))); + } + bw.newLine(); + } + + public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { + try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { + bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); + bw.newLine(); + writeKeys(bw, merkleTree.get(0)); + for (int i = 1; i < merkleTree.size(); i++) { + writeKeys(bw, merkleTree.get(i)); + } + } + } + + public static List> readKeyfile(File keyfile) throws IOException { + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + int depth = Integer.parseInt(fields[0]); + List> result = new ArrayList<>(depth + 1); + for (int i = 0; i <= depth; i++) { + fields = br.readLine().split(" "); + int leadingNulls = Integer.parseInt(fields[0]); + List row = new ArrayList<>(); + for (int j = 0; j < leadingNulls; j++) { + row.add(Hash.NULL_HASH); + } + for (int j = 0; j < fields[1].length() / 64; j++) { + row.add(HashFactory.ADDRESS.create(fields[1].substring(j * 64, (j + 1) * 64))); + } + result.add(row); + } + return result; + } + } + + private static Hash createKeyHash(String seed, int security, int idx) { + return HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security)); + } + + private static String writeMerkleNode(MerkleNode key) { + return key.toString(); + } +} diff --git a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java index 929737d7..a42cbf60 100644 --- a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java @@ -5,8 +5,11 @@ import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.utils.RoundIndexUtil; +import net.helix.pendulum.utils.KeyfileUtil; import net.helix.pendulum.utils.Serializer; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -163,13 +166,13 @@ private byte[] addBundleHash(List bundle, SpongeFactory.Mode mode) { private void signBundle(String filepath, byte[] merkleTransaction, List senderTransactions, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { // Get merkle path and store in signatureMessageFragment of Sibling Transaction File keyfile = new File(filepath); - List> merkleTree = Merkle.readKeyfile(keyfile); - String seed = Merkle.getSeed(keyfile); + List> merkleTree = KeyfileUtil.readKeyfile(keyfile); + String seed = KeyfileUtil.getSeed(keyfile); int security = senderTransactions.size(); // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); - byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); + List merklePath = new MerkleTreeImpl().getMerklePath(merkleTree, keyIndex % maxKeyIndex); + byte[] path = Hex.decode(merklePath.stream().map(m -> m.toString()).collect(Collectors.joining())); System.arraycopy(path, 0, merkleTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); // sign bundle hash and store signature in Milestone Transaction diff --git a/src/test/java/net/helix/pendulum/crypto/merkle/MerkleTreeTest.java b/src/test/java/net/helix/pendulum/crypto/merkle/MerkleTreeTest.java new file mode 100644 index 00000000..83a9fec8 --- /dev/null +++ b/src/test/java/net/helix/pendulum/crypto/merkle/MerkleTreeTest.java @@ -0,0 +1,58 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.TransactionTestUtils; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; +import net.helix.pendulum.model.TransactionHash; +import net.helix.pendulum.utils.KeyfileUtil; +import org.bouncycastle.util.encoders.Hex; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class MerkleTreeTest { + + @Test + public void createMerkleTreeAndValidateIt() { + int keyDepth = 5; + int firstIndex = 0; + int security = 2; + int keyIndex = 0; + int maxKeyIndex = (int) Math.pow(2, keyDepth); + String seed = "aabbccdd00000000000000000000000000000000000000000000000000000000"; + + byte[] root = KeyfileUtil.getKeyTreeRoot(seed, keyDepth, firstIndex, maxKeyIndex, security); + + Hash senderAddress = HashFactory.ADDRESS.create(root); + MerkleTree merkle = new MerkleTreeImpl(); + MerkleOptions options = new MerkleOptions(SpongeFactory.Mode.S256, senderAddress, security, keyDepth); + + List> merkleTree = KeyfileUtil.buildMerkleKeyTree(seed, keyDepth, maxKeyIndex * keyIndex, maxKeyIndex, security); + + Assert.assertTrue(senderAddress.equals(HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size() - 1).get(0).bytes()))); + + List merklePath = new MerkleTreeImpl().getMerklePath(merkleTree, keyIndex % maxKeyIndex); + byte[] path = Hex.decode(merklePath.stream().map(m -> m.toString()).collect(Collectors.joining())); + byte[] merkleTransactionSignature = new byte[TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; + + System.arraycopy(path, 0, merkleTransactionSignature, 0, path.length); + byte[] merkleRootTest = merkle.getMerkleRoot(options.getMode(), senderAddress.bytes(), + merkleTransactionSignature, 0, keyIndex, options.getDepth()); + + Assert.assertTrue(HashFactory.ADDRESS.create(merkleRootTest).equals(options.getAddress())); + } + + private List getHashes(int noOfLeaves) { + List transactions = new ArrayList(); + for (int i = 0; i < noOfLeaves; i++) { + transactions.add(TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes())); + } + return transactions; + } +} diff --git a/src/test/java/net/helix/pendulum/crypto/MerkleTreeTest.java b/src/test/java/net/helix/pendulum/crypto/merkle/TransactionMerkleTreeTest.java similarity index 68% rename from src/test/java/net/helix/pendulum/crypto/MerkleTreeTest.java rename to src/test/java/net/helix/pendulum/crypto/merkle/TransactionMerkleTreeTest.java index e47b5e7f..d24329e6 100644 --- a/src/test/java/net/helix/pendulum/crypto/MerkleTreeTest.java +++ b/src/test/java/net/helix/pendulum/crypto/merkle/TransactionMerkleTreeTest.java @@ -1,7 +1,9 @@ -package net.helix.pendulum.crypto; +package net.helix.pendulum.crypto.merkle; import net.helix.pendulum.TransactionTestUtils; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.impl.TransactionMerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.TransactionHash; import org.junit.Assert; @@ -12,9 +14,9 @@ import java.util.*; -public class MerkleTest { +public class TransactionMerkleTreeTest { - private static final Logger log = LoggerFactory.getLogger(MerkleTest.class); + private static final Logger log = LoggerFactory.getLogger(TransactionMerkleTreeTest.class); @Test public void testTreeTransactionGeneration() { @@ -28,13 +30,15 @@ private void checkMerkleVirtualIndexes(int noOfLeaves, Set expectedIndexes for (int i = 0; i < noOfLeaves; i++) { transactions.add(TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes())); } - - List virtualTransactions = Merkle.buildMerkleTransactionTree(transactions, milestoneHash, Hash.NULL_HASH); + MerkleOptions options = MerkleOptions.getDefault(); + options.setMilestoneHash(milestoneHash); + options.setAddress(Hash.NULL_HASH); + List virtualTransactions = new TransactionMerkleTreeImpl().buildMerkle(transactions, options); Assert.assertTrue(virtualTransactions.size() > 0); Set merkleIndexes = new HashSet(); virtualTransactions.forEach(t -> { try { - TransactionViewModel virtualTransaction = new TransactionViewModel(t, SpongeFactory.Mode.S256); + TransactionViewModel virtualTransaction = (TransactionViewModel)t; Assert.assertTrue(virtualTransaction.isVirtual()); merkleIndexes.add(virtualTransaction.getTagLongValue()); From 588dd0f325df508f4ce4ffad871bb773fe108c50 Mon Sep 17 00:00:00 2001 From: Cristina Date: Thu, 17 Oct 2019 11:16:57 +0300 Subject: [PATCH 05/14] Removed unused imports --- src/main/java/net/helix/pendulum/SignedFiles.java | 8 +------- .../net/helix/pendulum/controllers/RoundViewModel.java | 10 +--------- src/main/java/net/helix/pendulum/service/API.java | 7 ++++--- .../service/milestone/impl/MilestonePublisher.java | 1 - .../service/milestone/impl/MilestoneServiceImpl.java | 1 - .../impl/ValidatorManagerServiceImpl.java | 1 - .../service/validator/impl/ValidatorServiceImpl.java | 1 - .../net/helix/pendulum/utils/bundle/BundleUtils.java | 1 - 8 files changed, 6 insertions(+), 24 deletions(-) diff --git a/src/main/java/net/helix/pendulum/SignedFiles.java b/src/main/java/net/helix/pendulum/SignedFiles.java index 692d7e4f..0a4c96b0 100644 --- a/src/main/java/net/helix/pendulum/SignedFiles.java +++ b/src/main/java/net/helix/pendulum/SignedFiles.java @@ -1,6 +1,5 @@ package net.helix.pendulum; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.Sha3; import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; @@ -11,12 +10,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.bouncycastle.util.encoders.Hex; -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UncheckedIOException; +import java.io.*; import java.util.Arrays; public class SignedFiles { diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index beafce77..8d215891 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -1,7 +1,6 @@ package net.helix.pendulum.controllers; import net.helix.pendulum.TransactionValidator; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.crypto.merkle.MerkleTree; import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; @@ -16,14 +15,7 @@ import net.helix.pendulum.utils.Pair; import net.helix.pendulum.utils.Serializer; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; +import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 6e8893ee..65d1eef2 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -10,15 +10,16 @@ import net.helix.pendulum.conf.APIConfig; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.*; -import net.helix.pendulum.crypto.*; -import net.helix.pendulum.crypto.Merkle; +import net.helix.pendulum.crypto.GreedyMiner; +import net.helix.pendulum.crypto.Sha3; +import net.helix.pendulum.crypto.Sponge; +import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.merkle.MerkleNode; import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.crypto.merkle.impl.TransactionMerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; -import net.helix.pendulum.model.TransactionHash; import net.helix.pendulum.model.persistables.Transaction; import net.helix.pendulum.network.Neighbor; import net.helix.pendulum.network.Node; diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java index 9763e551..527fcad1 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java @@ -1,7 +1,6 @@ package net.helix.pendulum.service.milestone.impl; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.merkle.MerkleNode; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java index 0c8ba294..a927bf92 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java @@ -6,7 +6,6 @@ import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.controllers.RoundViewModel; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; diff --git a/src/main/java/net/helix/pendulum/service/validatomanager/impl/ValidatorManagerServiceImpl.java b/src/main/java/net/helix/pendulum/service/validatomanager/impl/ValidatorManagerServiceImpl.java index d479b8b6..262e794b 100644 --- a/src/main/java/net/helix/pendulum/service/validatomanager/impl/ValidatorManagerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatomanager/impl/ValidatorManagerServiceImpl.java @@ -3,7 +3,6 @@ import net.helix.pendulum.BundleValidator; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; diff --git a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java index 3f17ff92..205dc0bf 100644 --- a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java @@ -3,7 +3,6 @@ import net.helix.pendulum.BundleValidator; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; diff --git a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java index a42cbf60..642c1f44 100644 --- a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java @@ -1,7 +1,6 @@ package net.helix.pendulum.utils.bundle; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; From 434ebd2c22e0401080db3156bef16fceaafb9238 Mon Sep 17 00:00:00 2001 From: Cristina Date: Wed, 30 Oct 2019 17:17:49 +0200 Subject: [PATCH 06/14] Filter milestone and removed them from round --- .../pendulum/conf/BasePendulumConfig.java | 8 ++ .../helix/pendulum/conf/ConsensusConfig.java | 12 ++ .../pendulum/controllers/RoundViewModel.java | 94 ++++++++++++++- .../controllers/TransactionViewModel.java | 22 +++- .../pendulum/model/persistables/Round.java | 8 +- .../java/net/helix/pendulum/service/API.java | 26 ++-- .../service/ledger/LedgerService.java | 6 + .../ledger/impl/LedgerServiceImpl.java | 111 +++++++++++++----- .../milestone/impl/MilestoneServiceImpl.java | 9 +- .../milestone/impl/MilestoneTrackerImpl.java | 13 ++ 10 files changed, 258 insertions(+), 51 deletions(-) diff --git a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java index a7a5f2e5..cdc2a940 100644 --- a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java +++ b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java @@ -937,6 +937,11 @@ protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; } + @Override + public double getConfirmationQuorumPercentage() { + return Defaults.CONFIRMATION_QUORUM_PERCENTAGE; + } + public interface Defaults { //API int API_PORT = 8085; @@ -1059,5 +1064,8 @@ public interface Defaults { boolean SAVELOG_ENABLED = false; String SAVELOG_BASE_PATH = "./logs/"; String SAVELOG_XML_FILE = "/logback-save.xml"; + + //Consensus + double CONFIRMATION_QUORUM_PERCENTAGE = 2.0 / 3.0; } } diff --git a/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java b/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java index 1626e3d2..c38d63eb 100644 --- a/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java @@ -7,4 +7,16 @@ * the current code base and will be changed in the future */ public interface ConsensusConfig extends SnapshotConfig, MilestoneConfig, ValidatorManagerConfig { + + /** + * @return {@value PoWConfig.Descriptions#POW_THREADS} + */ + double getConfirmationQuorumPercentage(); + + /** + * Field descriptions + */ + interface Descriptions { + String CONFIRMATION_QUORUM_PERCENTAGE = "Confirmation quorum percentage"; + } } diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 6f0b3858..46ec14fe 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -1,6 +1,10 @@ package net.helix.pendulum.controllers; import net.helix.pendulum.TransactionValidator; +import net.helix.pendulum.conf.Config; +import net.helix.pendulum.conf.PendulumConfig; +import net.helix.pendulum.crypto.Sponge; +import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.crypto.merkle.MerkleTree; import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; @@ -9,7 +13,12 @@ import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.IntegerIndex; import net.helix.pendulum.model.persistables.Round; +import net.helix.pendulum.service.ledger.LedgerException; +import net.helix.pendulum.service.ledger.LedgerService; import net.helix.pendulum.service.milestone.MilestoneTracker; +import net.helix.pendulum.service.snapshot.Snapshot; +import net.helix.pendulum.service.snapshot.SnapshotException; +import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.storage.Indexable; import net.helix.pendulum.storage.Persistable; import net.helix.pendulum.storage.Tangle; @@ -18,6 +27,7 @@ import net.helix.pendulum.utils.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.bouncycastle.util.encoders.Hex; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -411,17 +421,17 @@ public Set getReferencedTransactions(Tangle tangle, Set tips) throws // we can add the tx to confirmed transactions, because it is a parent of confirmedTips transactions.add(hashPointer); // traverse parents and add new candidates to queue - if(!seenTransactions.contains(transaction.getTrunkTransactionHash())){ + if (!seenTransactions.contains(transaction.getTrunkTransactionHash())) { seenTransactions.add(transaction.getTrunkTransactionHash()); nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); } - if(!seenTransactions.contains(transaction.getBranchTransactionHash())){ + if (!seenTransactions.contains(transaction.getBranchTransactionHash())) { seenTransactions.add(transaction.getBranchTransactionHash()); nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); } - // roundIndex already set, i.e. tx is already confirmed. + // roundIndex already set, i.e. tx is already confirmed. } else { continue; } @@ -429,6 +439,43 @@ public Set getReferencedTransactions(Tangle tangle, Set tips) throws return transactions; } + /** + * Returns a maps of milesteons with a set of confirmed tips for each milestone + * @param tangle + * @param config Pendulum config + * @return maps of milestones + * @throws Exception + */ + public Map> getConfirmedTipsPerMilestone(Tangle tangle, PendulumConfig config) throws Exception { + + Map occurrences = new HashMap<>(); + Map> milestoneHashes = new HashMap<>(); + + int quorum = (int)(config.getConfirmationQuorumPercentage() * BasePendulumConfig.Defaults.NUMBER_OF_ACTIVE_VALIDATORS); + + for (Hash milestoneHash : getHashes()) { + Set tips = getTipSet(tangle, milestoneHash, config.getValidatorSecurity()); + + for (Hash tip : tips) { + if (occurrences.containsKey(tip)) { + occurrences.put(tip, occurrences.get(tip) + 1); + } else { + occurrences.put(tip, 1); + } + } + milestoneHashes.put(milestoneHash, tips); + } + Set tips = occurrences.entrySet().stream() + .filter(entry -> entry.getValue() >= quorum) + .map(entry -> entry.getKey()) + .collect(Collectors.toSet()); + + milestoneHashes.keySet().forEach(k -> { + milestoneHashes.get(k).stream().filter(h -> tips.contains(h)); + }); + return milestoneHashes; + } + public Hash getRandomMilestone(Tangle tangle) throws Exception { Set confirmingMilestones = getHashes(); // todo getConfirmingMilestones(tangle); if (!confirmingMilestones.isEmpty()) { @@ -475,6 +522,11 @@ public boolean addMilestone(Hash milestoneHash) { return getHashes().add(milestoneHash); } + public boolean removeMilestone(Hash milestoneHash) { + round.inconsistentMilestones.add(milestoneHash); + return getHashes().remove(milestoneHash); + } + public void update(Tangle tangle) throws Exception { tangle.update(round, round.index, "round"); } @@ -489,11 +541,47 @@ public Integer index() { return round.index.getValue(); } + /** + * Return merkle root for accepted milestone from this round. + * + * @return hash of the merkle root. + */ public Hash getMerkleRoot() { MerkleTree merkle = new MerkleTreeImpl(); return HashFactory.TRANSACTION.create(merkle.getMerkleRoot(new LinkedList<>(getHashes()), MerkleOptions.getDefault())); } + public Hash getMerkleRoot(Tangle tangle, SnapshotProvider snapshotProvider, LedgerService ledgerService) { + List nonConflictualMilestones = getMilestonesToApprove(tangle, snapshotProvider, ledgerService); + return HashFactory.TRANSACTION.create( new MerkleTreeImpl().getMerkleRoot(new LinkedList<>(nonConflictualMilestones), MerkleOptions.getDefault())); + } + + public List getMilestonesToApprove(Tangle tangle, SnapshotProvider snapshotProvider, LedgerService ledgerService) { + + List targetRoundMilestones = TransactionViewModel.fromHashes(getHashes(), tangle); + Snapshot snapshot = snapshotProvider.getInitialSnapshot(); + + if (snapshot.isConsistent()) { + try { + Set approvedMilestones = new HashSet(); + ledgerService.getSnapshotService().replayMilestones(snapshot, index() - 1); + ledgerService.applyRoundToLedger(this, snapshot, approvedMilestones, true); + targetRoundMilestones.stream().filter(m -> !approvedMilestones.contains(m)).forEach(m -> { + removeMilestone(m.getHash()); + }); + update(tangle); + } catch (LedgerException e) { + log.error("Error during appling milestone states simultation", e); + } catch (SnapshotException e) { + log.error("Error during appling milestone states simultation", e); + } catch (Exception e) { + log.error("Error during appling milestone states simultation", e); + } + } + + return new ArrayList<>(getHashes()); + } + /** * Removes the {@link Round} object from the database. * diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index 2279cdae..6fd2e1da 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -13,8 +13,11 @@ import net.helix.pendulum.utils.Converter; import net.helix.pendulum.utils.Pair; import net.helix.pendulum.utils.Serializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; /** @@ -24,6 +27,7 @@ */ public class TransactionViewModel implements MerkleNode { + private static final Logger log = LoggerFactory.getLogger(TransactionViewModel.class); private final Transaction transaction; public static final int SIZE = 768; @@ -205,9 +209,9 @@ public void update(Tangle tangle, Snapshot initialSnapshot, String item) throws getBundleNonceHash(); setAttachmentData(); setMetadata(); - if (initialSnapshot.hasSolidEntryPoint(hash)) { - return; - } +// if (initialSnapshot.hasSolidEntryPoint(hash)) { +// return; +// } tangle.update(transaction, hash, item); } @@ -809,6 +813,18 @@ public String getSender() { return transaction.sender; } + public static List fromHashes(Set hashes, Tangle tangle) { + return hashes.stream().map( + h -> { + try { + return TransactionViewModel.fromHash(tangle, h); + } catch (Exception e) { + log.error("Could not get transaction for hash " + h, e); + } + return null; + }).filter(t -> t != null).collect(Collectors.toList()); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/net/helix/pendulum/model/persistables/Round.java b/src/main/java/net/helix/pendulum/model/persistables/Round.java index 84ee065f..d4d17fb0 100644 --- a/src/main/java/net/helix/pendulum/model/persistables/Round.java +++ b/src/main/java/net/helix/pendulum/model/persistables/Round.java @@ -1,15 +1,21 @@ package net.helix.pendulum.model.persistables; +import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.IntegerIndex; import net.helix.pendulum.utils.Serializer; import org.apache.commons.lang3.ArrayUtils; - /** +import java.util.LinkedHashSet; +import java.util.Set; + +/** * The Round model class consists of a set of milestone hashes and a corresponding index. */ public class Round extends Hashes { public IntegerIndex index; + public Set inconsistentMilestones = new LinkedHashSet<>(); + @Override public byte[] bytes() { return ArrayUtils.addAll(index.bytes(), super.bytes()); diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index c94f9e76..62883a76 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -411,7 +411,8 @@ private AbstractResponse checkConsistencyStatement(List transactionsList * @return false if we received at least a solid milestone, otherwise true */ public boolean invalidSubtangleStatus() { - return (snapshotProvider.getLatestSnapshot().getIndex() == snapshotProvider.getInitialSnapshot().getIndex()); + return (snapshotProvider.getLatestSnapshot().getIndex() == 0 && + snapshotProvider.getLatestSnapshot().getIndex() == snapshotProvider.getInitialSnapshot().getIndex()); } /** * Returns the set of neighbors you are connected with, as well as their activity statistics (or counters). @@ -1133,7 +1134,8 @@ private AbstractResponse getBalancesStatement(List addresses, final int index = snapshotProvider.getLatestSnapshot().getIndex(); if (tips == null || tips.size() == 0) { - hashes = new LinkedList<>(RoundViewModel.get(tangle, index).getHashes()); + RoundViewModel round = RoundViewModel.get(tangle, index); + hashes = round != null ? new LinkedList<>(round.getHashes()) : new ArrayList<>(); } else { hashes = tips.stream() .map(tip -> (HashFactory.TRANSACTION.create(tip))) @@ -1747,17 +1749,22 @@ private List getConfirmedTips() throws Exception { snapshotProvider.getLatestSnapshot().lockRead(); try { WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); - for (Hash transaction : tipsViewModel.getTips()) { - TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); + + List transactionViewModelStream = TransactionViewModel.fromHashes(tipsViewModel.getTips(), tangle); + + transactionViewModelStream.sort( + Comparator.comparing((TransactionViewModel m) -> m.getTimestamp())); + + for (TransactionViewModel txVM : transactionViewModelStream) { if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && txVM.getCurrentIndex() == 0 && txVM.isSolid() && BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { - if (walkValidator.isValid(transaction)) { - confirmedTips.add(transaction); + if (walkValidator.isValid(txVM.getHash())) { + confirmedTips.add(txVM.getHash()); } else if(txVM.isSolid()){ - log.warn("Inconsistent transaction has been removed from tips: " + transaction.toString()); - tipsViewModel.removeTipHash(transaction); + log.warn("Inconsistent transaction has been removed from tips: {} ", txVM.getHash()); + tipsViewModel.removeTipHash(txVM.getHash()); } } } @@ -1781,8 +1788,9 @@ private List addMilestoneReferences(List confirmedTips, int roundInd if (previousRound == null){ txToApprove.add(Hash.NULL_HASH); } else { - txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones + txToApprove.add(previousRound.getMerkleRoot(tangle, snapshotProvider, ledgerService)); // merkle root of latest milestones } + //branch Hash merkleRoot = HashFactory.TRANSACTION.create(new MerkleTreeImpl().getMerkleRoot(confirmedTips, MerkleOptions.getDefault())); txToApprove.add(merkleRoot); diff --git a/src/main/java/net/helix/pendulum/service/ledger/LedgerService.java b/src/main/java/net/helix/pendulum/service/ledger/LedgerService.java index 754d1f00..5135ce29 100644 --- a/src/main/java/net/helix/pendulum/service/ledger/LedgerService.java +++ b/src/main/java/net/helix/pendulum/service/ledger/LedgerService.java @@ -2,6 +2,8 @@ import net.helix.pendulum.controllers.RoundViewModel; import net.helix.pendulum.model.Hash; +import net.helix.pendulum.service.snapshot.Snapshot; +import net.helix.pendulum.service.snapshot.SnapshotService; import java.util.List; import java.util.Map; @@ -42,6 +44,8 @@ public interface LedgerService { */ boolean applyRoundToLedger(RoundViewModel round) throws LedgerException; + boolean applyRoundToLedger(RoundViewModel round, Snapshot snaphshot, Set nonConflictualMilestones, boolean simulation) throws LedgerException; + /** * Checks the consistency of the combined balance changes of the given tips.
    *
    @@ -92,4 +96,6 @@ Map generateBalanceDiff(Set visitedTransactions, Set sta throws LedgerException; boolean updateDiff(Set approvedHashes, final Map diff, Hash tip); // temporary + + SnapshotService getSnapshotService(); } diff --git a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java index 6c687092..7de10a4b 100644 --- a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java @@ -5,15 +5,19 @@ import net.helix.pendulum.controllers.RoundViewModel; import net.helix.pendulum.controllers.StateDiffViewModel; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Sponge; +import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.ledger.LedgerException; import net.helix.pendulum.service.ledger.LedgerService; import net.helix.pendulum.service.milestone.MilestoneService; +import net.helix.pendulum.service.snapshot.Snapshot; import net.helix.pendulum.service.snapshot.SnapshotException; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; import net.helix.pendulum.service.snapshot.impl.SnapshotStateDiffImpl; import net.helix.pendulum.storage.Tangle; +import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -100,9 +104,14 @@ public void restoreLedgerState() throws LedgerException { @Override public boolean applyRoundToLedger(RoundViewModel round) throws LedgerException { - if(generateStateDiff(round)) { + return applyRoundToLedger(round, snapshotProvider.getLatestSnapshot(), new HashSet<>(), false); + } + + @Override + public boolean applyRoundToLedger(RoundViewModel round, Snapshot snapshot, Set nonConflictualMilestones, boolean simulation) throws LedgerException { + if(generateStateDiff(round, nonConflictualMilestones, simulation)) { try { - snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), round.index()); + snapshotService.replayMilestones(snapshot, round.index()); } catch (SnapshotException e) { throw new LedgerException("failed to apply the balance changes to the ledger state", e); } @@ -266,49 +275,89 @@ public Map generateBalanceDiff(Set visitedTransactions, Set successfullyProcessedMilestones, boolean simulation) throws LedgerException { try { - /*TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, round.getHash()); - - if (!transactionViewModel.isSolid()) { - return false; + if (round.getHashes().isEmpty()) { + return true; } - - final int transactionSnapshotIndex = transactionViewModel.snapshotIndex(); - boolean successfullyProcessed = transactionSnapshotIndex == round.index(); - if (!successfullyProcessed) { - // if the snapshotIndex of our transaction was set already, we have processed our milestones in - // the wrong order (i.e. while rescanning the db) - if (transactionSnapshotIndex != 0) { - milestoneService.resetCorruptedRound(round.index()); - }*/ - snapshotProvider.getLatestSnapshot().lockRead(); - boolean successfullyProcessed; + + boolean successfullyProcessed = true; try { - Set confirmedTips = milestoneService.getConfirmedTips(round.index()); - //todo remove: System.out.println("round.index(): " + round.index() + ", " + confirmedTips); - Map balanceChanges = generateBalanceDiff(new HashSet<>(), confirmedTips == null? new HashSet<>() : confirmedTips, - snapshotProvider.getLatestSnapshot().getIndex() + 1); - successfullyProcessed = balanceChanges != null; - if (successfullyProcessed) { - successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( - new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); - if (successfullyProcessed) { - milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); + Set processedTips = new HashSet<>(); + Map totalBalanceChanges = new HashMap<>(); + + Map> confirmedTips = round.getConfirmedTipsPerMilestone(tangle, config); + List targetRoundMilestones = getSortedMilestones(round, confirmedTips); + Snapshot localSnapshot = snapshotProvider.getLatestSnapshot(); + + for (int i = 0; i < targetRoundMilestones.size(); i++) { + + Hash milestone = targetRoundMilestones.get(i).getHash(); + Set newTipsFromMilestone = confirmedTips.get(milestone).stream().filter(h -> + !processedTips.contains(h)).collect(Collectors.toSet()); + + if (newTipsFromMilestone.isEmpty()) { + successfullyProcessedMilestones.add(milestone); + continue; + } + + Set testTransactions = (new HashSet<>(processedTips)); + testTransactions.addAll(newTipsFromMilestone); + + Map balanceChanges = generateBalanceDiff(new HashSet<>(), testTransactions, localSnapshot.getIndex() + 1); + + if (balanceChanges != null) { + boolean localProcessed = localSnapshot.patchedState(new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); + if (localProcessed) { + processedTips.addAll(newTipsFromMilestone); + successfullyProcessedMilestones.add(milestone); if (!balanceChanges.isEmpty()) { - new StateDiffViewModel(balanceChanges, round.index()).store(tangle); + totalBalanceChanges.putAll(balanceChanges); } } + successfullyProcessed = successfullyProcessed && localProcessed; } - } finally { - snapshotProvider.getLatestSnapshot().unlockRead(); } + if (!simulation && !processedTips.isEmpty()) { + milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); + if (!totalBalanceChanges.isEmpty()) { + new StateDiffViewModel(totalBalanceChanges, round.index()).store(tangle); + } + } + } finally { + snapshotProvider.getLatestSnapshot().unlockRead(); + } + return successfullyProcessed; } catch (Exception e) { throw new LedgerException("unexpected error while generating the StateDiff for Round" + round.index(), e); } } + + private List getSortedMilestones(RoundViewModel round, Map> confirmedTips) { + Set milestoneHashes = confirmedTips.keySet(); + + List targetRoundMilestones = TransactionViewModel.fromHashes(milestoneHashes, tangle); + Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); + + targetRoundMilestones.sort( + Comparator.comparing((TransactionViewModel m) -> Hex.toHexString(getHash(sponge, + (m.getSignature().toString() + round.index()).getBytes())))); + return targetRoundMilestones; + } + + public SnapshotService getSnapshotService(){ + return this.snapshotService; + } + + private byte[] getHash(Sponge sponge, byte[] data) { + byte[] result = new byte[Sponge.HASH_LENGTH]; + sponge.reset(); + sponge.absorb(data, 0, data.length); + sponge.squeeze(result, 0, Sponge.HASH_LENGTH); + return result; + } } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java index 3425150d..b5afb3b6 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java @@ -360,18 +360,19 @@ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { * @throws MilestoneException if anything unexpected happens while updating the milestone index * @param processedTransactions a set of transactions that have been processed already (for the recursive calls) */ - private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, + public void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, Set processedTransactions) throws MilestoneException { Set inconsistentMilestones = new HashSet<>(); try { // update milestones RoundViewModel round = RoundViewModel.get(tangle, newIndex); - if(round != null) { - for (Hash milestoneHash : round.getHashes()) { + if(round == null) { + return; + } + for (Hash milestoneHash : round.getHashes()) { TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); updateRoundIndexOfSingleTransaction(milestoneTx, newIndex); - } } // update confirmed transactions final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex)); diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java index d3267a62..3f6e7b3e 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java @@ -181,7 +181,16 @@ public void addMilestoneToRoundLog(Hash milestoneHash, int roundIndex, int numbe tangle.publish("lmi %s %d", milestoneHash, roundIndex); // todo: temporarily log hardcoded number of _active_ validators instead of numberOfValidators log.delegate().debug("New milestone {} ({}/{}) added to round #{}", milestoneHash, numberOfMilestones, BasePendulumConfig.Defaults.NUMBER_OF_ACTIVE_VALIDATORS, roundIndex); + Set tips = null; + try { + tips = RoundViewModel.getTipSet(tangle, milestoneHash, config.getValidatorSecurity()); + } catch (Exception e) { + log.error("Add milestone to round log!", e); + } + tips.forEach(t -> + log.delegate().debug("Valid tips transaction: {}, from milesltone {} ", t, milestoneHash) + ); } /** @@ -328,6 +337,10 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw addMilestoneToRoundLog(transaction.getHash(), roundIndex, currentRoundViewModel.size(), validators.size()); setRoundIndexAndConfirmations(currentRoundViewModel, transaction, roundIndex); publishMilestoneRefs(transaction); + } else { + log.delegate().debug("Failed to validate milestone {} in round #{}", transaction.getHash(), roundIndex); + Set tips = RoundViewModel.getTipSet(tangle, transaction.getHash(), config.getValidatorSecurity()); + tips.forEach(t -> log.delegate().debug("Missed transaction: {}, from skipped milesltone {} ", t, transaction.getHash())); } if (!transaction.isSolid()) { From 986a4fb55616af604bfd854c1f545e9d072864c6 Mon Sep 17 00:00:00 2001 From: Cristina Date: Fri, 1 Nov 2019 17:24:42 +0200 Subject: [PATCH 07/14] Marked transaction as confirmed only after the balance is applied. --- src/main/java/net/helix/pendulum/service/API.java | 6 ++++-- .../service/ledger/impl/LedgerServiceImpl.java | 10 ++++++++++ .../service/milestone/impl/MilestoneTrackerImpl.java | 2 -- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 62883a76..ce22e946 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -699,7 +699,7 @@ private AbstractResponse getConfirmationStatesStatement(final List trans log.trace("tx_confirmations {}:[{}:{}]", transaction.getHash().toString(), transaction.getConfirmations(), (double) transaction.getConfirmations() / n); // is transaction finalized - if(((double)transaction.getConfirmations() / n) > threshold) { + if(transaction.getRoundIndex() > 0 && ((double)transaction.getConfirmations() / n) > threshold) { confirmationStates[count] = 1; } // not finalized yet @@ -1611,10 +1611,10 @@ private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List transactionStrings = attachAndStore(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); + broadcastTransactionsStatement(transactionStrings); if (createVirtualTransactions) { createAndBroadcastVirtualTransactions(data, transactionStrings); } - broadcastTransactionsStatement(transactionStrings); } /** @@ -1764,6 +1764,8 @@ private List getConfirmedTips() throws Exception { confirmedTips.add(txVM.getHash()); } else if(txVM.isSolid()){ log.warn("Inconsistent transaction has been removed from tips: {} ", txVM.getHash()); + txVM.setConfirmations(-1); + txVM.update(tangle, snapshotProvider.getInitialSnapshot(), "confirmation"); tipsViewModel.removeTipHash(txVM.getHash()); } } diff --git a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java index 7de10a4b..1564984c 100644 --- a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java @@ -322,6 +322,16 @@ private boolean generateStateDiff(RoundViewModel round, Set successfullyPr } if (!simulation && !processedTips.isEmpty()) { + + TransactionViewModel.fromHashes(processedTips, tangle).forEach(tvm -> { + try { + tvm.setRoundIndex(tvm.getRoundIndex() == 0 ? round.index() : tvm.getRoundIndex()); + tvm.update(tangle, snapshotProvider.getInitialSnapshot(), "roundIndex"); + } catch (Exception e){ + log.error("Error during transaction round index update: " + tvm.getHash(), e); + } + }); + milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); if (!totalBalanceChanges.isEmpty()) { new StateDiffViewModel(totalBalanceChanges, round.index()).store(tangle); diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java index 3f6e7b3e..4e775b01 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java @@ -207,8 +207,6 @@ public void setRoundIndexAndConfirmations(RoundViewModel currentRVM, Transaction // The confirmation counter should be incremented with each milestone reference for (Hash tx : referencedTipSet) { TransactionViewModel txvm = TransactionViewModel.fromHash(tangle, tx); - txvm.setRoundIndex(txvm.getRoundIndex() == 0 ? roundIndex : txvm.getRoundIndex()); - txvm.update(tangle, snapshotProvider.getInitialSnapshot(), "roundIndex"); txvm.setConfirmations(txvm.getConfirmations() + 1); txvm.update(tangle, snapshotProvider.getInitialSnapshot(), "confirmation"); } From 74cbed7c9ab3890cdc3576d34f3cded092b5f2c4 Mon Sep 17 00:00:00 2001 From: Cristina Date: Mon, 4 Nov 2019 17:19:08 +0200 Subject: [PATCH 08/14] Remove milestone special management (use virtual transactions) --- .../helix/pendulum/TransactionValidator.java | 60 +++------- .../pendulum/controllers/RoundViewModel.java | 112 ++++++++---------- .../java/net/helix/pendulum/service/API.java | 9 +- .../ledger/impl/LedgerServiceImpl.java | 10 +- .../impl/LatestSolidMilestoneTrackerImpl.java | 13 +- .../milestone/impl/MilestonePublisher.java | 4 +- .../milestone/impl/MilestoneTrackerImpl.java | 4 +- .../impl/VirtualTransactionServiceImpl.java | 2 +- .../pendulum/utils/bundle/BundleUtils.java | 3 + 9 files changed, 89 insertions(+), 128 deletions(-) diff --git a/src/main/java/net/helix/pendulum/TransactionValidator.java b/src/main/java/net/helix/pendulum/TransactionValidator.java index 47993977..0ffee291 100644 --- a/src/main/java/net/helix/pendulum/TransactionValidator.java +++ b/src/main/java/net/helix/pendulum/TransactionValidator.java @@ -14,22 +14,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.Queue; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; -import static net.helix.pendulum.controllers.TransactionViewModel.PREFILLED_SLOT; -import static net.helix.pendulum.controllers.TransactionViewModel.SIZE; -import static net.helix.pendulum.controllers.TransactionViewModel.VALUE_OFFSET; -import static net.helix.pendulum.controllers.TransactionViewModel.VALUE_SIZE; -import static net.helix.pendulum.controllers.TransactionViewModel.VALUE_USABLE_SIZE; -import static net.helix.pendulum.controllers.TransactionViewModel.fromHash; -import static net.helix.pendulum.controllers.TransactionViewModel.updateSolidTransactions; +import static net.helix.pendulum.controllers.TransactionViewModel.*; public class TransactionValidator { private static final Logger log = LoggerFactory.getLogger(TransactionValidator.class); @@ -275,46 +263,38 @@ public boolean checkSolidity(Hash hash, boolean milestone) throws Exception { * @throws Exception if anything goes wrong while trying to solidify the transaction */ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTransactions) throws Exception { - if(fromHash(tangle, hash).isSolid()) { + if (fromHash(tangle, hash).isSolid()) { return true; } Set analyzedHashes = new HashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); - if(maxProcessedTransactions != Integer.MAX_VALUE) { + if (maxProcessedTransactions != Integer.MAX_VALUE) { maxProcessedTransactions += analyzedHashes.size(); } + log.debug("Check solidity for hash " + hash); boolean solid = true; final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); Hash hashPointer; while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { if (analyzedHashes.add(hashPointer)) { - if(analyzedHashes.size() >= maxProcessedTransactions) { + if (analyzedHashes.size() >= maxProcessedTransactions) { return false; } final TransactionViewModel transaction = fromHash(tangle, hashPointer); - if(!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { + if (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { if (transaction.getType() == PREFILLED_SLOT) { solid = false; if (!transactionRequester.isTransactionRequested(hashPointer, milestone)) { transactionRequester.requestTransaction(hashPointer, milestone); + solid = false; break; } } else { - // transaction of milestone bundle - TransactionViewModel milestoneTx; - if ((milestoneTx = transaction.isMilestoneBundle(tangle)) != null){ - Set parents = RoundViewModel.getMilestoneTrunk(tangle, transaction, milestoneTx); - parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transaction, milestoneTx, config.getValidatorSecurity())); - for (Hash parent : parents){ - nonAnalyzedTransactions.offer(parent); - } - } - // normal transaction - else { - nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); - nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); - } + nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); + log.debug("Check solidity for hash:" + transaction.getHash() + " trunk " + transaction.getTrunkTransactionHash()); + nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); + log.debug("Check solidity for hash:" + transaction.getHash() + " branch " + transaction.getBranchTransactionHash()); } } } @@ -421,20 +401,12 @@ protected void propagateSolidTransactions() { //what transaction we gossip. public void updateStatus(TransactionViewModel transactionViewModel) throws Exception { transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); - if(transactionViewModel.getApprovers(tangle).size() == 0) { + + if (transactionViewModel.getApprovers(tangle).size() == 0 && !transactionViewModel.isVirtual()) { tipsViewModel.addTipHash(transactionViewModel.getHash()); } else { - TransactionViewModel milestoneTx; - if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ - Set parents = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); - parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx, config.getValidatorSecurity())); - for (Hash parent : parents){ - tipsViewModel.removeTipHash(parent); - } - } else { - tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); - tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); - } + tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); + tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); } if(quickSetSolid(transactionViewModel)) { diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 46ec14fe..e383d825 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -1,14 +1,11 @@ package net.helix.pendulum.controllers; import net.helix.pendulum.TransactionValidator; -import net.helix.pendulum.conf.Config; +import net.helix.pendulum.conf.BasePendulumConfig; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.crypto.Sponge; -import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.crypto.merkle.MerkleTree; import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; -import net.helix.pendulum.conf.BasePendulumConfig; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.IntegerIndex; @@ -27,7 +24,6 @@ import net.helix.pendulum.utils.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.bouncycastle.util.encoders.Hex; import java.util.*; import java.util.concurrent.ConcurrentHashMap; @@ -254,32 +250,12 @@ public static int getRoundIndex(TransactionViewModel milestoneTransaction) { } // todo this may be very inefficient - public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ + public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception { Set trunk = new HashSet<>(); - int round = RoundViewModel.getRoundIndex(milestoneTx); // idx = n: milestone merkle root in trunk if (transaction.getCurrentIndex() == transaction.lastIndex()) { - // add previous milestones to non analyzed transactions - RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); - if (prevMilestone == null) { - if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { - trunk.add(Hash.NULL_HASH); - } - } else { - Set prevMilestones = prevMilestone.getHashes(); - MerkleTree merkle = new MerkleTreeImpl(); - byte[] merkleTreeRoot = merkle.getMerkleRoot(new ArrayList<>(prevMilestones), MerkleOptions.getDefault()); - if (transaction.getTrunkTransactionHash().equals(HashFactory.TRANSACTION.create(merkleTreeRoot))) { - //System.out.println("trunk (prev. milestones): "); - if (prevMilestones.isEmpty()) { - trunk.add(Hash.NULL_HASH); - } else { - trunk.addAll(prevMilestones); - } - } - } - } - else { + trunk.addAll(splitMilestonesMerkleRoot(tangle, transaction.getTrunkTransactionHash(), milestoneTx)); + } else { // idx = 0 - (n-1): merkle root in branch, trunk is normal tx hash trunk.add(transaction.getTrunkTransactionHash()); } @@ -289,50 +265,64 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr return trunk; } - public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx, int security) throws Exception{ + + public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx, int security) throws Exception { Set branch = new HashSet<>(); - int round = RoundViewModel.getRoundIndex(milestoneTx); // idx = n: milestone merkle root in trunk and tips merkle root in branch if (transaction.getCurrentIndex() == transaction.lastIndex()) { - // tips merkle root - Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); + branch.addAll(splitTipsMerkleRoot(tangle, transaction, milestoneTx, security)); + } else { + branch.addAll(splitMilestonesMerkleRoot(tangle, transaction.getBranchTransactionHash(), milestoneTx)); + } + + if (log.isTraceEnabled()) { + log.trace("Milestone branch: {}", PendulumUtils.logHashList(branch, 8)); + } + return branch; + } + + + private static Set splitMilestonesMerkleRoot(Tangle tangle, Hash expectedMilestoneMerkleRoot, TransactionViewModel milestoneTx) throws Exception { + Set result = new HashSet<>(); + int round = RoundViewModel.getRoundIndex(milestoneTx); + RoundViewModel prevMilestone = RoundViewModel.get(tangle, round - 1); + if (prevMilestone == null) { + if (Hash.NULL_HASH.equals(expectedMilestoneMerkleRoot)) { + result.add(Hash.NULL_HASH); + } + } else { + Set prevMilestones = prevMilestone.getHashes(); MerkleTree merkle = new MerkleTreeImpl(); - byte[] root = merkle.getMerkleRoot(new ArrayList<>(confirmedTips), MerkleOptions.getDefault()); - //System.out.println("merkleRoot: " + transaction.getBranchTransactionHash().hexString()); - //System.out.println("recalculated merkleRoot: " + merkleTree.get(merkleTree.size()-1).get(0).hexString()); - if (transaction.getBranchTransactionHash().equals(HashFactory.TRANSACTION.create(root))) { - //System.out.println("branch (tips): "); - if (confirmedTips.isEmpty()){ - branch.add(Hash.NULL_HASH); + Hash merkleTreeRoot = HashFactory.TRANSACTION.create(merkle.getMerkleRoot(new ArrayList<>(prevMilestones), MerkleOptions.getDefault())); + if (expectedMilestoneMerkleRoot.equals(merkleTreeRoot)) { + if (prevMilestones.isEmpty()) { + result.add(Hash.NULL_HASH); } else { - branch.addAll(confirmedTips); + result.addAll(prevMilestones); } + } else { + log.error("Milestone merkle tree can not be splitted into milestones, milestone merkle root {} for milestone {}, computed merkle root based on previous round milestones: {}", expectedMilestoneMerkleRoot, milestoneTx.getHash(), merkleTreeRoot); } } - else { - // add previous milestones to non analyzed transactions - RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); - if (prevMilestone == null) { - if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { - branch.add(Hash.NULL_HASH); - } + return result; + } + + private static Set splitTipsMerkleRoot(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx, int security) throws Exception { + Set result = new HashSet<>(); + Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); + Hash root = HashFactory.TRANSACTION.create(new MerkleTreeImpl().getMerkleRoot(new ArrayList<>(confirmedTips), MerkleOptions.getDefault())); + if (transaction.getBranchTransactionHash().equals(root)) { + if (confirmedTips.isEmpty()) { + result.add(Hash.NULL_HASH); } else { - Set prevMilestones = prevMilestone.getHashes(); - byte[] merkleTreeRoot = new MerkleTreeImpl().getMerkleRoot(new ArrayList<>(prevMilestones), MerkleOptions.getDefault()); - if (transaction.getBranchTransactionHash().equals(HashFactory.TRANSACTION.create(merkleTreeRoot))) { - - if (prevMilestones.isEmpty()) { - branch.add(Hash.NULL_HASH); - } else { - branch.addAll(prevMilestones); - } - } + result.addAll(confirmedTips); } + } else { + //TODO check this + //confirmedTips.add(transaction.getBranchTransactionHash()); + log.error("Tips merkle tree can not be splitted into tips, tips merkle root: {} for milestone {} ", transaction.getBranchTransactionHash(), milestoneTx.getHash()); } - if (log.isTraceEnabled()) { - log.trace("Milestone branch: {}", PendulumUtils.logHashList(branch, 8)); - } - return branch; + return result; } public static Set getTipSet(Tangle tangle, Hash milestone, int security) throws Exception { diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index ce22e946..6069b115 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -1623,7 +1623,7 @@ private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List milestoneBundle) { + private void createAndBroadcastVirtualTransactions(byte[] tipsBytes, List milestoneBundle) throws Exception { TransactionViewModel milestone = getFirstTransactionFromBundle(milestoneBundle); if (milestone != null) { List tips = extractTipsFromData(tipsBytes); @@ -1634,7 +1634,12 @@ private void createAndBroadcastVirtualTransactions(byte[] tipsBytes, List merkleTransactions = new TransactionMerkleTreeImpl().buildMerkle(tips, options); + if(previousRound != null) { + merkleTransactions.addAll(new TransactionMerkleTreeImpl().buildMerkle(new ArrayList<>(previousRound.getHashes()), options)); + } List virtualTransactions = merkleTransactions.stream().map(t -> { try { @@ -1643,7 +1648,7 @@ private void createAndBroadcastVirtualTransactions(byte[] tipsBytes, List generateBalanceDiff(Set visitedTransactions, Set parents = RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx, config.getValidatorSecurity()); - for (Hash parent : parents) { - nonAnalyzedTransactions.offer(parent); - } - } else { - nonAnalyzedTransactions.offer(transactionViewModel.getBranchTransactionHash()); - } + nonAnalyzedTransactions.offer(transactionViewModel.getBranchTransactionHash()); } } } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 23f8d94f..53094a2c 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -161,10 +161,12 @@ public void trackLatestSolidMilestones() throws MilestoneException { } if (isRoundSolid(nextRound)) { applyRoundToLedger(nextRound); - logChange(currentSolidRoundIndex); currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); - tangle.publish("ctx %s %d", nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)), nextRound.index()); - } + if(nextRound.index() == currentSolidRoundIndex){ + logChange(currentSolidRoundIndex); + tangle.publish("ctx %s %d", nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)), nextRound.index()); + } + } } } catch (Exception e) { throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); @@ -274,10 +276,7 @@ private void stopRepair() { private void logChange(int prevSolidRoundIndex) { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); int latestRoundIndex = latestSnapshot.getIndex(); - - if (prevSolidRoundIndex != latestRoundIndex) { - log.debug("Round #" + latestRoundIndex + " is SOLID"); - } + log.debug("Round #" + latestRoundIndex + " is SOLID"); } /** diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java index 84aa62fa..5f01036e 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java @@ -204,7 +204,7 @@ private Runnable getRunnablePublishMilestone() { try { publishMilestone(); } catch (Exception e) { - e.printStackTrace(); + log.error("Error during processing milestone", e); } }; } @@ -215,7 +215,7 @@ public void shutdown() { try { writeKeyIndex(); } catch (Exception e) { - e.printStackTrace(); + log.error("Shut down milestone publisher", e); } scheduledExecutorService.shutdown(); } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java index 4e775b01..0085ef89 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java @@ -165,8 +165,8 @@ public MilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide private void publishMilestoneRefs(TransactionViewModel transaction) throws Exception { BundleViewModel bundle = BundleViewModel.load(tangle, transaction.getBundleHash()); - for (Hash tx: bundle.getHashes()) { - tangle.publish("lmr %s %s %s", tx, "Branch " + RoundViewModel.getMilestoneBranch(tangle, TransactionViewModel.fromHash(tangle, tx), transaction, config.getValidatorSecurity()), "Trunk " + RoundViewModel.getMilestoneTrunk(tangle, TransactionViewModel.fromHash(tangle, tx), transaction)); + for (Hash tx : bundle.getHashes()) { + tangle.publish("lmr %s %s %s", tx, "Branch " + transaction.getBranchTransactionHash(), "Trunk " + transaction.getTrunkTransactionHash(), transaction); } } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java index fab27f33..bdbd232e 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java @@ -128,7 +128,7 @@ private TransactionViewModel reconstructParent(TransactionViewModel t1, Transact Hash trunk = RoundViewModel.getRoundIndex(t1) > RoundViewModel.getRoundIndex(t2) ? t2.getHash() : t1.getHash(); Hash branch = RoundViewModel.getRoundIndex(t1) < RoundViewModel.getRoundIndex(t2) ? t2.getHash() : t1.getHash(); - byte[] newTransaction = BundleUtils.createVirtualTransaction(branch, trunk, getParentIndex(RoundViewModel.getRoundIndex(t1)), + byte[] newTransaction = BundleUtils.createVirtualTransaction(branch, trunk, getParentIndex(RoundViewModel.getRoundIndex(t1)), t1.getSignature(), t1.getAddressHash()); TransactionViewModel newTransactionViewModel = new TransactionViewModel(newTransaction, SpongeFactory.Mode.S256); diff --git a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java index 642c1f44..92fb6fef 100644 --- a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java @@ -222,6 +222,9 @@ public static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, l System.arraycopy(nonce, 0, transaction, TransactionViewModel.NONCE_OFFSET, TransactionViewModel.NONCE_SIZE); System.arraycopy(branchHash.bytes(), 0, transaction, TransactionViewModel.BRANCH_TRANSACTION_OFFSET, TransactionViewModel.BRANCH_TRANSACTION_SIZE); System.arraycopy(trunkHash.bytes(), 0, transaction, TransactionViewModel.TRUNK_TRANSACTION_OFFSET, TransactionViewModel.TRUNK_TRANSACTION_SIZE); + + bundleUtils.addBundleHash(Arrays.asList(transaction), SpongeFactory.Mode.S256); + return transaction; } } From 29523afbf6afdafb9508be6da1c97d701b8c8ff6 Mon Sep 17 00:00:00 2001 From: Cristina Date: Tue, 5 Nov 2019 13:44:34 +0200 Subject: [PATCH 09/14] Virtual transaction request bugfix --- .../helix/pendulum/TransactionValidator.java | 20 +++++++++++++------ .../impl/LatestSolidMilestoneTrackerImpl.java | 8 ++++---- .../tipselection/impl/WalkValidatorImpl.java | 3 +++ 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/src/main/java/net/helix/pendulum/TransactionValidator.java b/src/main/java/net/helix/pendulum/TransactionValidator.java index 0ffee291..334b560e 100644 --- a/src/main/java/net/helix/pendulum/TransactionValidator.java +++ b/src/main/java/net/helix/pendulum/TransactionValidator.java @@ -150,9 +150,12 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { || transactionViewModel.getAttachmentTimestamp() > System.currentTimeMillis() + MAX_TIMESTAMP_FUTURE_MS; } - private boolean isTransactionRequested(TransactionViewModel transactionViewModel) { - if(transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) { - //todo if is virtual compute it locally + private boolean isTransactionRequested(TransactionViewModel transactionViewModel) throws Exception { + if (transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) { + if (TransactionViewModel.exists(tangle, transactionViewModel.getHash())) { + transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); + return false; + } return true; } return false; @@ -178,10 +181,15 @@ public void runValidation(TransactionViewModel transactionViewModel, final int m transactionViewModel.setMetadata(); transactionViewModel.setAttachmentData(); - if (!transactionViewModel.isVirtual() && isTransactionRequested(transactionViewModel)) { - log.error("Waiting for transaction... " + transactionViewModel.getHash()); - throw new IllegalStateException("Transaction is requested {} " + transactionViewModel.getHash()); + try { + if (isTransactionRequested(transactionViewModel)) { + log.error("Waiting for transaction... " + transactionViewModel.getHash()); + throw new IllegalStateException("Transaction is requested {} " + transactionViewModel.getHash()); + } + } catch (Exception e) { + throw new IllegalStateException("Transaction is requested {} " + transactionViewModel.getHash(), e); } + if(hasInvalidTimestamp(transactionViewModel)) { log.debug("Invalid timestamp for txHash/addressHash: {} {}", transactionViewModel.getHash().toString(), transactionViewModel.getAddressHash().toString()); throw new StaleTimestampException("Invalid transaction timestamp."); diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 53094a2c..8f676f96 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -163,10 +163,10 @@ public void trackLatestSolidMilestones() throws MilestoneException { applyRoundToLedger(nextRound); currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); if(nextRound.index() == currentSolidRoundIndex){ - logChange(currentSolidRoundIndex); - tangle.publish("ctx %s %d", nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)), nextRound.index()); - } - } + logChange(currentSolidRoundIndex); + tangle.publish("ctx %s %d", nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)), nextRound.index()); + } + } } } catch (Exception e) { throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); diff --git a/src/main/java/net/helix/pendulum/service/tipselection/impl/WalkValidatorImpl.java b/src/main/java/net/helix/pendulum/service/tipselection/impl/WalkValidatorImpl.java index a73c5e2a..0aa88dab 100644 --- a/src/main/java/net/helix/pendulum/service/tipselection/impl/WalkValidatorImpl.java +++ b/src/main/java/net/helix/pendulum/service/tipselection/impl/WalkValidatorImpl.java @@ -59,6 +59,9 @@ public WalkValidatorImpl(Tangle tangle, SnapshotProvider snapshotProvider, Ledge public boolean isValid(Hash transactionHash) throws Exception { TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, transactionHash); + if(Hash.NULL_HASH.equals(transactionHash)){ + return true; + } if (transactionViewModel.getType() == TransactionViewModel.PREFILLED_SLOT) { log.debug("transactionViewModel: {} ", transactionViewModel.getBytes()); log.debug("transactionViewModel.Type: {} ", transactionViewModel.getType()); From 36aa6eb0244e0be6cc36c2823315c254109180aa Mon Sep 17 00:00:00 2001 From: Cristina Date: Tue, 5 Nov 2019 13:45:45 +0200 Subject: [PATCH 10/14] Add leaves padding and sorting in merkle --- .../merkle/impl/AbstractMerkleTree.java | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java index 13478984..4f1316cc 100644 --- a/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java @@ -15,6 +15,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.List; public abstract class AbstractMerkleTree implements MerkleTree { @@ -70,6 +71,8 @@ public List buildMerkle(List leaves, MerkleOptions options) { byte[] buffer; Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); int row = 1; + addPaddingLeaves(leaves); + sortLeaves(leaves); List merkleNodes = new ArrayList<>(); int depth = getTreeDepth(leaves.size()); while (leaves.size() > 1) { @@ -90,14 +93,19 @@ public List buildMerkle(List leaves, MerkleOptions options) { return merkleNodes; } + private void sortLeaves(List leaves) { + leaves.sort(Comparator.comparing((Hash m) -> m.toString())); + } + @Override public List> buildMerkleTree(List leaves, MerkleOptions options) { if (leaves.isEmpty()) { leaves.add(getDefaultMerkleHash()); } byte[] buffer; + addPaddingLeaves(leaves); + sortLeaves(leaves); Sponge sha3 = SpongeFactory.create(options.getMode()); - int depth = getTreeDepth(leaves.size()); List> merkleTree = new ArrayList<>(); if (options.isIncludeLeavesInTree()) { @@ -150,8 +158,26 @@ private boolean validateMerklePath(byte[] path, int keyIndex, byte[] address, Me return HashFactory.ADDRESS.create(merkleRoot).equals(options.getAddress()); } + protected void addPaddingLeaves(List leaves){ + int closestPow = (int) Math.pow(2.0, getClosestPow(leaves.size())); + int leaveSize = leaves.size(); + while(leaveSize < closestPow){ + leaves.add(leaveSize++, Hash.NULL_HASH); + } + } + protected int getTreeDepth(int leavesNumber) { - return (int) Math.ceil((float) (Math.log(leavesNumber) / Math.log(2))); + return getClosestPow(leavesNumber); + } + + protected int getClosestPow(int i) { + int j = 1; + int power = 0; + while (j < i) { + j = j << 1; + power++; + } + return power; } protected Hash getLeaves(List leaves, int index) { From 3b96501f80d92f4565f74907df33099cdd57f68b Mon Sep 17 00:00:00 2001 From: Cristina Date: Tue, 5 Nov 2019 13:56:09 +0200 Subject: [PATCH 11/14] Fix travis errors --- .../pendulum/crypto/merkle/impl/AbstractMerkleTree.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java index 4f1316cc..593bc8fa 100644 --- a/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java @@ -78,7 +78,9 @@ public List buildMerkle(List leaves, MerkleOptions options) { while (leaves.size() > 1) { List nextKeys = Arrays.asList(new Hash[getParentNodesSize(leaves)]); for (int i = 0; i < nextKeys.size(); i++) { - if (areLeavesNull(leaves, i)) continue; + if (areLeavesNull(leaves, i)) { + continue; + } sha3.reset(); Hash k1 = getLeaves(leaves, i * 2); Hash k2 = getLeaves(leaves, i * 2 + 1); @@ -115,7 +117,9 @@ public List> buildMerkleTree(List leaves, MerkleOptions o while (leaves.size() > 1) { List nextKeys = Arrays.asList(new Hash[getParentNodesSize(leaves)]); for (int i = 0; i < nextKeys.size(); i++) { - if (areLeavesNull(leaves, i)) continue; + if (areLeavesNull(leaves, i)) { + continue; + } sha3.reset(); Hash k1 = getLeaves(leaves, i * 2); Hash k2 = getLeaves(leaves, i * 2 + 1); From 93d120225879e82727ad6cac64102b43288cb201 Mon Sep 17 00:00:00 2001 From: Cristina Date: Mon, 11 Nov 2019 09:16:47 +0200 Subject: [PATCH 12/14] Fix code review issue --- src/main/java/net/helix/pendulum/Pendulum.java | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/src/main/java/net/helix/pendulum/Pendulum.java b/src/main/java/net/helix/pendulum/Pendulum.java index 08f48ff5..d8959803 100644 --- a/src/main/java/net/helix/pendulum/Pendulum.java +++ b/src/main/java/net/helix/pendulum/Pendulum.java @@ -11,6 +11,7 @@ import net.helix.pendulum.network.replicator.Replicator; import net.helix.pendulum.service.TipsSolidifier; import net.helix.pendulum.service.ledger.impl.LedgerServiceImpl; +import net.helix.pendulum.service.milestone.VirtualTransactionService; import net.helix.pendulum.service.milestone.impl.*; import net.helix.pendulum.service.snapshot.SnapshotException; import net.helix.pendulum.service.snapshot.impl.LocalSnapshotManagerImpl; @@ -19,16 +20,8 @@ import net.helix.pendulum.service.spentaddresses.SpentAddressesException; import net.helix.pendulum.service.spentaddresses.impl.SpentAddressesProviderImpl; import net.helix.pendulum.service.spentaddresses.impl.SpentAddressesServiceImpl; -import net.helix.pendulum.service.tipselection.EntryPointSelector; -import net.helix.pendulum.service.tipselection.RatingCalculator; -import net.helix.pendulum.service.tipselection.TailFinder; -import net.helix.pendulum.service.tipselection.TipSelector; -import net.helix.pendulum.service.tipselection.Walker; -import net.helix.pendulum.service.tipselection.impl.CumulativeWeightCalculator; -import net.helix.pendulum.service.tipselection.impl.EntryPointSelectorImpl; -import net.helix.pendulum.service.tipselection.impl.TailFinderImpl; -import net.helix.pendulum.service.tipselection.impl.TipSelectorImpl; -import net.helix.pendulum.service.tipselection.impl.WalkerAlpha; +import net.helix.pendulum.service.tipselection.*; +import net.helix.pendulum.service.tipselection.impl.*; import net.helix.pendulum.service.transactionpruning.TransactionPruningException; import net.helix.pendulum.service.transactionpruning.async.AsyncTransactionPruner; import net.helix.pendulum.service.validatormanager.impl.CandidateSolidifierImpl; @@ -101,7 +94,7 @@ public class Pendulum { public final MilestoneSolidifierImpl milestoneSolidifier; public final CandidateSolidifierImpl candidateSolidifier; public final TransactionRequesterWorkerImpl transactionRequesterWorker; - public final VirtualTransactionServiceImpl virtualTransactionService; + public final VirtualTransactionService virtualTransactionService; public final Tangle tangle; public final TransactionValidator transactionValidator; From f3cefbc22dd631d304914af259d69ae742bd1869 Mon Sep 17 00:00:00 2001 From: Cristina Date: Mon, 11 Nov 2019 09:19:57 +0200 Subject: [PATCH 13/14] Add bundle nonce model hash & equals --- .../controllers/BundleNonceViewModel.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java b/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java index 78393117..ad5fe0f4 100644 --- a/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java @@ -11,6 +11,7 @@ import java.util.HashMap; import java.util.Map; +import java.util.Objects; import java.util.Set; /** @@ -138,4 +139,18 @@ public BundleNonceViewModel next(Tangle tangle) throws Exception { } return null; } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BundleNonceViewModel that = (BundleNonceViewModel) o; + return Objects.equals(self, that.self) && + Objects.equals(hash, that.hash); + } + + @Override + public int hashCode() { + return Objects.hash(self, hash); + } } From 47a2b1ebcab65d6a3c64771c50c5965e83827e27 Mon Sep 17 00:00:00 2001 From: Cristina Date: Mon, 11 Nov 2019 11:10:58 +0200 Subject: [PATCH 14/14] Fix code review issues --- .../java/net/helix/pendulum/controllers/RoundViewModel.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index eee710a5..8437df46 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -557,12 +557,8 @@ public List getMilestonesToApprove(Tangle tangle, SnapshotProvider snapsho removeMilestone(m.getHash()); }); update(tangle); - } catch (LedgerException e) { - log.error("Error during appling milestone states simultation", e); - } catch (SnapshotException e) { - log.error("Error during appling milestone states simultation", e); } catch (Exception e) { - log.error("Error during appling milestone states simultation", e); + log.error("Error during appling milestone states simultaneously", e); } }