diff --git a/src/main/java/net/helix/pendulum/Pendulum.java b/src/main/java/net/helix/pendulum/Pendulum.java index bd1cb26b..d8959803 100644 --- a/src/main/java/net/helix/pendulum/Pendulum.java +++ b/src/main/java/net/helix/pendulum/Pendulum.java @@ -11,11 +11,8 @@ import net.helix.pendulum.network.replicator.Replicator; import net.helix.pendulum.service.TipsSolidifier; import net.helix.pendulum.service.ledger.impl.LedgerServiceImpl; -import net.helix.pendulum.service.milestone.impl.LatestSolidMilestoneTrackerImpl; -import net.helix.pendulum.service.milestone.impl.MilestoneServiceImpl; -import net.helix.pendulum.service.milestone.impl.MilestoneSolidifierImpl; -import net.helix.pendulum.service.milestone.impl.MilestoneTrackerImpl; -import net.helix.pendulum.service.milestone.impl.SeenMilestonesRetrieverImpl; +import net.helix.pendulum.service.milestone.VirtualTransactionService; +import net.helix.pendulum.service.milestone.impl.*; import net.helix.pendulum.service.snapshot.SnapshotException; import net.helix.pendulum.service.snapshot.impl.LocalSnapshotManagerImpl; import net.helix.pendulum.service.snapshot.impl.SnapshotProviderImpl; @@ -23,16 +20,8 @@ import net.helix.pendulum.service.spentaddresses.SpentAddressesException; import net.helix.pendulum.service.spentaddresses.impl.SpentAddressesProviderImpl; import net.helix.pendulum.service.spentaddresses.impl.SpentAddressesServiceImpl; -import net.helix.pendulum.service.tipselection.EntryPointSelector; -import net.helix.pendulum.service.tipselection.RatingCalculator; -import net.helix.pendulum.service.tipselection.TailFinder; -import net.helix.pendulum.service.tipselection.TipSelector; -import net.helix.pendulum.service.tipselection.Walker; -import net.helix.pendulum.service.tipselection.impl.CumulativeWeightCalculator; -import net.helix.pendulum.service.tipselection.impl.EntryPointSelectorImpl; -import net.helix.pendulum.service.tipselection.impl.TailFinderImpl; -import net.helix.pendulum.service.tipselection.impl.TipSelectorImpl; -import net.helix.pendulum.service.tipselection.impl.WalkerAlpha; +import net.helix.pendulum.service.tipselection.*; +import net.helix.pendulum.service.tipselection.impl.*; import net.helix.pendulum.service.transactionpruning.TransactionPruningException; import net.helix.pendulum.service.transactionpruning.async.AsyncTransactionPruner; import net.helix.pendulum.service.validatormanager.impl.CandidateSolidifierImpl; @@ -105,6 +94,7 @@ public class Pendulum { public final MilestoneSolidifierImpl milestoneSolidifier; public final CandidateSolidifierImpl candidateSolidifier; public final TransactionRequesterWorkerImpl transactionRequesterWorker; + public final VirtualTransactionService virtualTransactionService; public final Tangle tangle; public final TransactionValidator transactionValidator; @@ -149,7 +139,7 @@ public Pendulum(PendulumConfig configuration) throws TransactionPruningException ? new AsyncTransactionPruner() : null; transactionRequesterWorker = new TransactionRequesterWorkerImpl(); - + virtualTransactionService = new VirtualTransactionServiceImpl(); // legacy code bundleValidator = new BundleValidator(); tangle = new Tangle(); @@ -157,7 +147,7 @@ public Pendulum(PendulumConfig configuration) throws TransactionPruningException transactionRequester = new TransactionRequester(tangle, snapshotProvider); transactionValidator = new TransactionValidator(tangle, snapshotProvider, tipsViewModel, transactionRequester, configuration); node = new Node(tangle, snapshotProvider, transactionValidator, transactionRequester, tipsViewModel, - latestMilestoneTracker, configuration); + latestMilestoneTracker, virtualTransactionService, configuration); replicator = new Replicator(node, configuration); udpReceiver = new UDPReceiver(node, configuration); tipsSolidifier = new TipsSolidifier(tangle, transactionValidator, tipsViewModel, configuration); @@ -228,6 +218,7 @@ private void injectDependencies() throws SnapshotException, TransactionPruningEx localSnapshotManager.init(snapshotProvider, snapshotService, transactionPruner, configuration); } milestoneService.init(tangle, snapshotProvider, snapshotService, transactionValidator, configuration); + virtualTransactionService.init(tangle, snapshotProvider, transactionValidator); validatorManagerService.init(tangle, snapshotProvider, snapshotService, configuration); candidateTracker.init(tangle, snapshotProvider, validatorManagerService, candidateSolidifier, configuration); latestMilestoneTracker.init(tangle, snapshotProvider, milestoneService, milestoneSolidifier, candidateTracker, configuration); diff --git a/src/main/java/net/helix/pendulum/SignedFiles.java b/src/main/java/net/helix/pendulum/SignedFiles.java index 415ba60f..0a4c96b0 100644 --- a/src/main/java/net/helix/pendulum/SignedFiles.java +++ b/src/main/java/net/helix/pendulum/SignedFiles.java @@ -1,20 +1,16 @@ package net.helix.pendulum; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.Sha3; import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import org.apache.commons.lang3.ArrayUtils; import org.bouncycastle.util.encoders.Hex; -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.UncheckedIOException; +import java.io.*; import java.util.Arrays; public class SignedFiles { @@ -31,7 +27,7 @@ private static boolean validateSignature(String signatureFilename, String public byte[] bundle = Winternitz.normalizedBundle(digest); byte[] root; int i; - + MerkleTree merkle = new MerkleTreeImpl(); try (InputStream inputStream = SignedFiles.class.getResourceAsStream(signatureFilename); BufferedReader reader = new BufferedReader((inputStream == null) ? new FileReader(signatureFilename) : new InputStreamReader(inputStream))) { @@ -46,7 +42,7 @@ private static boolean validateSignature(String signatureFilename, String public if ((line = reader.readLine()) != null) { byte[] lineBytes = Hex.decode(line); - root = Merkle.getMerkleRoot(mode, Winternitz.address(mode, digests), lineBytes, 0, index, depth); + root = merkle.getMerkleRoot(mode, Winternitz.address(mode, digests), lineBytes, 0, index, depth); } else { root = Winternitz.address(mode, digests); @@ -75,7 +71,7 @@ private static byte[] digestFile(String filename, Sponge sha3) throws IOExceptio messageBytes = Hash.NULL_HASH.bytes(); } int requiredLength = (int) Math.ceil(messageBytes.length / 32.0) * 32; - byte[] finalizedMessage = Merkle.padding(messageBytes, requiredLength); + byte[] finalizedMessage = MerkleTree.padding(messageBytes, requiredLength); // crypto snapshot message sha3.absorb(finalizedMessage, 0, finalizedMessage.length); byte[] signature = new byte[Sha3.HASH_LENGTH]; diff --git a/src/main/java/net/helix/pendulum/TransactionValidator.java b/src/main/java/net/helix/pendulum/TransactionValidator.java index 66890dc2..334b560e 100644 --- a/src/main/java/net/helix/pendulum/TransactionValidator.java +++ b/src/main/java/net/helix/pendulum/TransactionValidator.java @@ -14,22 +14,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.LinkedList; -import java.util.Queue; -import java.util.Set; +import java.util.*; import java.util.concurrent.atomic.AtomicBoolean; -import static net.helix.pendulum.controllers.TransactionViewModel.PREFILLED_SLOT; -import static net.helix.pendulum.controllers.TransactionViewModel.SIZE; -import static net.helix.pendulum.controllers.TransactionViewModel.VALUE_OFFSET; -import static net.helix.pendulum.controllers.TransactionViewModel.VALUE_SIZE; -import static net.helix.pendulum.controllers.TransactionViewModel.VALUE_USABLE_SIZE; -import static net.helix.pendulum.controllers.TransactionViewModel.fromHash; -import static net.helix.pendulum.controllers.TransactionViewModel.updateSolidTransactions; +import static net.helix.pendulum.controllers.TransactionViewModel.*; public class TransactionValidator { private static final Logger log = LoggerFactory.getLogger(TransactionValidator.class); @@ -154,7 +142,7 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { snapshotProvider.getInitialSnapshot().getTimestamp(), snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(transactionViewModel.getHash())); - if (transactionViewModel.getAttachmentTimestamp() == 0) { + if (transactionViewModel.getAttachmentTimestamp() == 0) { return transactionViewModel.getTimestamp() < snapshotProvider.getInitialSnapshot().getTimestamp() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(transactionViewModel.getHash()) || transactionViewModel.getTimestamp() > (System.currentTimeMillis() / 1000) + MAX_TIMESTAMP_FUTURE; } @@ -162,6 +150,17 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { || transactionViewModel.getAttachmentTimestamp() > System.currentTimeMillis() + MAX_TIMESTAMP_FUTURE_MS; } + private boolean isTransactionRequested(TransactionViewModel transactionViewModel) throws Exception { + if (transactionRequester.isTransactionRequested(transactionViewModel.getHash(), true)) { + if (TransactionViewModel.exists(tangle, transactionViewModel.getHash())) { + transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); + return false; + } + return true; + } + return false; + } + /** * Runs the following validation checks on a transaction: *
    @@ -181,16 +180,23 @@ private boolean hasInvalidTimestamp(TransactionViewModel transactionViewModel) { public void runValidation(TransactionViewModel transactionViewModel, final int minWeightMagnitude) { transactionViewModel.setMetadata(); transactionViewModel.setAttachmentData(); + + try { + if (isTransactionRequested(transactionViewModel)) { + log.error("Waiting for transaction... " + transactionViewModel.getHash()); + throw new IllegalStateException("Transaction is requested {} " + transactionViewModel.getHash()); + } + } catch (Exception e) { + throw new IllegalStateException("Transaction is requested {} " + transactionViewModel.getHash(), e); + } + if(hasInvalidTimestamp(transactionViewModel)) { log.debug("Invalid timestamp for txHash/addressHash: {} {}", transactionViewModel.getHash().toString(), transactionViewModel.getAddressHash().toString()); throw new StaleTimestampException("Invalid transaction timestamp."); } - for (int i = VALUE_OFFSET + VALUE_USABLE_SIZE; i < VALUE_OFFSET + VALUE_SIZE; i++) { // todo always false. - if (transactionViewModel.getBytes()[i] != 0) { - throw new IllegalStateException("Invalid transaction value"); - } + if(transactionViewModel.isVirtual()){ + return; } - int weightMagnitude = transactionViewModel.weightMagnitude; if((weightMagnitude < minWeightMagnitude)) { throw new IllegalStateException("Invalid transaction hash"); @@ -211,7 +217,8 @@ public void runValidation(TransactionViewModel transactionViewModel, final int m * @throws RuntimeException if validation fails */ public TransactionViewModel validateBytes(final byte[] bytes, int minWeightMagnitude) { - TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, TransactionHash.calculate(bytes, 0, bytes.length, SpongeFactory.create(SpongeFactory.Mode.S256))); + TransactionViewModel transactionViewModel = new TransactionViewModel(bytes, SpongeFactory.Mode.S256); + runValidation(transactionViewModel, minWeightMagnitude); return transactionViewModel; } @@ -264,46 +271,38 @@ public boolean checkSolidity(Hash hash, boolean milestone) throws Exception { * @throws Exception if anything goes wrong while trying to solidify the transaction */ public boolean checkSolidity(Hash hash, boolean milestone, int maxProcessedTransactions) throws Exception { - if(fromHash(tangle, hash).isSolid()) { + if (fromHash(tangle, hash).isSolid()) { return true; } Set analyzedHashes = new HashSet<>(snapshotProvider.getInitialSnapshot().getSolidEntryPoints().keySet()); - if(maxProcessedTransactions != Integer.MAX_VALUE) { + if (maxProcessedTransactions != Integer.MAX_VALUE) { maxProcessedTransactions += analyzedHashes.size(); } + log.debug("Check solidity for hash " + hash); boolean solid = true; final Queue nonAnalyzedTransactions = new LinkedList<>(Collections.singleton(hash)); Hash hashPointer; while ((hashPointer = nonAnalyzedTransactions.poll()) != null) { if (analyzedHashes.add(hashPointer)) { - if(analyzedHashes.size() >= maxProcessedTransactions) { + if (analyzedHashes.size() >= maxProcessedTransactions) { return false; } final TransactionViewModel transaction = fromHash(tangle, hashPointer); - if(!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { + if (!transaction.isSolid() && !snapshotProvider.getInitialSnapshot().hasSolidEntryPoint(hashPointer)) { if (transaction.getType() == PREFILLED_SLOT) { solid = false; if (!transactionRequester.isTransactionRequested(hashPointer, milestone)) { transactionRequester.requestTransaction(hashPointer, milestone); + solid = false; break; } } else { - // transaction of milestone bundle - TransactionViewModel milestoneTx; - if ((milestoneTx = transaction.isMilestoneBundle(tangle)) != null){ - Set parents = RoundViewModel.getMilestoneTrunk(tangle, transaction, milestoneTx); - parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transaction, milestoneTx, config.getValidatorSecurity())); - for (Hash parent : parents){ - nonAnalyzedTransactions.offer(parent); - } - } - // normal transaction - else { - nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); - nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); - } + nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); + log.debug("Check solidity for hash:" + transaction.getHash() + " trunk " + transaction.getTrunkTransactionHash()); + nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); + log.debug("Check solidity for hash:" + transaction.getHash() + " branch " + transaction.getBranchTransactionHash()); } } } @@ -410,20 +409,12 @@ protected void propagateSolidTransactions() { //what transaction we gossip. public void updateStatus(TransactionViewModel transactionViewModel) throws Exception { transactionRequester.clearTransactionRequest(transactionViewModel.getHash()); - if(transactionViewModel.getApprovers(tangle).size() == 0) { + + if (transactionViewModel.getApprovers(tangle).size() == 0 && !transactionViewModel.isVirtual()) { tipsViewModel.addTipHash(transactionViewModel.getHash()); } else { - TransactionViewModel milestoneTx; - if ((milestoneTx = transactionViewModel.isMilestoneBundle(tangle)) != null){ - Set parents = RoundViewModel.getMilestoneTrunk(tangle, transactionViewModel, milestoneTx); - parents.addAll(RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx, config.getValidatorSecurity())); - for (Hash parent : parents){ - tipsViewModel.removeTipHash(parent); - } - } else { - tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); - tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); - } + tipsViewModel.removeTipHash(transactionViewModel.getTrunkTransactionHash()); + tipsViewModel.removeTipHash(transactionViewModel.getBranchTransactionHash()); } if(quickSetSolid(transactionViewModel)) { diff --git a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java index b6763176..9c46eeeb 100644 --- a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java +++ b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java @@ -937,6 +937,11 @@ protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; } + @Override + public double getConfirmationQuorumPercentage() { + return Defaults.CONFIRMATION_QUORUM_PERCENTAGE; + } + public interface Defaults { //API int API_PORT = 8085; @@ -1059,5 +1064,8 @@ public interface Defaults { boolean SAVELOG_ENABLED = false; String SAVELOG_BASE_PATH = "./logs/"; String SAVELOG_XML_FILE = "/logback-save.xml"; + + //Consensus + double CONFIRMATION_QUORUM_PERCENTAGE = 2.0 / 3.0; } } diff --git a/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java b/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java index 1626e3d2..c38d63eb 100644 --- a/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java @@ -7,4 +7,16 @@ * the current code base and will be changed in the future */ public interface ConsensusConfig extends SnapshotConfig, MilestoneConfig, ValidatorManagerConfig { + + /** + * @return {@value PoWConfig.Descriptions#POW_THREADS} + */ + double getConfirmationQuorumPercentage(); + + /** + * Field descriptions + */ + interface Descriptions { + String CONFIRMATION_QUORUM_PERCENTAGE = "Confirmation quorum percentage"; + } } diff --git a/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java b/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java new file mode 100644 index 00000000..ad5fe0f4 --- /dev/null +++ b/src/main/java/net/helix/pendulum/controllers/BundleNonceViewModel.java @@ -0,0 +1,156 @@ +package net.helix.pendulum.controllers; + +import net.helix.pendulum.model.BundleHash; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.persistables.Bundle; +import net.helix.pendulum.model.persistables.BundleNonce; +import net.helix.pendulum.storage.Indexable; +import net.helix.pendulum.storage.Persistable; +import net.helix.pendulum.storage.Tangle; +import net.helix.pendulum.utils.Pair; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** +* The BundleViewModel class is an implementation of the HashesViewModel interface. +* It consists of an Indexable bundle hash and the Bundle model, +* which contains the set of transaction hashes that are part of the bundle. +*/ +public class BundleNonceViewModel implements HashesViewModel { + private BundleNonce self; + private Indexable hash; + + /** + * Constructor with bundle hash + * @param hash bundle hash + */ + public BundleNonceViewModel(Hash hash) { + this.hash = hash; + } + + /** + * Constructor with bundle hash and related bundle model + * @param hashes bundle model + * @param hash transaction hash + */ + private BundleNonceViewModel(BundleNonce hashes, Indexable hash) { + self = hashes == null || hashes.set == null ? new BundleNonce(): hashes; + this.hash = hash; + } + + /** + * Get the BundleNonceViewModel of a given bundle hash from the database. + * @param tangle + * @param hash bundle hash + * @return BundleNoncefViewModel + */ + public static BundleNonceViewModel load(Tangle tangle, Indexable hash) throws Exception { + return new BundleNonceViewModel((BundleNonce) tangle.load(BundleNonce.class, hash), hash); + } + + /** + * Convert a transaction set hash into the bundle model. + * @param hash transaction hash + * @param hashToMerge mergable set hash + * @return Map.Entry map entry of bundle hash and Bundle model + */ + public static Map.Entry getEntry(Hash hash, Hash hashToMerge) throws Exception { + BundleNonce hashes = new BundleNonce(); + hashes.set.add(hashToMerge); + return new HashMap.SimpleEntry<>(hash, hashes); + } + + /** + * Store the bundle hash (key) + belonging transaction hashes (value) in the database. + * @param tangle + * @return boolean success + */ + public boolean store(Tangle tangle) throws Exception { + return tangle.save(self, hash); + } + + /** + * Get the number of transactions belonging to the bundle. + * @return int number + */ + public int size() { + return self.set.size(); + } + + /** + * Add a transaction hash to the bundle. + * @param theHash transaction hash + * @return boolean success + */ + public boolean addHash(Hash theHash) { + return getHashes().add(theHash); + } + + /** + * Get the bundle hash / index. + * @return Indexable bundle hash + */ + public Indexable getIndex() { + return hash; + } + + /** + * Get the set of transaction hashes belonging to the bundle. + * @return Set transaction hashes + */ + public Set getHashes() { + return self.set; + } + + /** + * Delete the bundle from the database. + * @param tangle + */ + @Override + public void delete(Tangle tangle) throws Exception { + tangle.delete(BundleNonce.class,hash); + } + + /** + * Get the first bundle from the database. + * @param tangle + * @return bundleNonceViewModel + */ + public static BundleNonceViewModel first(Tangle tangle) throws Exception { + Pair bundlePair = tangle.getFirst(Bundle.class, BundleHash.class); + if(bundlePair != null && bundlePair.hi != null) { + return new BundleNonceViewModel((BundleNonce) bundlePair.hi, bundlePair.low); + } + return null; + } + + /** + * Get the next bundle from the database. + * @param tangle + * @return bundleNonceViewModel + */ + public BundleNonceViewModel next(Tangle tangle) throws Exception { + Pair bundlePair = tangle.next(Bundle.class, hash); + if(bundlePair != null && bundlePair.hi != null) { + return new BundleNonceViewModel((BundleNonce) bundlePair.hi, bundlePair.low); + } + return null; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + BundleNonceViewModel that = (BundleNonceViewModel) o; + return Objects.equals(self, that.self) && + Objects.equals(hash, that.hash); + } + + @Override + public int hashCode() { + return Objects.hash(self, hash); + } +} diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 8d72113d..8437df46 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -2,12 +2,20 @@ import net.helix.pendulum.TransactionValidator; import net.helix.pendulum.conf.BasePendulumConfig; -import net.helix.pendulum.crypto.Merkle; +import net.helix.pendulum.conf.PendulumConfig; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.IntegerIndex; import net.helix.pendulum.model.persistables.Round; +import net.helix.pendulum.service.ledger.LedgerException; +import net.helix.pendulum.service.ledger.LedgerService; import net.helix.pendulum.service.milestone.MilestoneTracker; +import net.helix.pendulum.service.snapshot.Snapshot; +import net.helix.pendulum.service.snapshot.SnapshotException; +import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.storage.Indexable; import net.helix.pendulum.storage.Persistable; import net.helix.pendulum.storage.Tangle; @@ -242,72 +250,76 @@ public static int getRoundIndex(TransactionViewModel milestoneTransaction) { } // todo this may be very inefficient - public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception{ + public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx) throws Exception { Set trunk = new HashSet<>(); - int round = RoundViewModel.getRoundIndex(milestoneTx); // idx = n: milestone merkle root in trunk if (transaction.getCurrentIndex() == transaction.lastIndex()) { - // add previous milestones to non analyzed transactions - RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); - if (prevMilestone == null) { - if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { - trunk.add(Hash.NULL_HASH); - } - } else { - Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { - if (prevMilestones.isEmpty()) { - trunk.add(Hash.NULL_HASH); - } else { - trunk.addAll(prevMilestones); - } - } - } - } - else { + trunk.addAll(splitMilestonesMerkleRoot(tangle, transaction.getTrunkTransactionHash(), milestoneTx)); + } else { // idx = 0 - (n-1): merkle root in branch, trunk is normal tx hash trunk.add(transaction.getTrunkTransactionHash()); } return trunk; } - public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx, int security) throws Exception{ + + public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx, int security) throws Exception { Set branch = new HashSet<>(); - int round = RoundViewModel.getRoundIndex(milestoneTx); // idx = n: milestone merkle root in trunk and tips merkle root in branch if (transaction.getCurrentIndex() == transaction.lastIndex()) { - // tips merkle root - Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { - if (confirmedTips.isEmpty()){ - branch.add(Hash.NULL_HASH); + branch.addAll(splitTipsMerkleRoot(tangle, transaction, milestoneTx, security)); + } else { + branch.addAll(splitMilestonesMerkleRoot(tangle, transaction.getBranchTransactionHash(), milestoneTx)); + } + + if (log.isTraceEnabled()) { + log.trace("Milestone branch: {}", PendulumUtils.logHashList(branch, 8)); + } + return branch; + } + + + private static Set splitMilestonesMerkleRoot(Tangle tangle, Hash expectedMilestoneMerkleRoot, TransactionViewModel milestoneTx) throws Exception { + Set result = new HashSet<>(); + int round = RoundViewModel.getRoundIndex(milestoneTx); + RoundViewModel prevMilestone = RoundViewModel.get(tangle, round - 1); + if (prevMilestone == null) { + if (Hash.NULL_HASH.equals(expectedMilestoneMerkleRoot)) { + result.add(Hash.NULL_HASH); + } + } else { + Set prevMilestones = prevMilestone.getHashes(); + MerkleTree merkle = new MerkleTreeImpl(); + Hash merkleTreeRoot = HashFactory.TRANSACTION.create(merkle.getMerkleRoot(new ArrayList<>(prevMilestones), MerkleOptions.getDefault())); + if (expectedMilestoneMerkleRoot.equals(merkleTreeRoot)) { + if (prevMilestones.isEmpty()) { + result.add(Hash.NULL_HASH); } else { - branch.addAll(confirmedTips); + result.addAll(prevMilestones); } + } else { + log.error("Milestone merkle tree can not be splitted into milestones, milestone merkle root {} for milestone {}, computed merkle root based on previous round milestones: {}", expectedMilestoneMerkleRoot, milestoneTx.getHash(), merkleTreeRoot); } } - else { - // add previous milestones to non analyzed transactions - RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); - if (prevMilestone == null) { - if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { - branch.add(Hash.NULL_HASH); - } + return result; + } + + private static Set splitTipsMerkleRoot(Tangle tangle, TransactionViewModel transaction, TransactionViewModel milestoneTx, int security) throws Exception { + Set result = new HashSet<>(); + Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); + Hash root = HashFactory.TRANSACTION.create(new MerkleTreeImpl().getMerkleRoot(new ArrayList<>(confirmedTips), MerkleOptions.getDefault())); + if (transaction.getBranchTransactionHash().equals(root)) { + if (confirmedTips.isEmpty()) { + result.add(Hash.NULL_HASH); } else { - Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { - if (prevMilestones.isEmpty()) { - branch.add(Hash.NULL_HASH); - } else { - branch.addAll(prevMilestones); - } - } + result.addAll(confirmedTips); } + } else { + //TODO check this + //confirmedTips.add(transaction.getBranchTransactionHash()); + log.error("Tips merkle tree can not be splitted into tips, tips merkle root: {} for milestone {} ", transaction.getBranchTransactionHash(), milestoneTx.getHash()); } - return branch; + return result; } public static Set getTipSet(Tangle tangle, Hash milestone, int security) throws Exception { @@ -396,17 +408,17 @@ public Set getReferencedTransactions(Tangle tangle, Set tips) throws // we can add the tx to confirmed transactions, because it is a parent of confirmedTips transactions.add(hashPointer); // traverse parents and add new candidates to queue - if(!seenTransactions.contains(transaction.getTrunkTransactionHash())){ + if (!seenTransactions.contains(transaction.getTrunkTransactionHash())) { seenTransactions.add(transaction.getTrunkTransactionHash()); nonAnalyzedTransactions.offer(transaction.getTrunkTransactionHash()); } - if(!seenTransactions.contains(transaction.getBranchTransactionHash())){ + if (!seenTransactions.contains(transaction.getBranchTransactionHash())) { seenTransactions.add(transaction.getBranchTransactionHash()); nonAnalyzedTransactions.offer(transaction.getBranchTransactionHash()); } - // roundIndex already set, i.e. tx is already confirmed. + // roundIndex already set, i.e. tx is already confirmed. } else { continue; } @@ -414,6 +426,43 @@ public Set getReferencedTransactions(Tangle tangle, Set tips) throws return transactions; } + /** + * Returns a maps of milesteons with a set of confirmed tips for each milestone + * @param tangle + * @param config Pendulum config + * @return maps of milestones + * @throws Exception + */ + public Map> getConfirmedTipsPerMilestone(Tangle tangle, PendulumConfig config) throws Exception { + + Map occurrences = new HashMap<>(); + Map> milestoneHashes = new HashMap<>(); + + int quorum = (int)(config.getConfirmationQuorumPercentage() * BasePendulumConfig.Defaults.NUMBER_OF_ACTIVE_VALIDATORS); + + for (Hash milestoneHash : getHashes()) { + Set tips = getTipSet(tangle, milestoneHash, config.getValidatorSecurity()); + + for (Hash tip : tips) { + if (occurrences.containsKey(tip)) { + occurrences.put(tip, occurrences.get(tip) + 1); + } else { + occurrences.put(tip, 1); + } + } + milestoneHashes.put(milestoneHash, tips); + } + Set tips = occurrences.entrySet().stream() + .filter(entry -> entry.getValue() >= quorum) + .map(entry -> entry.getKey()) + .collect(Collectors.toSet()); + + milestoneHashes.keySet().forEach(k -> { + milestoneHashes.get(k).stream().filter(h -> tips.contains(h)); + }); + return milestoneHashes; + } + public Hash getRandomMilestone(Tangle tangle) throws Exception { Set confirmingMilestones = getHashes(); // todo getConfirmingMilestones(tangle); if (!confirmingMilestones.isEmpty()) { @@ -460,6 +509,11 @@ public boolean addMilestone(Hash milestoneHash) { return getHashes().add(milestoneHash); } + public boolean removeMilestone(Hash milestoneHash) { + round.inconsistentMilestones.add(milestoneHash); + return getHashes().remove(milestoneHash); + } + public void update(Tangle tangle) throws Exception { tangle.update(round, round.index, "round"); } @@ -474,10 +528,41 @@ public Integer index() { return round.index.getValue(); } + /** + * Return merkle root for accepted milestone from this round. + * + * @return hash of the merkle root. + */ public Hash getMerkleRoot() { - List> merkleTree = Merkle.buildMerkleTree(new LinkedList<>(getHashes())); - Hash root = merkleTree.get(merkleTree.size()-1).get(0); - return root; + MerkleTree merkle = new MerkleTreeImpl(); + return HashFactory.TRANSACTION.create(merkle.getMerkleRoot(new LinkedList<>(getHashes()), MerkleOptions.getDefault())); + } + + public Hash getMerkleRoot(Tangle tangle, SnapshotProvider snapshotProvider, LedgerService ledgerService) { + List nonConflictualMilestones = getMilestonesToApprove(tangle, snapshotProvider, ledgerService); + return HashFactory.TRANSACTION.create( new MerkleTreeImpl().getMerkleRoot(new LinkedList<>(nonConflictualMilestones), MerkleOptions.getDefault())); + } + + public List getMilestonesToApprove(Tangle tangle, SnapshotProvider snapshotProvider, LedgerService ledgerService) { + + List targetRoundMilestones = TransactionViewModel.fromHashes(getHashes(), tangle); + Snapshot snapshot = snapshotProvider.getInitialSnapshot(); + + if (snapshot.isConsistent()) { + try { + Set approvedMilestones = new HashSet(); + ledgerService.getSnapshotService().replayMilestones(snapshot, index() - 1); + ledgerService.applyRoundToLedger(this, snapshot, approvedMilestones, true); + targetRoundMilestones.stream().filter(m -> !approvedMilestones.contains(m)).forEach(m -> { + removeMilestone(m.getHash()); + }); + update(tangle); + } catch (Exception e) { + log.error("Error during appling milestone states simultaneously", e); + } + } + + return new ArrayList<>(getHashes()); } /** diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index 5079801c..6fd2e1da 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -1,5 +1,8 @@ package net.helix.pendulum.controllers; +import net.helix.pendulum.TransactionValidator; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleNode; import net.helix.pendulum.model.*; import net.helix.pendulum.model.persistables.*; import net.helix.pendulum.service.milestone.MilestoneTracker; @@ -9,8 +12,12 @@ import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Converter; import net.helix.pendulum.utils.Pair; +import net.helix.pendulum.utils.Serializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; /** @@ -18,8 +25,9 @@ * of trunk and branch and the hash of a transaction. * The size and offset of the transaction attributes and the supply are also defined here. */ -public class TransactionViewModel { +public class TransactionViewModel implements MerkleNode { + private static final Logger log = LoggerFactory.getLogger(TransactionViewModel.class); private final Transaction transaction; public static final int SIZE = 768; @@ -148,6 +156,28 @@ public TransactionViewModel(final byte[] bytes, Hash hash) throws RuntimeExcepti transaction.type = FILLED_SLOT; } + /** + * Constructor with transaction bytes and transaction hash. + * @param bytes transaction bytes + * @param mode mode of the hash function used to compute transaction hash + */ + public TransactionViewModel(final byte[] bytes, SpongeFactory.Mode mode) throws RuntimeException { + transaction = new Transaction(); + transaction.bytes = new byte[SIZE]; + System.arraycopy(bytes, 0, transaction.bytes, 0, SIZE); + if (isVirtual()) { + byte[] buffer = new byte[2 * Hash.SIZE_IN_BYTES]; + System.arraycopy(bytes, BRANCH_TRANSACTION_OFFSET, buffer, 0, Hash.SIZE_IN_BYTES); + System.arraycopy(bytes, TRUNK_TRANSACTION_OFFSET, buffer, Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); + this.hash = TransactionHash.calculate(buffer, 0, buffer.length, SpongeFactory.create(mode)); + } else { + this.hash = TransactionHash.calculate(bytes, 0, bytes.length, SpongeFactory.create(mode)); + } + + weightMagnitude = this.hash.leadingZeros(); + transaction.type = FILLED_SLOT; + } + /** * Get the number of transactins in the database. * @param tangle @@ -157,6 +187,7 @@ public static int getNumberOfStoredTransactions(Tangle tangle) throws Exception return tangle.getCount(Transaction.class).intValue(); } + /** * This method updates the metadata contained in the {@link Transaction} object, and updates the object in the * database. First, all the most recent {@link Hash} identifiers are fetched to make sure the object's metadata is @@ -178,9 +209,9 @@ public void update(Tangle tangle, Snapshot initialSnapshot, String item) throws getBundleNonceHash(); setAttachmentData(); setMetadata(); - if (initialSnapshot.hasSolidEntryPoint(hash)) { - return; - } +// if (initialSnapshot.hasSolidEntryPoint(hash)) { +// return; +// } tangle.update(transaction, hash, item); } @@ -314,6 +345,19 @@ public boolean store(Tangle tangle, Snapshot initialSnapshot) throws Exception { return tangle.saveBatch(batch); } + + public void storeTransactionLocal(Tangle tangle, Snapshot initialSnapshot, TransactionValidator transactionValidator) throws Exception { + if(store(tangle, initialSnapshot)) { + setArrivalTime(System.currentTimeMillis() / 1000L); + if (isMilestoneBundle(tangle) == null) { + transactionValidator.updateStatus(this); + } + updateSender("local"); + update(tangle, initialSnapshot, "sender"); + } + } + + /** * Creates a copy of the underlying {@link Transaction} object. * @@ -386,6 +430,10 @@ public byte[] getBytes() { return transaction.bytes; } + public byte[] bytes(){ + return getBytes(); + } + public Hash getHash() { return hash; } @@ -487,6 +535,15 @@ public Hash getTagValue() { return transaction.tag; } + /** + * Gets the {@link Long} identifier of a {@link Transaction}. + * + * @return The {@link Long} identifier. + */ + public long getTagLongValue() { + return Serializer.getLong(getBytes(), TransactionViewModel.TAG_OFFSET); + } + /** * Gets the {@link Transaction#attachmentTimestamp}. The Attachment Timestapm is used to show when a * transaction has been attached to the database. @@ -685,6 +742,16 @@ public boolean isMilestone() { return transaction.milestone; } + /** + * The {@link Transaction#milestone} flag indicates if the {@link Transaction} is a virtual transaction + * @return true if the {@link Transaction} is virtual and false otherwise + */ + public boolean isVirtual() { + byte[] nonceForVirtual = new byte[NONCE_SIZE]; + Arrays.fill(nonceForVirtual, (byte) 0xff); + return Arrays.equals(getNonce(),nonceForVirtual); + } + public TransactionViewModel isMilestoneBundle(Tangle tangle) throws Exception{ for (Hash bundleTx : BundleViewModel.load(tangle, getBundleHash()).getHashes()){ TransactionViewModel tx = TransactionViewModel.fromHash(tangle, bundleTx); @@ -746,6 +813,18 @@ public String getSender() { return transaction.sender; } + public static List fromHashes(Set hashes, Tangle tangle) { + return hashes.stream().map( + h -> { + try { + return TransactionViewModel.fromHash(tangle, h); + } catch (Exception e) { + log.error("Could not get transaction for hash " + h, e); + } + return null; + }).filter(t -> t != null).collect(Collectors.toList()); + } + @Override public boolean equals(Object o) { if (this == o) { diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java deleted file mode 100644 index aea7ffc2..00000000 --- a/src/main/java/net/helix/pendulum/crypto/Merkle.java +++ /dev/null @@ -1,193 +0,0 @@ -package net.helix.pendulum.crypto; - -import net.helix.pendulum.controllers.RoundViewModel; -import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.model.Hash; -import net.helix.pendulum.model.HashFactory; -import org.bouncycastle.util.encoders.Hex; - -import java.io.*; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -public class Merkle { - - public static byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { - int index = indexIn; - final Sponge sha3 = SpongeFactory.create(mode); - for (int i = 0; i < size; i++) { - sha3.reset(); - if ((index & 1) == 0) { - sha3.absorb(hash, 0, hash.length); - sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); - } else { - sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); - sha3.absorb(hash, 0, hash.length); - } - sha3.squeeze(hash, 0, hash.length); - index >>= 1; - } - if(index != 0) { - return Hash.NULL_HASH.bytes(); - } - return hash; - } - - public static List getMerklePath(List> merkleTree, int keyIndex){ - List merklePath = new ArrayList<>((merkleTree.size()-1) * 32); - for (int i = 0; i < merkleTree.size()-1; i++) { - Hash subkey = merkleTree.get(i).get(keyIndex ^ 1); - merklePath.add(subkey == null ? Hash.NULL_HASH : subkey); - keyIndex /= 2; - } - return merklePath; - } - - public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security){ - List keys = new ArrayList<>(1 << pubkeyDepth); - for (int i = 0; i < pubkeyCount; i++) { - int idx = firstIndex + i; - keys.add(HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security))); - } - return buildMerkleTree(keys); - } - - - public static List> buildMerkleTree(List leaves){ - if (leaves.isEmpty()) { - leaves.add(Hash.NULL_HASH); - } - byte[] buffer; - Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); - int depth = (int) Math.ceil(Math.sqrt(leaves.size())); - List> merkleTree = new ArrayList<>(depth + 1); - merkleTree.add(0, leaves); - int row = 1; - // hash two following keys together until only one is left -> merkle tree - while (leaves.size() > 1) { - // Take two following keys (i=0: (k0,k1), i=1: (k2,k3), ...) and get one crypto of them - List nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]); - for (int i = 0; i < nextKeys.size(); i++) { - if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { - // leave the combined key null as well - continue; - } - sha3.reset(); - Hash k1 = leaves.get(i * 2); - Hash k2 = leaves.get(i * 2 + 1); - buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32); - sha3.absorb(buffer, 0, buffer.length); - buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32); - sha3.absorb(buffer, 0, buffer.length); - sha3.squeeze(buffer, 0, buffer.length); - nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); - } - leaves = nextKeys; - merkleTree.add(row++, leaves); - } - return merkleTree; - } - - public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) { - - final TransactionViewModel merkleTx = bundleTransactionViewModels.get(securityLevel); - int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex - - //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) - //TODO: check if its okay here to use bundle hash instead of tx hash - byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); - - //validate leaf signature - ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * securityLevel); - - for (int i = 0; i < securityLevel; i++) { - byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i+1)); - byte[] digest = Winternitz.digest(mode, bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); - bb.put(digest); - } - - byte[] digests = bb.array(); - byte[] address = Winternitz.address(mode, digests); - - //validate Merkle path - byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, - merkleTx.getSignature(), 0, keyIndex, depth); - return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); - } - - public static List> readKeyfile(File keyfile) throws IOException { - try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { - String[] fields = br.readLine().split(" "); - int depth = Integer.parseInt(fields[0]); - List> result = new ArrayList<>(depth + 1); - for (int i = 0; i <= depth; i++) { - fields = br.readLine().split(" "); - int leadingNulls = Integer.parseInt(fields[0]); - List row = new ArrayList<>(); - for (int j = 0; j < leadingNulls; j++) { - row.add(Hash.NULL_HASH); - } - for (int j = 0; j < fields[1].length() / 64; j++) { - row.add(HashFactory.TRANSACTION.create(fields[1].substring(j * 64, (j+1) * 64))); - } - result.add(row); - } - return result; - } - } - - public static String getSeed(File keyfile) throws IOException { - StringBuilder seedBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { - String[] fields = br.readLine().split(" "); - seedBuilder.append(fields[1]); - } - return seedBuilder.toString(); - } - - - public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { - // fill buffer - try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { - // write pubkey depth and seed into buffer - bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); - bw.newLine(); - writeKeys(bw, merkleTree.get(0)); - for (int i = 1; i < merkleTree.size(); i++) { - writeKeys(bw, merkleTree.get(i)); - } - } - } - - private static void writeKeys(BufferedWriter bw, List keys) throws IOException { - int leadingNulls = 0; - while (keys.get(leadingNulls) == null) { - leadingNulls++; - } - bw.write(leadingNulls + " "); - for (int i = leadingNulls; i < keys.size(); i++) { - if (keys.get(i) == null) { - break; - } - bw.write(keys.get(i).toString()); - } - bw.newLine(); - } - - public static byte[] padding(byte[] input, int length){ - if (input.length < length) { - byte[] output = new byte[length]; - System.arraycopy(input, 0, output, length - input.length, input.length); - return output; - } else { - if (input.length > length) { - return Arrays.copyOfRange(input, 0, length); - } else { - return Arrays.copyOfRange(input, 0, input.length); - } - - } - } -} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleNode.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleNode.java new file mode 100644 index 00000000..1d6bd4fb --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleNode.java @@ -0,0 +1,9 @@ +package net.helix.pendulum.crypto.merkle; + +/** + * Used as node in merkle tree + */ +public interface MerkleNode { + + byte[] bytes(); +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java new file mode 100644 index 00000000..54e330a7 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java @@ -0,0 +1,87 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.model.Hash; + +public class MerkleOptions { + + private boolean includeLeavesInTree; + private Hash milestoneHash; + private Hash address; + private SpongeFactory.Mode mode; + private int securityLevel; + private int depth; + + /** + * Parameters needed fro validation + * @param mode + * @param address + * @param securityLevel + * @param depth + */ + public MerkleOptions(SpongeFactory.Mode mode, Hash address, int securityLevel, int depth) { + this.address = address; + this.mode = mode; + this.securityLevel = securityLevel; + this.depth = depth; + this.includeLeavesInTree = true; + } + + public MerkleOptions() { + } + + public static MerkleOptions getDefault() { + MerkleOptions op = new MerkleOptions(); + op.includeLeavesInTree = true; + op.mode = SpongeFactory.Mode.S256; + return op; + } + + public boolean isIncludeLeavesInTree() { + return includeLeavesInTree; + } + + public void setIncludeLeavesInTree(boolean includeLeavesInTree) { + this.includeLeavesInTree = includeLeavesInTree; + } + + public Hash getMilestoneHash() { + return milestoneHash; + } + + public void setMilestoneHash(Hash milestoneHash) { + this.milestoneHash = milestoneHash; + } + + public Hash getAddress() { + return address; + } + + public void setAddress(Hash address) { + this.address = address; + } + + public SpongeFactory.Mode getMode() { + return mode; + } + + public void setMode(SpongeFactory.Mode mode) { + this.mode = mode; + } + + public int getSecurityLevel() { + return securityLevel; + } + + public void setSecurityLevel(int securityLevel) { + this.securityLevel = securityLevel; + } + + public int getDepth() { + return depth; + } + + public void setDepth(int depth) { + this.depth = depth; + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java new file mode 100644 index 00000000..239a1842 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java @@ -0,0 +1,78 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.model.Hash; + +import java.util.Arrays; +import java.util.List; + +public interface MerkleTree { + + /** + * Gets merkle path from the specified key index (Used in merkle transaction message + * @param merkleTree + * @param keyIndex + * @return + */ + List getMerklePath(List> merkleTree, int keyIndex); + + /** + * Build flat merkle tree + * @param leaves hash leaves + * @param options + * @return + */ + List buildMerkle(List leaves, MerkleOptions options); + + /** + * Build merkle tree as lists of lists + * @param leaves + * @param options + * @return + */ + List> buildMerkleTree(List leaves, MerkleOptions options); + + /** + * Gets merkle tree root for given leaves + * @param leaves + * @param options + * @return + */ + byte[] getMerkleRoot(List leaves, MerkleOptions options); + + /** + * Get merkle tree root, based on merkle path + * @param mode + * @param hash + * @param bytes merkle path bytes + * @param offset + * @param indexIn + * @param size of elements in path + * @return + */ + byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, int indexIn, int size); + + /** + * Validate bundles which contains merkle transaction + * First it validates sender transactions signature + * @param bundleTransactionViewModels + * @param options + * @return + */ + boolean validateMerkleSignature(List bundleTransactionViewModels, MerkleOptions options); + + static byte[] padding(byte[] input, int length) { + if (input.length < length) { + byte[] output = new byte[length]; + System.arraycopy(input, 0, output, length - input.length, input.length); + return output; + } else { + if (input.length > length) { + return Arrays.copyOfRange(input, 0, length); + } else { + return Arrays.copyOfRange(input, 0, input.length); + } + } + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java new file mode 100644 index 00000000..593bc8fa --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/AbstractMerkleTree.java @@ -0,0 +1,229 @@ +package net.helix.pendulum.crypto.merkle.impl; + +import net.helix.pendulum.controllers.RoundViewModel; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Sha3; +import net.helix.pendulum.crypto.Sponge; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +public abstract class AbstractMerkleTree implements MerkleTree { + + protected abstract MerkleNode createMerkleNode(Hash hashParent, Hash h1, Hash h2, int row, long index, MerkleOptions options); + + protected abstract List createMerkleNodes(List leaves, MerkleOptions options); + + @Override + public byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { + int index = indexIn; + final Sponge sha3 = SpongeFactory.create(mode); + for (int i = 0; i < size; i++) { + sha3.reset(); + if ((index & 1) == 0) { + sha3.absorb(hash, 0, hash.length); + sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); + } else { + sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); + sha3.absorb(hash, 0, hash.length); + } + sha3.squeeze(hash, 0, hash.length); + index >>= 1; + } + if (index != 0) { + return getDefaultMerkleHash().bytes(); + } + return hash; + } + + @Override + public List getMerklePath(List> merkleTree, int keyIndex) { + List merklePath = new ArrayList<>((merkleTree.size() - 1) * Hash.SIZE_IN_BYTES); + for (int i = 0; i < merkleTree.size() - 1; i++) { + MerkleNode subkey = merkleTree.get(i).get(keyIndex ^ 1); + merklePath.add(subkey == null ? getDefaultMerkleHash() : subkey); + keyIndex /= 2; + } + return merklePath; + } + + @Override + public byte[] getMerkleRoot(List leaves, MerkleOptions options) { + List> merkleTree = buildMerkleTree(leaves, options); + return (merkleTree.get(merkleTree.size() - 1).get(0)).bytes(); + } + + @Override + public List buildMerkle(List leaves, MerkleOptions options) { + if (leaves.isEmpty()) { + leaves.add(getDefaultMerkleHash()); + } + byte[] buffer; + Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); + int row = 1; + addPaddingLeaves(leaves); + sortLeaves(leaves); + List merkleNodes = new ArrayList<>(); + int depth = getTreeDepth(leaves.size()); + while (leaves.size() > 1) { + List nextKeys = Arrays.asList(new Hash[getParentNodesSize(leaves)]); + for (int i = 0; i < nextKeys.size(); i++) { + if (areLeavesNull(leaves, i)) { + continue; + } + sha3.reset(); + Hash k1 = getLeaves(leaves, i * 2); + Hash k2 = getLeaves(leaves, i * 2 + 1); + buffer = computeParentHash(sha3, k1, k2); + Hash parentHash = HashFactory.TRANSACTION.create(buffer); + nextKeys.set(i, parentHash); + merkleNodes.add(createMerkleNode(parentHash, k1, k2, row, getParentMerkleIndex(row, depth, i * 2), options)); + } + leaves = nextKeys; + row++; + } + return merkleNodes; + } + + private void sortLeaves(List leaves) { + leaves.sort(Comparator.comparing((Hash m) -> m.toString())); + } + + @Override + public List> buildMerkleTree(List leaves, MerkleOptions options) { + if (leaves.isEmpty()) { + leaves.add(getDefaultMerkleHash()); + } + byte[] buffer; + addPaddingLeaves(leaves); + sortLeaves(leaves); + Sponge sha3 = SpongeFactory.create(options.getMode()); + List> merkleTree = new ArrayList<>(); + + if (options.isIncludeLeavesInTree()) { + merkleTree.add(0, createMerkleNodes(leaves, options)); + } + int row = 1; + while (leaves.size() > 1) { + List nextKeys = Arrays.asList(new Hash[getParentNodesSize(leaves)]); + for (int i = 0; i < nextKeys.size(); i++) { + if (areLeavesNull(leaves, i)) { + continue; + } + sha3.reset(); + Hash k1 = getLeaves(leaves, i * 2); + Hash k2 = getLeaves(leaves, i * 2 + 1); + buffer = computeParentHash(sha3, k1, k2); + nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); + } + leaves = nextKeys; + merkleTree.add(row++, createMerkleNodes(leaves, options)); + } + return merkleTree; + } + + public boolean validateMerkleSignature(List bundleTransactionViewModels, MerkleOptions options) { + + final TransactionViewModel merkleTx = bundleTransactionViewModels.get(options.getSecurityLevel()); + int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex + + //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) + //TODO: check if its okay here to use bundle hash instead of tx hash + byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); + + //validate leaf signature + ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * options.getSecurityLevel()); + + for (int i = 0; i < options.getSecurityLevel(); i++) { + byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i + 1)); + byte[] digest = Winternitz.digest(options.getMode(), bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); + bb.put(digest); + } + + byte[] digests = bb.array(); + byte[] address = Winternitz.address(options.getMode(), digests); + + return validateMerklePath(merkleTx.getSignature(), keyIndex, address, options); + } + + private boolean validateMerklePath(byte[] path, int keyIndex, byte[] address, MerkleOptions options) { + byte[] merkleRoot = getMerkleRoot(options.getMode(), address, + path, 0, keyIndex, options.getDepth()); + return HashFactory.ADDRESS.create(merkleRoot).equals(options.getAddress()); + } + + protected void addPaddingLeaves(List leaves){ + int closestPow = (int) Math.pow(2.0, getClosestPow(leaves.size())); + int leaveSize = leaves.size(); + while(leaveSize < closestPow){ + leaves.add(leaveSize++, Hash.NULL_HASH); + } + } + + protected int getTreeDepth(int leavesNumber) { + return getClosestPow(leavesNumber); + } + + protected int getClosestPow(int i) { + int j = 1; + int power = 0; + while (j < i) { + j = j << 1; + power++; + } + return power; + } + + protected Hash getLeaves(List leaves, int index) { + return index < leaves.size() ? leaves.get(index) : getDefaultMerkleHash(); + } + + protected static long getParentMerkleIndex(int row, int depth, int i) { + if (row == depth) { + return 0; + } + long index = depth - row; + return (long) Math.pow(2, index) + i / 2 - 1; + } + + protected int getParentNodesSize(List leaves) { + return leaves.size() % 2 == 0 ? (leaves.size() / 2) : (leaves.size() / 2 + 1); + } + + private boolean areLeavesNull(List leaves, int i) { + if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { + return true; + } + return false; + } + + private byte[] computeParentHash(Sponge sha3, byte[] k1, byte[] k2) { + byte[] buffer = new byte[Hash.SIZE_IN_BYTES]; + sha3.absorb(k1, 0, k1.length); + sha3.absorb(k2, 0, k2.length); + sha3.squeeze(buffer, 0, buffer.length); + return buffer; + } + + private byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { + return computeParentHash(sha3, copyHash(k1), copyHash(k2)); + } + + private byte[] copyHash(Hash k2) { + return Arrays.copyOfRange(k2 == null ? getDefaultMerkleHash().bytes() : k2.bytes(), 0, Hash.SIZE_IN_BYTES); + } + + private Hash getDefaultMerkleHash() { + return Hash.NULL_HASH; + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java new file mode 100644 index 00000000..abb9c96b --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java @@ -0,0 +1,23 @@ +package net.helix.pendulum.crypto.merkle.impl; + +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.model.Hash; + +import java.util.ArrayList; +import java.util.List; + +public class MerkleTreeImpl extends AbstractMerkleTree { + + @Override + protected MerkleNode createMerkleNode(Hash hashParent, Hash h1, Hash h2, int row, long index, MerkleOptions options) { + return hashParent; + } + + @Override + protected List createMerkleNodes(List leaves, MerkleOptions options) { + List result = new ArrayList<>(); + result.addAll(leaves); + return result; + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/TransactionMerkleTreeImpl.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/TransactionMerkleTreeImpl.java new file mode 100644 index 00000000..d39246d4 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/TransactionMerkleTreeImpl.java @@ -0,0 +1,35 @@ +package net.helix.pendulum.crypto.merkle.impl; + +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.utils.bundle.BundleUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; + +public class TransactionMerkleTreeImpl extends AbstractMerkleTree { + + private static final Logger log = LoggerFactory.getLogger(TransactionMerkleTreeImpl.class); + + @Override + protected MerkleNode createMerkleNode(Hash parentHash, Hash h1, Hash h2, int row, long index, MerkleOptions options) { + log.debug("New virtual transaction + " + parentHash + " for milestone: " + options.getMilestoneHash() + " with merkle index: " + index + " [" + h1 + ", " + h2 + " ]"); + byte[] virtualTransaction = BundleUtils.createVirtualTransaction(h1, h2, index, options.getMilestoneHash().bytes(), options.getAddress()); + return new TransactionViewModel(virtualTransaction, options.getMode()); + } + + @Override + protected List createMerkleNodes(List leaves, MerkleOptions options) { + + int depth = getTreeDepth(leaves.size()); + List result = new ArrayList<>(); + for (int i = 0; i < leaves.size(); i++) { + result.add(createMerkleNode(leaves.get(i), Hash.NULL_HASH, Hash.NULL_HASH, 0, getParentMerkleIndex(0, depth, i), options)); + } + return result; + } +} diff --git a/src/main/java/net/helix/pendulum/model/Hash.java b/src/main/java/net/helix/pendulum/model/Hash.java index 4fa078dc..e40de872 100644 --- a/src/main/java/net/helix/pendulum/model/Hash.java +++ b/src/main/java/net/helix/pendulum/model/Hash.java @@ -1,6 +1,7 @@ package net.helix.pendulum.model; import net.helix.pendulum.crypto.Sha3; +import net.helix.pendulum.crypto.merkle.MerkleNode; import net.helix.pendulum.storage.Indexable; @@ -9,7 +10,7 @@ * The model class contains a hash Hash , the size of the hash, lock * and the inner classes ByteSafe */ -public interface Hash extends Indexable, HashId { +public interface Hash extends Indexable, HashId, MerkleNode { /** * Creates a null transaction hash with from a byte array of length {@value Sha3#HASH_LENGTH}. diff --git a/src/main/java/net/helix/pendulum/model/persistables/Round.java b/src/main/java/net/helix/pendulum/model/persistables/Round.java index 84ee065f..d4d17fb0 100644 --- a/src/main/java/net/helix/pendulum/model/persistables/Round.java +++ b/src/main/java/net/helix/pendulum/model/persistables/Round.java @@ -1,15 +1,21 @@ package net.helix.pendulum.model.persistables; +import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.IntegerIndex; import net.helix.pendulum.utils.Serializer; import org.apache.commons.lang3.ArrayUtils; - /** +import java.util.LinkedHashSet; +import java.util.Set; + +/** * The Round model class consists of a set of milestone hashes and a corresponding index. */ public class Round extends Hashes { public IntegerIndex index; + public Set inconsistentMilestones = new LinkedHashSet<>(); + @Override public byte[] bytes() { return ArrayUtils.addAll(index.bytes(), super.bytes()); diff --git a/src/main/java/net/helix/pendulum/network/Node.java b/src/main/java/net/helix/pendulum/network/Node.java index 20440e53..f72d5632 100644 --- a/src/main/java/net/helix/pendulum/network/Node.java +++ b/src/main/java/net/helix/pendulum/network/Node.java @@ -13,6 +13,7 @@ import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.TransactionHash; import net.helix.pendulum.service.milestone.MilestoneTracker; +import net.helix.pendulum.service.milestone.VirtualTransactionService; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.storage.Tangle; import org.apache.commons.lang3.StringUtils; @@ -75,6 +76,7 @@ public class Node { private final TipsViewModel tipsViewModel; private final TransactionValidator transactionValidator; private final TransactionRequester transactionRequester; + private final VirtualTransactionService virtualTransactionService; private static final SecureRandom rnd = new SecureRandom(); @@ -109,7 +111,8 @@ public class Node { * @param configuration Contains all the config. * */ - public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final MilestoneTracker milestoneTracker, final NodeConfig configuration + public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final TransactionValidator transactionValidator, final TransactionRequester transactionRequester, final TipsViewModel tipsViewModel, final MilestoneTracker milestoneTracker, + final VirtualTransactionService virtualTransactionService, final NodeConfig configuration ) { this.configuration = configuration; this.tangle = tangle; @@ -121,7 +124,7 @@ public Node(final Tangle tangle, SnapshotProvider snapshotProvider, final Transa int packetSize = configuration.getTransactionPacketSize(); this.sendingPacket = new DatagramPacket(new byte[packetSize], packetSize); this.tipRequestingPacket = new DatagramPacket(new byte[packetSize], packetSize); - + this.virtualTransactionService = virtualTransactionService; } /** @@ -313,6 +316,7 @@ public void preProcessReceivedData(byte[] receivedData, SocketAddress senderAddr } //if valid - add to receive queue (receivedTransactionViewModel, neighbor) + buildVirtualTransaction(receivedTransactionViewModel); addReceivedDataToReceiveQueue(receivedTransactionViewModel, neighbor); } @@ -400,6 +404,12 @@ public void preProcessReceivedData(byte[] receivedData, SocketAddress senderAddr } } + public void buildVirtualTransaction(TransactionViewModel model){ + if(model.isVirtual()){ + virtualTransactionService.rebuildVirtualTransactionsIfPossible(model); + } + } + /** * Adds incoming transactions to the {@link Node#receiveQueue} to be processed later. */ diff --git a/src/main/java/net/helix/pendulum/network/TransactionRequester.java b/src/main/java/net/helix/pendulum/network/TransactionRequester.java index de522101..5c56a2e5 100644 --- a/src/main/java/net/helix/pendulum/network/TransactionRequester.java +++ b/src/main/java/net/helix/pendulum/network/TransactionRequester.java @@ -90,7 +90,8 @@ public void requestTransaction(Hash hash, boolean milestone) throws Exception { protected void popEldestTransactionToRequest() { Iterator iterator = transactionsToRequest.iterator(); if (iterator.hasNext()) { - iterator.next(); + Hash transactionHash = iterator.next(); + log.debug("Transaction: " + transactionHash + " has been removed from requestList!"); iterator.remove(); } } diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 449cd9e7..cec6a324 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -11,7 +11,14 @@ import net.helix.pendulum.conf.BasePendulumConfig; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.*; -import net.helix.pendulum.crypto.*; +import net.helix.pendulum.crypto.GreedyMiner; +import net.helix.pendulum.crypto.Sha3; +import net.helix.pendulum.crypto.Sponge; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; +import net.helix.pendulum.crypto.merkle.impl.TransactionMerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.persistables.Transaction; @@ -26,6 +33,7 @@ import net.helix.pendulum.service.spentaddresses.SpentAddressesService; import net.helix.pendulum.service.tipselection.TipSelector; import net.helix.pendulum.service.tipselection.impl.WalkValidatorImpl; +import net.helix.pendulum.service.utils.RoundIndexUtil; import net.helix.pendulum.service.validatormanager.CandidateTracker; import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Serializer; @@ -116,6 +124,7 @@ public class API { private final String[] features; + //endregion //////////////////////////////////////////////////////////////////////////////////////////////////////// private final Gson gson = new GsonBuilder().create(); @@ -178,7 +187,6 @@ public API(ApiArgs args) { commandRoute.put(ApiCommand.GET_MISSING_TRANSACTIONS, getMissingTransactions()); commandRoute.put(ApiCommand.CHECK_CONSISTENCY, checkConsistency()); commandRoute.put(ApiCommand.WERE_ADDRESSES_SPENT_FROM, wereAddressesSpentFrom()); - // commandRoute.put(ApiCommand.GET_MILESTONES, wereAddressesSpentFrom()); } /** @@ -403,7 +411,8 @@ private AbstractResponse checkConsistencyStatement(List transactionsList * @return false if we received at least a solid milestone, otherwise true */ public boolean invalidSubtangleStatus() { - return (snapshotProvider.getLatestSnapshot().getIndex() == snapshotProvider.getInitialSnapshot().getIndex()); + return (snapshotProvider.getLatestSnapshot().getIndex() == 0 && + snapshotProvider.getLatestSnapshot().getIndex() == snapshotProvider.getInitialSnapshot().getIndex()); } /** * Returns the set of neighbors you are connected with, as well as their activity statistics (or counters). @@ -591,19 +600,7 @@ private synchronized AbstractResponse getTipsStatement() throws Exception { public void storeTransactionsStatement(final List txString) throws Exception { final List elements = addValidTxvmToList(txString); for (final TransactionViewModel transactionViewModel : elements) { - //store transactions - if(transactionViewModel.store(tangle, snapshotProvider.getInitialSnapshot())) { - transactionViewModel.setArrivalTime(System.currentTimeMillis()); - if (transactionViewModel.isMilestoneBundle(tangle) == null) { - transactionValidator.updateStatus(transactionViewModel); - } - transactionViewModel.updateSender("local"); - transactionViewModel.update(tangle, snapshotProvider.getInitialSnapshot(), "sender"); - if (tangle != null) { - tangle.publish("vis %s %s %s", transactionViewModel.getHash(), transactionViewModel.getTrunkTransactionHash(), transactionViewModel.getBranchTransactionHash()); - } - log.trace("Stored_txhash = {}", transactionViewModel.getHash().toString()); - } + transactionViewModel.storeTransactionLocal(tangle, snapshotProvider.getInitialSnapshot(), transactionValidator); } } @@ -702,7 +699,7 @@ private AbstractResponse getConfirmationStatesStatement(final List trans log.trace("tx_confirmations {}:[{}:{}]", transaction.getHash().toString(), transaction.getConfirmations(), (double) transaction.getConfirmations() / n); // is transaction finalized - if(((double)transaction.getConfirmations() / n) > threshold) { + if(transaction.getRoundIndex() > 0 && ((double)transaction.getConfirmations() / n) > threshold) { confirmationStates[count] = 1; } // not finalized yet @@ -787,7 +784,7 @@ private AbstractResponse getConfirmationStatesStatementObsolete(final List transactions, final List tips) throws Exception { + public AbstractResponse getInclusionStatesStatement(final List transactions, final List tips) throws Exception { final List trans = transactions.stream() .map(HashFactory.TRANSACTION::create) @@ -1128,6 +1125,14 @@ private AbstractResponse getBalancesStatement(List addresses, int thresh snapshotProvider.getLatestSnapshot().lockRead(); final int index = snapshotProvider.getLatestSnapshot().getIndex(); + if (tips == null || tips.size() == 0) { + RoundViewModel round = RoundViewModel.get(tangle, index); + hashes = round != null ? new LinkedList<>(round.getHashes()) : new ArrayList<>(); + } else { + hashes = tips.stream() + .map(tip -> (HashFactory.TRANSACTION.create(tip))) + .collect(Collectors.toCollection(LinkedList::new)); + } try { // Get the balance for each address at the last snapshot for (final Hash address : addressList) { @@ -1247,12 +1252,7 @@ public synchronized List attachToTangleStatement(final Hash trunkTransac if(IntStream.range(TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_OFFSET + TransactionViewModel.TAG_SIZE).allMatch(idx -> txBytes[idx] == ((byte) 0))) { System.arraycopy(txBytes, TransactionViewModel.BUNDLE_NONCE_OFFSET, txBytes, TransactionViewModel.TAG_OFFSET, TransactionViewModel.TAG_SIZE); } - System.arraycopy(Serializer.serialize(timestamp),0, txBytes,TransactionViewModel.ATTACHMENT_TIMESTAMP_OFFSET, - TransactionViewModel.ATTACHMENT_TIMESTAMP_SIZE); - System.arraycopy(Serializer.serialize(0L),0,txBytes, TransactionViewModel.ATTACHMENT_TIMESTAMP_LOWER_BOUND_OFFSET, - TransactionViewModel.ATTACHMENT_TIMESTAMP_LOWER_BOUND_SIZE); - System.arraycopy(Serializer.serialize(MAX_TIMESTAMP_VALUE),0,txBytes,TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_OFFSET, - TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_SIZE); + fillAttachmentTransactionFields(txBytes, timestamp); if (!miner.mine(txBytes, minWeightMagnitude, 4)) { transactionViewModels.clear(); @@ -1285,6 +1285,15 @@ public synchronized List attachToTangleStatement(final Hash trunkTransac return elements; } + private void fillAttachmentTransactionFields(byte[] txBytes, long timestamp) { + System.arraycopy(Serializer.serialize(timestamp),0, txBytes, TransactionViewModel.ATTACHMENT_TIMESTAMP_OFFSET, + TransactionViewModel.ATTACHMENT_TIMESTAMP_SIZE); + System.arraycopy(Serializer.serialize(0L),0,txBytes, TransactionViewModel.ATTACHMENT_TIMESTAMP_LOWER_BOUND_OFFSET, + TransactionViewModel.ATTACHMENT_TIMESTAMP_LOWER_BOUND_SIZE); + System.arraycopy(Serializer.serialize(MAX_TIMESTAMP_VALUE),0,txBytes,TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_OFFSET, + TransactionViewModel.ATTACHMENT_TIMESTAMP_UPPER_BOUND_SIZE); + } + /** * Can be 0 or more, and is set to 0 every 100 requests. * Each increase indicates another 2 tips send. @@ -1520,11 +1529,11 @@ private void attachStoreAndBroadcast(final String address, final String message, * @param txs transactions list * @throws Exception if storing fails */ - private void storeAndBroadcast(Hash tip1, Hash tip2, int mwm, List txs) throws Exception{ + private List attachAndStore(Hash tip1, Hash tip2, int mwm, List txs) throws Exception{ List powResult = attachToTangleStatement(tip1, tip2, mwm, txs); log.trace("tips = [{}, {}]", tip1.toString(), tip2.toString()); storeTransactionsStatement(powResult); - broadcastTransactionsStatement(powResult); + return powResult; } // @@ -1569,10 +1578,90 @@ public void publish(BundleTypes type, final String address, final int minWeightM * @param txToApprove transactions to approve * @throws Exception if storing fails */ - private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, String keyfile, int security) throws Exception { + private void storeCustomBundle(final Hash sndAddr, final Hash rcvAddr, List txToApprove, byte[] data, final long tag, final int mwm, boolean sign, int keyIdx, int maxKeyIdx, String keyfile, int security, boolean createVirtualTransactions) throws Exception { BundleUtils bundle = new BundleUtils(sndAddr, rcvAddr); bundle.create(data, tag, sign, keyIdx, maxKeyIdx, keyfile, security); - storeAndBroadcast(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); + List transactionStrings = attachAndStore(txToApprove.get(0), txToApprove.get(1), mwm, bundle.getTransactions()); + + broadcastTransactionsStatement(transactionStrings); + if (createVirtualTransactions) { + createAndBroadcastVirtualTransactions(data, transactionStrings); + } + } + + /** + * Creates and broadcast virtual transactions + * + * @param tipsBytes + * @param milestoneBundle + */ + private void createAndBroadcastVirtualTransactions(byte[] tipsBytes, List milestoneBundle) throws Exception { + TransactionViewModel milestone = getFirstTransactionFromBundle(milestoneBundle); + if (milestone != null) { + List tips = extractTipsFromData(tipsBytes); + if (tips.size() == 0) { + return; + } + MerkleOptions options = MerkleOptions.getDefault(); + options.setMilestoneHash(milestone.getHash()); + options.setAddress(milestone.getAddressHash()); + + RoundViewModel previousRound = RoundViewModel.get(tangle, (int)milestone.getTagLongValue() - 1); + + List merkleTransactions = new TransactionMerkleTreeImpl().buildMerkle(tips, options); + if(previousRound != null) { + merkleTransactions.addAll(new TransactionMerkleTreeImpl().buildMerkle(new ArrayList<>(previousRound.getHashes()), options)); + } + + List virtualTransactions = merkleTransactions.stream().map(t -> { + try { + TransactionViewModel tvm = (TransactionViewModel)t; + fillAttachmentTransactionFields(tvm.getBytes(), RoundIndexUtil.getCurrentTime()); + tvm.storeTransactionLocal(tangle,snapshotProvider.getInitialSnapshot(),transactionValidator); + return Hex.toHexString(tvm.getBytes()); + } catch (Exception e) { + log.error("Could not generated virtual transaction for " + ((TransactionViewModel) t).getHash()); + } + return null; + }).collect(Collectors.toList()); + broadcastTransactionsStatement(virtualTransactions); + } + } + + /** + * Get transaction with index = 0 from a transaction list + * + * @param bundle + * @return TransactionViewModel with current index = 0, otherwise null + */ + private TransactionViewModel getFirstTransactionFromBundle(List bundle) { + for (String transactionString : bundle) { + final TransactionViewModel transactionViewModel = transactionValidator.validateBytes(Hex.decode(transactionString), + transactionValidator.getMinWeightMagnitude()); + if (transactionViewModel.getCurrentIndex() == 0) { + return transactionViewModel; + } + } + return null; + } + + /** + * Extract tips list from data byte array + * + * @param data + * @return + */ + private List extractTipsFromData(byte[] data) { + List tips = new ArrayList<>(); + int i = 0; + while (i < data.length / Hash.SIZE_IN_BYTES) { + Hash tip = HashFactory.TRANSACTION.create(data, i++ * Hash.SIZE_IN_BYTES, Hash.SIZE_IN_BYTES); + if (tip.equals(Hash.NULL_HASH)) { + break; + } + tips.add(tip); + } + return tips; } // refactoring WIP (the following methods will be moved from API) @@ -1592,7 +1681,8 @@ public void publishMilestone(final String address, final int minWeightMagnitude, byte[] tipsBytes = Hex.decode(confirmedTips.stream().map(Hash::toString).collect(Collectors.joining())); List txToApprove = addMilestoneReferences(confirmedTips, currentRoundIndex); - storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity()); + + storeCustomBundle(HashFactory.ADDRESS.create(address), Hash.NULL_HASH, txToApprove, tipsBytes, (long) currentRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity(), true); } public void publishRegistration(final String address, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex, boolean join) throws Exception { @@ -1601,7 +1691,7 @@ public void publishRegistration(final String address, final int minWeightMagnitu List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getValidatorManagerAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity()); + storeCustomBundle(HashFactory.ADDRESS.create(address), configuration.getValidatorManagerAddress(), txToApprove, data, join ? 1L : -1L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity(), false); } public void publishKeyChange(final String oldAddress, final Hash newAddress, final int minWeightMagnitude, boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1610,7 +1700,7 @@ public void publishKeyChange(final String oldAddress, final Hash newAddress, fin List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - storeCustomBundle(HashFactory.ADDRESS.create(oldAddress), configuration.getValidatorManagerAddress(), txToApprove, data, 0L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity()); + storeCustomBundle(HashFactory.ADDRESS.create(oldAddress), configuration.getValidatorManagerAddress(), txToApprove, data, 0L, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorKeyfile(), configuration.getValidatorSecurity(), false); } public void publishValidator(int startRoundDelay, final int minWeightMagnitude, Boolean sign, int keyIndex, int maxKeyIndex) throws Exception { @@ -1622,7 +1712,7 @@ public void publishValidator(int startRoundDelay, final int minWeightMagnitude, // get branch and trunk List txToApprove = getTransactionToApproveTips(3, Optional.empty()); - storeCustomBundle(configuration.getValidatorManagerAddress(), Hash.NULL_HASH, txToApprove, validatorBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorManagerKeyfile(), configuration.getValidatorManagerSecurity()); + storeCustomBundle(configuration.getValidatorManagerAddress(), Hash.NULL_HASH, txToApprove, validatorBytes, (long) startRoundIndex, minWeightMagnitude, sign, keyIndex, maxKeyIndex, configuration.getValidatorManagerKeyfile(), configuration.getValidatorManagerSecurity(), false); } // @@ -1641,17 +1731,24 @@ private List getConsistentTips() throws Exception { snapshotProvider.getLatestSnapshot().lockRead(); try { WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); - for (Hash transaction : tipsViewModel.getTips()) { - TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); + + List transactionViewModelStream = TransactionViewModel.fromHashes(tipsViewModel.getTips(), tangle); + + transactionViewModelStream.sort( + Comparator.comparing((TransactionViewModel m) -> m.getTimestamp())); + + for (TransactionViewModel txVM : transactionViewModelStream) { if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && txVM.getCurrentIndex() == 0 && txVM.isSolid() && BundleValidator.validate(tangle, snapshotProvider.getInitialSnapshot(), txVM.getHash()).size() != 0) { - if (walkValidator.isValid(transaction)) { - confirmedTips.add(transaction); - } else { - log.warn("Inconsistent transaction has been removed from tips: " + transaction.toString()); - tipsViewModel.removeTipHash(transaction); + if (walkValidator.isValid(txVM.getHash())) { + confirmedTips.add(txVM.getHash()); + } else if(txVM.isSolid()){ + log.warn("Inconsistent transaction has been removed from tips: {} ", txVM.getHash()); + txVM.setConfirmations(-1); + txVM.update(tangle, snapshotProvider.getInitialSnapshot(), "confirmation"); + tipsViewModel.removeTipHash(txVM.getHash()); } } } @@ -1675,11 +1772,14 @@ private List addMilestoneReferences(List confirmedTips, int roundInd if (previousRound == null){ txToApprove.add(Hash.NULL_HASH); } else { - txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones + txToApprove.add(previousRound.getMerkleRoot(tangle, snapshotProvider, ledgerService)); // merkle root of latest milestones } + //branch - List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); - txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips + Hash merkleRoot = HashFactory.TRANSACTION.create(new MerkleTreeImpl().getMerkleRoot(confirmedTips, MerkleOptions.getDefault())); + txToApprove.add(merkleRoot); + + log.debug("Milestone future branch transaction hash: " + merkleRoot); } return txToApprove; } diff --git a/src/main/java/net/helix/pendulum/service/ledger/LedgerService.java b/src/main/java/net/helix/pendulum/service/ledger/LedgerService.java index 6ac309ca..c8fbdc34 100644 --- a/src/main/java/net/helix/pendulum/service/ledger/LedgerService.java +++ b/src/main/java/net/helix/pendulum/service/ledger/LedgerService.java @@ -2,6 +2,8 @@ import net.helix.pendulum.controllers.RoundViewModel; import net.helix.pendulum.model.Hash; +import net.helix.pendulum.service.snapshot.Snapshot; +import net.helix.pendulum.service.snapshot.SnapshotService; import java.util.List; import java.util.Map; @@ -42,6 +44,8 @@ public interface LedgerService { */ boolean applyRoundToLedger(RoundViewModel round) throws LedgerException; + boolean applyRoundToLedger(RoundViewModel round, Snapshot snaphshot, Set nonConflictualMilestones, boolean simulation) throws LedgerException; + /** * Checks the consistency of the combined balance changes of the given tips.
    *
    @@ -90,4 +94,9 @@ public interface LedgerService { */ Map generateBalanceDiff(Set visitedTransactions, Set startTransactions, int milestoneIndex) throws LedgerException; + + boolean updateDiff(Set approvedHashes, final Map diff, Hash tip); // temporary + + SnapshotService getSnapshotService(); + } diff --git a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java index abe258bb..f4fb3d3b 100644 --- a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java @@ -5,15 +5,19 @@ import net.helix.pendulum.controllers.RoundViewModel; import net.helix.pendulum.controllers.StateDiffViewModel; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Sponge; +import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.ledger.LedgerException; import net.helix.pendulum.service.ledger.LedgerService; import net.helix.pendulum.service.milestone.MilestoneService; +import net.helix.pendulum.service.snapshot.Snapshot; import net.helix.pendulum.service.snapshot.SnapshotException; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; import net.helix.pendulum.service.snapshot.impl.SnapshotStateDiffImpl; import net.helix.pendulum.storage.Tangle; +import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -94,9 +98,14 @@ public void restoreLedgerState() throws LedgerException { @Override public boolean applyRoundToLedger(RoundViewModel round) throws LedgerException { - if(generateStateDiff(round)) { + return applyRoundToLedger(round, snapshotProvider.getLatestSnapshot(), new HashSet<>(), false); + } + + @Override + public boolean applyRoundToLedger(RoundViewModel round, Snapshot snapshot, Set nonConflictualMilestones, boolean simulation) throws LedgerException { + if(generateStateDiff(round, nonConflictualMilestones, simulation)) { try { - snapshotService.replayMilestones(snapshotProvider.getLatestSnapshot(), round.index()); + snapshotService.replayMilestones(snapshot, round.index()); } catch (SnapshotException e) { throw new LedgerException("failed to apply the balance changes to the ledger state", e); } @@ -220,15 +229,7 @@ public Map generateBalanceDiff(Set visitedTransactions, Set parents = RoundViewModel.getMilestoneBranch(tangle, transactionViewModel, milestoneTx, config.getValidatorSecurity()); - for (Hash parent : parents) { - nonAnalyzedTransactions.offer(parent); - } - } else { - nonAnalyzedTransactions.offer(transactionViewModel.getBranchTransactionHash()); - } + nonAnalyzedTransactions.offer(transactionViewModel.getBranchTransactionHash()); } } } @@ -258,49 +259,99 @@ public Map generateBalanceDiff(Set visitedTransactions, Set successfullyProcessedMilestones, boolean simulation) throws LedgerException { try { - /*TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, round.getHash()); - - if (!transactionViewModel.isSolid()) { - return false; + if (round.getHashes().isEmpty()) { + return true; } - - final int transactionSnapshotIndex = transactionViewModel.snapshotIndex(); - boolean successfullyProcessed = transactionSnapshotIndex == round.index(); - if (!successfullyProcessed) { - // if the snapshotIndex of our transaction was set already, we have processed our milestones in - // the wrong order (i.e. while rescanning the db) - if (transactionSnapshotIndex != 0) { - milestoneService.resetCorruptedRound(round.index()); - }*/ - snapshotProvider.getLatestSnapshot().lockRead(); - boolean successfullyProcessed; + + boolean successfullyProcessed = true; try { - Set confirmedTips = milestoneService.getConfirmedTips(round.index()); - //todo remove: System.out.println("round.index(): " + round.index() + ", " + confirmedTips); - Map balanceChanges = generateBalanceDiff(new HashSet<>(), confirmedTips == null? new HashSet<>() : confirmedTips, - snapshotProvider.getLatestSnapshot().getIndex() + 1); - successfullyProcessed = balanceChanges != null; - if (successfullyProcessed) { - successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( - new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); - if (successfullyProcessed) { - milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); + Set processedTips = new HashSet<>(); + Map totalBalanceChanges = new HashMap<>(); + + Map> confirmedTips = round.getConfirmedTipsPerMilestone(tangle, config); + List targetRoundMilestones = getSortedMilestones(round, confirmedTips); + Snapshot localSnapshot = snapshotProvider.getLatestSnapshot(); + + for (int i = 0; i < targetRoundMilestones.size(); i++) { + + Hash milestone = targetRoundMilestones.get(i).getHash(); + Set newTipsFromMilestone = confirmedTips.get(milestone).stream().filter(h -> + !processedTips.contains(h)).collect(Collectors.toSet()); + + if (newTipsFromMilestone.isEmpty()) { + successfullyProcessedMilestones.add(milestone); + continue; + } + + Set testTransactions = (new HashSet<>(processedTips)); + testTransactions.addAll(newTipsFromMilestone); + + Map balanceChanges = generateBalanceDiff(new HashSet<>(), testTransactions, localSnapshot.getIndex() + 1); + + if (balanceChanges != null) { + boolean localProcessed = localSnapshot.patchedState(new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); + if (localProcessed) { + processedTips.addAll(newTipsFromMilestone); + successfullyProcessedMilestones.add(milestone); if (!balanceChanges.isEmpty()) { - new StateDiffViewModel(balanceChanges, round.index()).store(tangle); + totalBalanceChanges.putAll(balanceChanges); } } + successfullyProcessed = successfullyProcessed && localProcessed; + } + } + + if (!simulation && !processedTips.isEmpty()) { + + TransactionViewModel.fromHashes(processedTips, tangle).forEach(tvm -> { + try { + tvm.setRoundIndex(tvm.getRoundIndex() == 0 ? round.index() : tvm.getRoundIndex()); + tvm.update(tangle, snapshotProvider.getInitialSnapshot(), "roundIndex"); + } catch (Exception e){ + log.error("Error during transaction round index update: " + tvm.getHash(), e); + } + }); + + milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); + if (!totalBalanceChanges.isEmpty()) { + new StateDiffViewModel(totalBalanceChanges, round.index()).store(tangle); } - } finally { - snapshotProvider.getLatestSnapshot().unlockRead(); } + } finally { + snapshotProvider.getLatestSnapshot().unlockRead(); + } return successfullyProcessed; } catch (Exception e) { throw new LedgerException("unexpected error while generating the StateDiff for Round" + round.index(), e); } } + + private List getSortedMilestones(RoundViewModel round, Map> confirmedTips) { + Set milestoneHashes = confirmedTips.keySet(); + + List targetRoundMilestones = TransactionViewModel.fromHashes(milestoneHashes, tangle); + Sponge sponge = SpongeFactory.create(SpongeFactory.Mode.S256); + + targetRoundMilestones.sort( + Comparator.comparing((TransactionViewModel m) -> Hex.toHexString(getHash(sponge, + (m.getSignature().toString() + round.index()).getBytes())))); + return targetRoundMilestones; + } + + public SnapshotService getSnapshotService(){ + return this.snapshotService; + } + + private byte[] getHash(Sponge sponge, byte[] data) { + byte[] result = new byte[Sponge.HASH_LENGTH]; + sponge.reset(); + sponge.absorb(data, 0, data.length); + sponge.squeeze(result, 0, Sponge.HASH_LENGTH); + return result; + } } diff --git a/src/main/java/net/helix/pendulum/service/milestone/VirtualTransactionService.java b/src/main/java/net/helix/pendulum/service/milestone/VirtualTransactionService.java new file mode 100644 index 00000000..699d3811 --- /dev/null +++ b/src/main/java/net/helix/pendulum/service/milestone/VirtualTransactionService.java @@ -0,0 +1,15 @@ +package net.helix.pendulum.service.milestone; + +import net.helix.pendulum.controllers.TransactionViewModel; + + +public interface VirtualTransactionService { + + /** + * It tries to rebuild the virtual parent transaction for given trasaction model, parent transaction could be build if the sibling transaction already exists, + * then it will try to rebuild the next parent transaction an so on. + * @param transactionViewModel + * @return number of the built transactions. + */ + int rebuildVirtualTransactionsIfPossible(TransactionViewModel transactionViewModel); +} diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 05675696..ec5645b0 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -175,12 +175,14 @@ public void trackLatestSolidMilestones() throws MilestoneException { //syncValidatorTracker(); //syncLatestMilestoneTracker(nextRound.index()); applyRoundToLedger(nextRound); - logChange(currentSolidRoundIndex); currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + if(nextRound.index() == currentSolidRoundIndex){ + logChange(currentSolidRoundIndex); tangle.publish("ctx %s %d", nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)), nextRound.index()); log.delegate().trace("ctx= " + nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)) + ", round=" + nextRound.index()); } } + } } catch (Exception e) { throw new MilestoneException("unexpected error while checking for new latest solid milestones", e); } @@ -290,10 +292,7 @@ private void stopRepair() { private void logChange(int prevSolidRoundIndex) { Snapshot latestSnapshot = snapshotProvider.getLatestSnapshot(); int latestRoundIndex = latestSnapshot.getIndex(); - - if (prevSolidRoundIndex != latestRoundIndex) { - log.debug("Round #" + latestRoundIndex + " is SOLID"); - } + log.debug("Round #" + latestRoundIndex + " is SOLID"); } /** diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java index 6f518758..5f01036e 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java @@ -1,11 +1,12 @@ package net.helix.pendulum.service.milestone.impl; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.crypto.Merkle; +import net.helix.pendulum.crypto.merkle.MerkleNode; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.service.API; import net.helix.pendulum.service.utils.RoundIndexUtil; +import net.helix.pendulum.utils.KeyfileUtil; import net.helix.pendulum.service.validatormanager.CandidateTracker; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -73,8 +74,8 @@ private void initSeed(PendulumConfig configuration) { } private void writeKeyIndex() throws IOException { - List> merkleTree = Merkle.readKeyfile(new File(keyfile)); - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); + List> merkleTree = KeyfileUtil.readKeyfile(new File(keyfile)); + KeyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); } private void readKeyfileMetadata() throws IOException { @@ -87,7 +88,7 @@ private void readKeyfileMetadata() throws IOException { seed = fields[1]; } } - List> merkleTree = Merkle.readKeyfile(new File(keyfile)); + List> merkleTree = KeyfileUtil.readKeyfile(new File(keyfile)); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size() - 1).get(0).bytes()); } @@ -106,7 +107,7 @@ private void doKeyChange() throws Exception { // generate new keyfile int newKeyfileIndex = keyfileIndex + 1; log.debug("Generating Keyfile (idx: " + newKeyfileIndex + ")"); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getValidatorSecurity()); + List> merkleTree = KeyfileUtil.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getValidatorSecurity()); Hash newAddress = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); // send keyChange bundle to register new address api.publishKeyChange(address.toString(), newAddress, mwm, sign, currentKeyIndex, maxKeyIndex); @@ -114,13 +115,13 @@ private void doKeyChange() throws Exception { keyfileIndex = newKeyfileIndex; address = newAddress; currentKeyIndex = maxKeyIndex * keyfileIndex; - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + KeyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); } private void generateKeyfile(String seed) throws Exception { log.debug("Generating Keyfile (idx: " + keyfileIndex + ")"); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getValidatorSecurity()); - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + List> merkleTree = KeyfileUtil.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getValidatorSecurity()); + KeyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } @@ -203,7 +204,7 @@ private Runnable getRunnablePublishMilestone() { try { publishMilestone(); } catch (Exception e) { - e.printStackTrace(); + log.error("Error during processing milestone", e); } }; } @@ -214,7 +215,7 @@ public void shutdown() { try { writeKeyIndex(); } catch (Exception e) { - e.printStackTrace(); + log.error("Shut down milestone publisher", e); } scheduledExecutorService.shutdown(); } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java index cae1f9a4..9f6c7a11 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java @@ -6,8 +6,9 @@ import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.controllers.RoundViewModel; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.IntegerIndex; import net.helix.pendulum.model.StateDiff; @@ -183,8 +184,10 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); - log.trace("valid signature: {}", validSignature); + boolean validSignature = new MerkleTreeImpl().validateMerkleSignature(bundleTransactionViewModels, + new MerkleOptions(mode, senderAddress, securityLevel, config.getMilestoneKeyDepth())); + //System.out.println("valid signature: " + validSignature); + if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { @@ -357,18 +360,19 @@ private boolean wasRoundAppliedToLedger(RoundViewModel round) throws Exception { * @throws MilestoneException if anything unexpected happens while updating the milestone index * @param processedTransactions a set of transactions that have been processed already (for the recursive calls) */ - private void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, + public void updateRoundIndexOfMilestoneTransactions(int correctIndex, int newIndex, Set processedTransactions) throws MilestoneException { Set inconsistentMilestones = new HashSet<>(); try { // update milestones RoundViewModel round = RoundViewModel.get(tangle, newIndex); - if(round != null) { - for (Hash milestoneHash : round.getHashes()) { + if(round == null) { + return; + } + for (Hash milestoneHash : round.getHashes()) { TransactionViewModel milestoneTx = TransactionViewModel.fromHash(tangle, milestoneHash); updateRoundIndexOfSingleTransaction(milestoneTx, newIndex); - } } // update confirmed transactions final Queue transactionsToUpdate = new LinkedList<>(getConfirmedTips(newIndex)); diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java index 07bc196b..ff71b70a 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java @@ -168,8 +168,8 @@ public MilestoneTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide private void publishMilestoneRefs(TransactionViewModel transaction) throws Exception { BundleViewModel bundle = BundleViewModel.load(tangle, transaction.getBundleHash()); - for (Hash tx: bundle.getHashes()) { - tangle.publish("lmr %s %s %s", tx, "Branch " + RoundViewModel.getMilestoneBranch(tangle, TransactionViewModel.fromHash(tangle, tx), transaction, config.getValidatorSecurity()), "Trunk " + RoundViewModel.getMilestoneTrunk(tangle, TransactionViewModel.fromHash(tangle, tx), transaction)); + for (Hash tx : bundle.getHashes()) { + tangle.publish("lmr %s %s %s", tx, "Branch " + transaction.getBranchTransactionHash(), "Trunk " + transaction.getTrunkTransactionHash(), transaction); } } @@ -184,7 +184,16 @@ public void addMilestoneToRoundLog(Hash milestoneHash, int roundIndex, int numbe tangle.publish("lmi %s %d", milestoneHash, roundIndex); // todo: temporarily log hardcoded number of _active_ validators instead of numberOfValidators log.delegate().debug("New milestone {} ({}/{}) added to round #{}", milestoneHash, numberOfMilestones, BasePendulumConfig.Defaults.NUMBER_OF_ACTIVE_VALIDATORS, roundIndex); + Set tips = null; + try { + tips = RoundViewModel.getTipSet(tangle, milestoneHash, config.getValidatorSecurity()); + } catch (Exception e) { + log.error("Add milestone to round log!", e); + } + tips.forEach(t -> + log.delegate().debug("Valid tips transaction: {}, from milesltone {} ", t, milestoneHash) + ); } /** @@ -201,8 +210,6 @@ public void setRoundIndexAndConfirmations(RoundViewModel currentRVM, Transaction // The confirmation counter should be incremented with each milestone reference for (Hash tx : referencedTipSet) { TransactionViewModel txvm = TransactionViewModel.fromHash(tangle, tx); - txvm.setRoundIndex(txvm.getRoundIndex() == 0 ? roundIndex : txvm.getRoundIndex()); - txvm.update(tangle, snapshotProvider.getInitialSnapshot(), "roundIndex"); txvm.setConfirmations(txvm.getConfirmations() + 1); txvm.update(tangle, snapshotProvider.getInitialSnapshot(), "confirmation"); } @@ -335,7 +342,11 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw setRoundIndexAndConfirmations(currentRoundViewModel, transaction, roundIndex); publishMilestoneRefs(transaction); } else { + log.delegate().debug("Failed to validate milestone {} in round #{}", transaction.getHash(), roundIndex); + Set tips = RoundViewModel.getTipSet(tangle, transaction.getHash(), config.getValidatorSecurity()); + tips.forEach(t -> log.delegate().debug("Missed transaction: {}, from skipped milesltone {} ", t, transaction.getHash())); tracer.trace("round is not active"); + } if (!transaction.isSolid()) { diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java new file mode 100644 index 00000000..bdbd232e --- /dev/null +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/VirtualTransactionServiceImpl.java @@ -0,0 +1,233 @@ +package net.helix.pendulum.service.milestone.impl; + +import net.helix.pendulum.TransactionValidator; +import net.helix.pendulum.controllers.BundleNonceViewModel; +import net.helix.pendulum.controllers.RoundViewModel; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.service.milestone.VirtualTransactionService; +import net.helix.pendulum.service.snapshot.SnapshotProvider; +import net.helix.pendulum.storage.Tangle; +import net.helix.pendulum.utils.bundle.BundleUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.stream.Collectors; + +import static net.helix.pendulum.controllers.TransactionViewModel.PREFILLED_SLOT; +import static net.helix.pendulum.controllers.TransactionViewModel.fromHash; + +/** + * Creates a service instance that allows us to perform virtual transaction operations.
    + *
    + */ +public class VirtualTransactionServiceImpl implements VirtualTransactionService { + + private static final Logger log = LoggerFactory.getLogger(VirtualTransactionServiceImpl.class); + + /** + * Holds the tangle object which acts as a database interface.
    + */ + private Tangle tangle; + + /** + * Holds the snapshot provider which gives us access to the relevant snapshots.
    + */ + private SnapshotProvider snapshotProvider; + + /** + * Holds transaction validator.
    + */ + private TransactionValidator transactionValidator; + + + /** + * This method initializes the instance and registers its dependencies.
    + *
    + * It simply stores the passed in values in their corresponding private properties.
    + *
    + * Note: Instead of handing over the dependencies in the constructor, we register them lazy. This allows us to have + * circular dependencies because the instantiation is separated from the dependency injection. To reduce the + * amount of code that is necessary to correctly instantiate this class, we return the instance itself which + * allows us to still instantiate, initialize and assign in one line - see Example:
    + *
    + * {@code milestoneService = new VirtualTransactionServiceImpl().init(...);} + * + * @param tangle Tangle object which acts as a database interface + * @param snapshotProvider snapshot provider which gives us access to the relevant snapshots + * @return the initialized instance itself to allow chaining + */ + public VirtualTransactionServiceImpl init(Tangle tangle, SnapshotProvider snapshotProvider, TransactionValidator transactionValidator) { + this.tangle = tangle; + this.snapshotProvider = snapshotProvider; + this.transactionValidator = transactionValidator; + return this; + } + + /** + * Rebuilds a new virtual transaction which is based on current virtual transaction and its sibling (it this transaction exists) + * Attached the new built transaction in local tangle + * + * @param transactionViewModel transaction view model of the child + * @return true if the new transaction is created, otherwise false. + */ + @Override + public int rebuildVirtualTransactionsIfPossible(TransactionViewModel transactionViewModel) { + if (!transactionViewModel.isVirtual()) { + return 0; + } + int noOfRebuitTransactions = 0; + TransactionViewModel nextTransaction = transactionViewModel; + while (nextTransaction != null) { + nextTransaction = rebuildViirtualParentTransactionIfPossible(nextTransaction); + if (nextTransaction != null) { + noOfRebuitTransactions++; + } + } + return noOfRebuitTransactions; + } + + private TransactionViewModel rebuildViirtualParentTransactionIfPossible(TransactionViewModel tvm) { + long currentIndex = tvm.getTagLongValue(); + long siblingIndex = getSiblingIndex(currentIndex); + long parentIndex = getParentIndex(currentIndex); + + Hash milestoneHash = tvm.getBundleNonceHash(); + Map transactions = findByTagAndMilestone(milestoneHash, Arrays.asList(siblingIndex, parentIndex)); + + if (!transactions.containsKey(parentIndex) && transactions.containsKey(siblingIndex)) { + try { + TransactionViewModel siblingTransaction = fromHash(tangle, transactions.get(siblingIndex)); + TransactionViewModel newTransaction = reconstructParent(tvm, siblingTransaction); + log.debug("Rebuild transaction virtual parent transaction {} for: {} ", newTransaction.getHash(), tvm.getHash()); + return newTransaction; + } catch (Exception e) { + log.error("Error during generation of virtual transaction parent!"); + } + } + return null; + } + + /** + * Reconstruct virtual parent based on 2 transactions which are consider the direct children of the new created transaction. + * Order of the t1 and t2 doesn't matter, they will be sorted according with their merkle index. + * + * @param t1 first child transaction + * @param t2 second child transactiopn + * @return + * @throws Exception + */ + private TransactionViewModel reconstructParent(TransactionViewModel t1, TransactionViewModel t2) throws Exception { + if (validVirtualTransaction(Arrays.asList(t1, t2))) { + log.warn("Transactions: {} {} are not valid virtual transaction!", t1.getHash(), t2.getHash()); + return null; + } + Hash trunk = RoundViewModel.getRoundIndex(t1) > RoundViewModel.getRoundIndex(t2) ? t2.getHash() : t1.getHash(); + Hash branch = RoundViewModel.getRoundIndex(t1) < RoundViewModel.getRoundIndex(t2) ? t2.getHash() : t1.getHash(); + + byte[] newTransaction = BundleUtils.createVirtualTransaction(branch, trunk, getParentIndex(RoundViewModel.getRoundIndex(t1)), + t1.getSignature(), t1.getAddressHash()); + + TransactionViewModel newTransactionViewModel = new TransactionViewModel(newTransaction, SpongeFactory.Mode.S256); + newTransactionViewModel.storeTransactionLocal(tangle, snapshotProvider.getInitialSnapshot(), transactionValidator); + return newTransactionViewModel; + } + + /** + * Get parent index of the current virtual transaction + * + * @param index merkle index of the current transaction + * @return merkle index of the parent transaction + */ + private long getParentIndex(long index) { + return index % 2 == 0 ? index / 2 - 1 : (index - 1) / 2L; + } + + /** + * Get sibling index, based on current transaction index. + * + * @param index of the current transaction + * @return the index of the other child (from merkle tree) + */ + private long getSiblingIndex(long index) { + return index % 2 == 0 ? index - 1 : index + 1; + } + + + /** + * Virtual transactions validator. Call areVirtualTransaction and areSiblingVirtualTransactions validators. + * + * @param transactions, list of transactions that will be validated + * @return true if all transaction from povided list are valid virtual transactions. + */ + private boolean validVirtualTransaction(List transactions) { + return areVirtualTransaction(transactions) && + areSiblingVirtualTransactions(transactions); + } + + /** + * Virtual transactions validator. Checks if all transactions are virtual. + * + * @param transactions, list of transactions that will be validated + * @return true if all transaction from povided list are valid virtual transactions. + */ + private boolean areVirtualTransaction(List transactions) { + if (CollectionUtils.isEmpty(transactions)) { + return false; + } + return transactions.stream().filter(t -> !t.isVirtual()).collect(Collectors.toList()).size() == 0; + } + + /** + * Virtual transactions validator. Checks if all transactions are siblings + * + * @param transactions, list of transactions that will be validated + * @return true if all transaction from povided list are valid virtual transactions. + */ + private boolean areSiblingVirtualTransactions(List transactions) { + if (CollectionUtils.isEmpty(transactions)) { + return false; + } + Hash firstTag = transactions.get(0).getTagValue(); + if (firstTag == null) { + return false; + } + return transactions.stream().filter(t -> firstTag.equals(t.getTagValue())).collect(Collectors.toList()).size() == 0; + } + + + /** + * Find all transaction which belongs to milestone and have tags from parameter. + * + * @param milestoneHash milestone hash + * @param tags List of longs which represents the index in merkle tree + * @return map of tag (long value) and transaction hashes + */ + public Map findByTagAndMilestone(Hash milestoneHash, List tags) { + Map tagTransactionHash = new HashMap(); + try { + Set transactionHashes = BundleNonceViewModel.load(tangle, milestoneHash) + .getHashes(); + for (Hash t : transactionHashes) { + TransactionViewModel tvm = fromHash(tangle, t); + if (tvm.getType() == PREFILLED_SLOT) { + continue; + } + tags.stream().filter(tag -> !tagTransactionHash.keySet().contains(tag)).forEach(tag -> { + if (tag == tvm.getTagLongValue()) { + tagTransactionHash.put(tag, t); + } + }); + if (tagTransactionHash.size() == tags.size()) { + break; + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return tagTransactionHash; + } +} diff --git a/src/main/java/net/helix/pendulum/service/tipselection/impl/WalkValidatorImpl.java b/src/main/java/net/helix/pendulum/service/tipselection/impl/WalkValidatorImpl.java index 51764765..b191bb22 100644 --- a/src/main/java/net/helix/pendulum/service/tipselection/impl/WalkValidatorImpl.java +++ b/src/main/java/net/helix/pendulum/service/tipselection/impl/WalkValidatorImpl.java @@ -59,6 +59,9 @@ public WalkValidatorImpl(Tangle tangle, SnapshotProvider snapshotProvider, Ledge public boolean isValid(Hash transactionHash) throws Exception { TransactionViewModel transactionViewModel = TransactionViewModel.fromHash(tangle, transactionHash); + if(Hash.NULL_HASH.equals(transactionHash)){ + return true; + } if (transactionViewModel.getType() == TransactionViewModel.PREFILLED_SLOT) { log.trace("tx_hash: {} ", transactionViewModel.getHash()); log.trace("type: {} ", transactionViewModel.getType()); diff --git a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java index 36780497..315b6840 100644 --- a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java @@ -3,8 +3,9 @@ import net.helix.pendulum.BundleValidator; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; @@ -58,7 +59,9 @@ public ValidatorValidity validateValidators(TransactionViewModel transactionView // validate signature Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getValidatorManagerKeyDepth()); + + boolean validSignature = new MerkleTreeImpl().validateMerkleSignature(bundleTransactionViewModels, + new MerkleOptions(mode, senderAddress, securityLevel, config.getMilestoneKeyDepth())); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getValidatorManagerAddress().equals(senderAddress) && validSignature)) { return VALID; diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java index ca25e285..43322cc0 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java @@ -3,8 +3,9 @@ import net.helix.pendulum.BundleValidator; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; @@ -74,7 +75,9 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM if (tail.getHash().equals(transactionViewModel.getHash()) && isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); + + boolean validSignature = new MerkleTreeImpl().validateMerkleSignature(bundleTransactionViewModels, + new MerkleOptions(mode, senderAddress, securityLevel, config.getMilestoneKeyDepth())); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validator.contains(senderAddress)) && validSignature) { return VALID; diff --git a/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java b/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java new file mode 100644 index 00000000..4ced4645 --- /dev/null +++ b/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java @@ -0,0 +1,98 @@ +package net.helix.pendulum.utils; + +import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; +import org.bouncycastle.util.encoders.Hex; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +/** + * It contains the required methods to manage key files (read, write, build) + */ +public class KeyfileUtil { + + public static String getSeed(File keyfile) throws IOException { + StringBuilder seedBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + seedBuilder.append(fields[1]); + } + return seedBuilder.toString(); + } + + public static byte[] getKeyTreeRoot(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security) { + List> tree = buildMerkleKeyTree(seed, pubkeyDepth,firstIndex,pubkeyCount,security); + return tree.get(tree.size()-1).get(0).bytes(); + + } + public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security) { + List keys = new ArrayList<>(1 << pubkeyDepth); + for (int i = 0; i < pubkeyCount; i++) { + int idx = firstIndex + i; + keys.add(createKeyHash(seed, security, idx)); + } + return (new MerkleTreeImpl()).buildMerkleTree(keys, MerkleOptions.getDefault()); + } + + public static void writeKeys(BufferedWriter bw, List keys) throws IOException { + int leadingNulls = 0; + while (keys.get(leadingNulls) == null) { + leadingNulls++; + } + bw.write(leadingNulls + " "); + for (int i = leadingNulls; i < keys.size(); i++) { + if (keys.get(i) == null) { + break; + } + bw.write(writeMerkleNode(keys.get(i))); + } + bw.newLine(); + } + + public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { + try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { + bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); + bw.newLine(); + writeKeys(bw, merkleTree.get(0)); + for (int i = 1; i < merkleTree.size(); i++) { + writeKeys(bw, merkleTree.get(i)); + } + } + } + + public static List> readKeyfile(File keyfile) throws IOException { + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + int depth = Integer.parseInt(fields[0]); + List> result = new ArrayList<>(depth + 1); + for (int i = 0; i <= depth; i++) { + fields = br.readLine().split(" "); + int leadingNulls = Integer.parseInt(fields[0]); + List row = new ArrayList<>(); + for (int j = 0; j < leadingNulls; j++) { + row.add(Hash.NULL_HASH); + } + for (int j = 0; j < fields[1].length() / 64; j++) { + row.add(HashFactory.ADDRESS.create(fields[1].substring(j * 64, (j + 1) * 64))); + } + result.add(row); + } + return result; + } + } + + private static Hash createKeyHash(String seed, int security, int idx) { + return HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security)); + } + + private static String writeMerkleNode(MerkleNode key) { + return key.toString(); + } +} diff --git a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java index df8134fa..92fb6fef 100644 --- a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java @@ -1,11 +1,14 @@ package net.helix.pendulum.utils.bundle; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleNode; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; import net.helix.pendulum.model.Hash; +import net.helix.pendulum.service.utils.RoundIndexUtil; +import net.helix.pendulum.utils.KeyfileUtil; import net.helix.pendulum.utils.Serializer; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -20,7 +23,7 @@ public class BundleUtils { - private final Logger log = LoggerFactory.getLogger(BundleUtils.class); + private static final Logger log = LoggerFactory.getLogger(BundleUtils.class); private List senderTransactions = new ArrayList<>(); private byte[] merkleTransaction; @@ -38,6 +41,9 @@ public BundleUtils(Hash senderAddress, Hash receiverAddress) { this.receiverAddress = receiverAddress.toString(); } + public BundleUtils() { + } + /** * @return transactions */ @@ -116,7 +122,7 @@ public void create(byte[] data, long tag, Boolean sign, int keyIndex, int maxKey * @param tag tag * @return transaction */ - private byte[] initTransaction(String address, int currentIndex, int lastIndex, long timestamp, long tag) { + public byte[] initTransaction(String address, int currentIndex, int lastIndex, long timestamp, long tag) { byte[] transaction = new byte[TransactionViewModel.SIZE]; System.arraycopy(Hex.decode(address), 0, transaction, TransactionViewModel.ADDRESS_OFFSET, TransactionViewModel.ADDRESS_SIZE); System.arraycopy(Serializer.serialize((long) currentIndex), 0, transaction, TransactionViewModel.CURRENT_INDEX_OFFSET, TransactionViewModel.CURRENT_INDEX_SIZE); @@ -159,13 +165,13 @@ private byte[] addBundleHash(List bundle, SpongeFactory.Mode mode) { private void signBundle(String filepath, byte[] merkleTransaction, List senderTransactions, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { // Get merkle path and store in signatureMessageFragment of Sibling Transaction File keyfile = new File(filepath); - List> merkleTree = Merkle.readKeyfile(keyfile); - String seed = Merkle.getSeed(keyfile); + List> merkleTree = KeyfileUtil.readKeyfile(keyfile); + String seed = KeyfileUtil.getSeed(keyfile); int security = senderTransactions.size(); // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); - byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); + List merklePath = new MerkleTreeImpl().getMerklePath(merkleTree, keyIndex % maxKeyIndex); + byte[] path = Hex.decode(merklePath.stream().map(m -> m.toString()).collect(Collectors.joining())); System.arraycopy(path, 0, merkleTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); // sign bundle hash and store signature in Milestone Transaction @@ -180,4 +186,45 @@ private void signBundle(String filepath, byte[] merkleTransaction, List System.arraycopy(signature, 0, senderTransactions.get(i), TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE); } } + + /** + * Create a virtual transaction, the following fields are filled: + *
  1. Branch and trunk transactions with information given in input parameters, should be the children of the current node in Merkle tree
    2. + *
  2. Tag with merkle index from input parameters
    3. + *
  3. Address with address from input parameters, should be the address of the validator
    4. + *
  4. SignatureMessage with bundleNonce from input parameters, should be the hash of the milestone
    5. + *
  5. BundleNonce with a default value for virtual transaction TransasctionViewModel.DEFAULT_VIRTUAL_BUNDLE_NONCE
    6. + *
  6. Timestamp with current timestamp
    7. + *
  7. Current index and last index are set as 0
    8. + * + * @param branchHash - branch transaction hash + * @param trunkHash - trunk transaction hash + * @param merkleIndex - index in merkle tree + * @param bundleNonce - list of bytes should contain the milestone hash + * @param address - current address of the validator + * @return + */ + public static byte[] createVirtualTransaction(Hash branchHash, Hash trunkHash, long merkleIndex, byte[] bundleNonce, Hash address) { + + if(bundleNonce.length > TransactionViewModel.BUNDLE_NONCE_SIZE){ + log.error("Tried to create a virtual trasaction with bundleNonce greater than {}, for branch{}, and trunk {}.", + TransactionViewModel.BUNDLE_NONCE_SIZE, branchHash.toString(), trunkHash.toString()); + return null; + } + + BundleUtils bundleUtils = new BundleUtils(); + byte[] transaction = bundleUtils.initTransaction(Hex.toHexString(address.bytes()), 0, 0, RoundIndexUtil.getCurrentTime(), merkleIndex); + System.arraycopy(bundleNonce, 0, transaction, TransactionViewModel.BUNDLE_NONCE_OFFSET, TransactionViewModel.BUNDLE_NONCE_SIZE); + + // mark this transaction as virtual (all bundle nonce bits are one) and nonce is zero. + byte[] nonce = new byte[TransactionViewModel.NONCE_SIZE]; + Arrays.fill(nonce, (byte) 0xff); + System.arraycopy(nonce, 0, transaction, TransactionViewModel.NONCE_OFFSET, TransactionViewModel.NONCE_SIZE); + System.arraycopy(branchHash.bytes(), 0, transaction, TransactionViewModel.BRANCH_TRANSACTION_OFFSET, TransactionViewModel.BRANCH_TRANSACTION_SIZE); + System.arraycopy(trunkHash.bytes(), 0, transaction, TransactionViewModel.TRUNK_TRANSACTION_OFFSET, TransactionViewModel.TRUNK_TRANSACTION_SIZE); + + bundleUtils.addBundleHash(Arrays.asList(transaction), SpongeFactory.Mode.S256); + + return transaction; + } } diff --git a/src/test/java/net/helix/pendulum/crypto/merkle/MerkleTreeTest.java b/src/test/java/net/helix/pendulum/crypto/merkle/MerkleTreeTest.java new file mode 100644 index 00000000..83a9fec8 --- /dev/null +++ b/src/test/java/net/helix/pendulum/crypto/merkle/MerkleTreeTest.java @@ -0,0 +1,58 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.TransactionTestUtils; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; +import net.helix.pendulum.model.TransactionHash; +import net.helix.pendulum.utils.KeyfileUtil; +import org.bouncycastle.util.encoders.Hex; +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class MerkleTreeTest { + + @Test + public void createMerkleTreeAndValidateIt() { + int keyDepth = 5; + int firstIndex = 0; + int security = 2; + int keyIndex = 0; + int maxKeyIndex = (int) Math.pow(2, keyDepth); + String seed = "aabbccdd00000000000000000000000000000000000000000000000000000000"; + + byte[] root = KeyfileUtil.getKeyTreeRoot(seed, keyDepth, firstIndex, maxKeyIndex, security); + + Hash senderAddress = HashFactory.ADDRESS.create(root); + MerkleTree merkle = new MerkleTreeImpl(); + MerkleOptions options = new MerkleOptions(SpongeFactory.Mode.S256, senderAddress, security, keyDepth); + + List> merkleTree = KeyfileUtil.buildMerkleKeyTree(seed, keyDepth, maxKeyIndex * keyIndex, maxKeyIndex, security); + + Assert.assertTrue(senderAddress.equals(HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size() - 1).get(0).bytes()))); + + List merklePath = new MerkleTreeImpl().getMerklePath(merkleTree, keyIndex % maxKeyIndex); + byte[] path = Hex.decode(merklePath.stream().map(m -> m.toString()).collect(Collectors.joining())); + byte[] merkleTransactionSignature = new byte[TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE]; + + System.arraycopy(path, 0, merkleTransactionSignature, 0, path.length); + byte[] merkleRootTest = merkle.getMerkleRoot(options.getMode(), senderAddress.bytes(), + merkleTransactionSignature, 0, keyIndex, options.getDepth()); + + Assert.assertTrue(HashFactory.ADDRESS.create(merkleRootTest).equals(options.getAddress())); + } + + private List getHashes(int noOfLeaves) { + List transactions = new ArrayList(); + for (int i = 0; i < noOfLeaves; i++) { + transactions.add(TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes())); + } + return transactions; + } +} diff --git a/src/test/java/net/helix/pendulum/crypto/merkle/TransactionMerkleTreeTest.java b/src/test/java/net/helix/pendulum/crypto/merkle/TransactionMerkleTreeTest.java new file mode 100644 index 00000000..d24329e6 --- /dev/null +++ b/src/test/java/net/helix/pendulum/crypto/merkle/TransactionMerkleTreeTest.java @@ -0,0 +1,51 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.TransactionTestUtils; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.impl.TransactionMerkleTreeImpl; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.TransactionHash; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + + +public class TransactionMerkleTreeTest { + + private static final Logger log = LoggerFactory.getLogger(TransactionMerkleTreeTest.class); + + @Test + public void testTreeTransactionGeneration() { + // 6 index is missing because null leaves are not added into merkle tree + checkMerkleVirtualIndexes(10, new HashSet(Arrays.asList(0L, 1L, 2L, 3L, 4L, 5L, 7L, 8L, 9L, 10L, 11L))); + } + + private void checkMerkleVirtualIndexes(int noOfLeaves, Set expectedIndexes) { + TransactionHash milestoneHash = TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes()); + List transactions = new ArrayList(); + for (int i = 0; i < noOfLeaves; i++) { + transactions.add(TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes())); + } + MerkleOptions options = MerkleOptions.getDefault(); + options.setMilestoneHash(milestoneHash); + options.setAddress(Hash.NULL_HASH); + List virtualTransactions = new TransactionMerkleTreeImpl().buildMerkle(transactions, options); + Assert.assertTrue(virtualTransactions.size() > 0); + Set merkleIndexes = new HashSet(); + virtualTransactions.forEach(t -> { + try { + TransactionViewModel virtualTransaction = (TransactionViewModel)t; + Assert.assertTrue(virtualTransaction.isVirtual()); + merkleIndexes.add(virtualTransaction.getTagLongValue()); + + } catch (Exception e) { + e.printStackTrace(); + } + }); + Assert.assertEquals(expectedIndexes, merkleIndexes); + } +} diff --git a/src/test/java/net/helix/pendulum/model/HashTest.java b/src/test/java/net/helix/pendulum/model/HashTest.java index 8e26a449..4cc65f1b 100644 --- a/src/test/java/net/helix/pendulum/model/HashTest.java +++ b/src/test/java/net/helix/pendulum/model/HashTest.java @@ -1,6 +1,8 @@ package net.helix.pendulum.model; +import net.helix.pendulum.TransactionTestUtils; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import org.bouncycastle.util.encoders.Hex; import org.junit.Assert; @@ -110,4 +112,28 @@ public void compareToTest() throws Exception { Assert.assertEquals(hash.compareTo(Hash.NULL_HASH), -Hash.NULL_HASH.compareTo(hash)); } + @Test + public void testHashAggregation() { + TransactionHash hash1 = TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes()); + TransactionHash hash2 = TransactionHash.calculate(SpongeFactory.Mode.S256, TransactionTestUtils.getTransactionBytes()); + + Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); + byte[] result = new byte[Hash.SIZE_IN_BYTES]; + sha3.reset(); + sha3.absorb(hash1.bytes(), 0, hash1.getByteSize()); + sha3.absorb(hash2.bytes(), 0, hash2.getByteSize()); + sha3.squeeze(result, 0, Hash.SIZE_IN_BYTES); + TransactionHash resultHash = (TransactionHash) HashFactory.TRANSACTION.create(result, 0, Hash.SIZE_IN_BYTES); + + byte[] buffer = new byte[2 * Hash.SIZE_IN_BYTES]; + System.arraycopy(hash1.bytes(), 0, buffer, 0, hash1.getByteSize()); + System.arraycopy(hash2.bytes(), 0, buffer, Hash.SIZE_IN_BYTES, hash2.getByteSize()); + + TransactionHash hash = TransactionHash.calculate(SpongeFactory.Mode.S256, buffer); + + Assert.assertFalse(hash.equals(hash1)); + Assert.assertTrue(hash.equals(resultHash)); + Assert.assertFalse(hash.equals(Hash.NULL_HASH)); + Assert.assertFalse(resultHash.equals(Hash.NULL_HASH)); + } } diff --git a/src/test/java/net/helix/pendulum/network/NodeTest.java b/src/test/java/net/helix/pendulum/network/NodeTest.java index d2cc9cd7..d4ed4f52 100644 --- a/src/test/java/net/helix/pendulum/network/NodeTest.java +++ b/src/test/java/net/helix/pendulum/network/NodeTest.java @@ -41,7 +41,7 @@ public void setUp() { // set up class under test nodeConfig = mock(NodeConfig.class); - classUnderTest = new Node(null, null, null, null, null, null, nodeConfig); + classUnderTest = new Node(null, null, null, null, null, null, null, nodeConfig); // verify config calls in Node constructor verify(nodeConfig).getRequestHashSize(); @@ -71,7 +71,7 @@ public void spawnNeighborDNSRefresherThreadTest() { @Test public void whenProcessReceivedDataSetArrivalTimeToCurrentMillis() throws Exception { - Node node = new Node(mock(Tangle.class), mock(SnapshotProvider.class), mock(TransactionValidator.class), null, null, null, mock(NodeConfig.class)); + Node node = new Node(mock(Tangle.class), mock(SnapshotProvider.class), mock(TransactionValidator.class), null, null, null, null, mock(NodeConfig.class)); TransactionViewModel transaction = mock(TransactionViewModel.class); // It is important to stub the getHash method here because processReceivedData will broadcast the transaction. // This might sometimes (concurrency issue) lead to a NPE in the process receiver thread.