From 1f6dd1e38bab6bf53e56e02c70a004f130955555 Mon Sep 17 00:00:00 2001 From: Cristina Date: Thu, 14 Nov 2019 11:02:19 +0200 Subject: [PATCH 01/36] Transaction should be confirmed only after balance is updated. --- .../controllers/TransactionViewModel.java | 26 +++++++++++++++++ .../java/net/helix/pendulum/service/API.java | 3 +- .../ledger/impl/LedgerServiceImpl.java | 29 ++++++++++++------- .../milestone/impl/MilestoneTrackerImpl.java | 2 -- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index 5079801c..7af1db1b 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -3,14 +3,17 @@ import net.helix.pendulum.model.*; import net.helix.pendulum.model.persistables.*; import net.helix.pendulum.service.milestone.MilestoneTracker; +import net.helix.pendulum.service.milestone.impl.MilestoneTrackerImpl; import net.helix.pendulum.service.snapshot.Snapshot; import net.helix.pendulum.storage.Indexable; import net.helix.pendulum.storage.Persistable; import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Converter; import net.helix.pendulum.utils.Pair; +import net.helix.pendulum.utils.log.interval.IntervalLogger; import java.util.*; +import java.util.stream.Collectors; /** @@ -22,6 +25,11 @@ public class TransactionViewModel { private final Transaction transaction; + /** + * Holds the logger of this class (a rate limited logger that doesn't spam the CLI output).
+ */ + private static final IntervalLogger log = new IntervalLogger(TransactionViewModel.class); + public static final int SIZE = 768; public static final long SUPPLY = 4292493394837504L; @@ -111,6 +119,24 @@ public static TransactionViewModel fromHash(Tangle tangle, final Hash hash) thro return transactionViewModel; } + /** + * Get TransactionViewModel of a given transaction hashes. Uses @see #Tangle.load(Class, Indexable) + * @param tangle + * @param hashes transaction hash + * @return TransactionViewModel of the transaction + */ + public static List fromHashes(Set hashes, Tangle tangle) { + return hashes.stream().map( + h -> { + try { + return TransactionViewModel.fromHash(tangle, h); + } catch (Exception e) { + log.error("Could not get transaction for hash " + h, e); + } + return null; + }).filter(t -> t != null).collect(Collectors.toList()); + } + /** * Get TransactionViewModel of a given transaction hash. Uses @see #Tangle.maybeHas(Class, Indexable), * which checks the possible existence of an entry in the database. diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 449cd9e7..58eb06e3 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -178,7 +178,6 @@ public API(ApiArgs args) { commandRoute.put(ApiCommand.GET_MISSING_TRANSACTIONS, getMissingTransactions()); commandRoute.put(ApiCommand.CHECK_CONSISTENCY, checkConsistency()); commandRoute.put(ApiCommand.WERE_ADDRESSES_SPENT_FROM, wereAddressesSpentFrom()); - // commandRoute.put(ApiCommand.GET_MILESTONES, wereAddressesSpentFrom()); } /** @@ -702,7 +701,7 @@ private AbstractResponse getConfirmationStatesStatement(final List trans log.trace("tx_confirmations {}:[{}:{}]", transaction.getHash().toString(), transaction.getConfirmations(), (double) transaction.getConfirmations() / n); // is transaction finalized - if(((double)transaction.getConfirmations() / n) > threshold) { + if(transaction.getRoundIndex() > 0 && ((double)transaction.getConfirmations() / n) > threshold) { confirmationStates[count] = 1; } // not finalized yet diff --git a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java index abe258bb..80b7c058 100644 --- a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java @@ -283,20 +283,27 @@ private boolean generateStateDiff(RoundViewModel round) throws LedgerException { Map balanceChanges = generateBalanceDiff(new HashSet<>(), confirmedTips == null? new HashSet<>() : confirmedTips, snapshotProvider.getLatestSnapshot().getIndex() + 1); successfullyProcessed = balanceChanges != null; - if (successfullyProcessed) { - successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( - new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); - if (successfullyProcessed) { - milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); - - if (!balanceChanges.isEmpty()) { - new StateDiffViewModel(balanceChanges, round.index()).store(tangle); - } + if (successfullyProcessed) { + successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( + new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); + TransactionViewModel.fromHashes(confirmedTips, tangle).forEach(tvm -> { + try { + tvm.setRoundIndex(tvm.getRoundIndex() == 0 ? round.index() : tvm.getRoundIndex()); + tvm.update(tangle, snapshotProvider.getInitialSnapshot(), "roundIndex"); + } catch (Exception e) { + log.error("Error during transaction round index update: " + tvm.getHash(), e); } + }); + + milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); + + if (!balanceChanges.isEmpty()) { + new StateDiffViewModel(balanceChanges, round.index()).store(tangle); } - } finally { - snapshotProvider.getLatestSnapshot().unlockRead(); } + } finally { + snapshotProvider.getLatestSnapshot().unlockRead(); + } return successfullyProcessed; } catch (Exception e) { diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java index 07bc196b..99d5ac00 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java @@ -201,8 +201,6 @@ public void setRoundIndexAndConfirmations(RoundViewModel currentRVM, Transaction // The confirmation counter should be incremented with each milestone reference for (Hash tx : referencedTipSet) { TransactionViewModel txvm = TransactionViewModel.fromHash(tangle, tx); - txvm.setRoundIndex(txvm.getRoundIndex() == 0 ? roundIndex : txvm.getRoundIndex()); - txvm.update(tangle, snapshotProvider.getInitialSnapshot(), "roundIndex"); txvm.setConfirmations(txvm.getConfirmations() + 1); txvm.update(tangle, snapshotProvider.getInitialSnapshot(), "confirmation"); } From 663a360f1c63d3226c9cb8743717ec1630f40ed3 Mon Sep 17 00:00:00 2001 From: Cristina Date: Thu, 14 Nov 2019 12:16:44 +0200 Subject: [PATCH 02/36] Sort merkle leaves and sort consistent tips --- src/main/java/net/helix/pendulum/crypto/Merkle.java | 6 ++++++ src/main/java/net/helix/pendulum/service/API.java | 3 +++ 2 files changed, 9 insertions(+) diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java index aea7ffc2..6b3991bb 100644 --- a/src/main/java/net/helix/pendulum/crypto/Merkle.java +++ b/src/main/java/net/helix/pendulum/crypto/Merkle.java @@ -10,6 +10,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; +import java.util.Comparator; import java.util.List; public class Merkle { @@ -59,6 +60,7 @@ public static List> buildMerkleTree(List leaves){ if (leaves.isEmpty()) { leaves.add(Hash.NULL_HASH); } + sortLeaves(leaves); byte[] buffer; Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); int depth = (int) Math.ceil(Math.sqrt(leaves.size())); @@ -190,4 +192,8 @@ public static byte[] padding(byte[] input, int length){ } } + + private void sortLeaves(List leaves) { + leaves.sort(Comparator.comparing((Hash m) -> m.toString())); + } } diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 449cd9e7..e6442417 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -1641,6 +1641,9 @@ private List getConsistentTips() throws Exception { snapshotProvider.getLatestSnapshot().lockRead(); try { WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); + final List transactions = new ArrayList<>(tipsViewModel.getTips()); + transactions.sort(Comparator.comparing((Hash h) -> h.toString())); + for (Hash transaction : tipsViewModel.getTips()) { TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && From cce6858a8f1355aebdd5c05a1fb88acddf9f9982 Mon Sep 17 00:00:00 2001 From: Cristina Date: Thu, 14 Nov 2019 14:23:14 +0200 Subject: [PATCH 03/36] Fix build issue --- src/main/java/net/helix/pendulum/crypto/Merkle.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java index 6b3991bb..cc2c40fc 100644 --- a/src/main/java/net/helix/pendulum/crypto/Merkle.java +++ b/src/main/java/net/helix/pendulum/crypto/Merkle.java @@ -193,7 +193,7 @@ public static byte[] padding(byte[] input, int length){ } } - private void sortLeaves(List leaves) { + static private void sortLeaves(List leaves) { leaves.sort(Comparator.comparing((Hash m) -> m.toString())); } } From a3633d7a92b671ba9c4bcfeea0887f6adecb7276 Mon Sep 17 00:00:00 2001 From: Cristina Date: Fri, 15 Nov 2019 11:42:27 +0200 Subject: [PATCH 04/36] Merkle refactoring --- .../java/net/helix/pendulum/SignedFiles.java | 8 +- .../pendulum/controllers/RoundViewModel.java | 12 +- .../net/helix/pendulum/crypto/Merkle.java | 199 ------------------ .../pendulum/crypto/merkle/MerkleFactory.java | 29 +++ .../pendulum/crypto/merkle/MerkleOptions.java | 72 +++++++ .../pendulum/crypto/merkle/MerkleTree.java | 71 +++++++ .../crypto/merkle/impl/MerkleTreeImpl.java | 198 +++++++++++++++++ .../java/net/helix/pendulum/service/API.java | 4 +- .../milestone/impl/MilestonePublisher.java | 23 +- .../milestone/impl/MilestoneServiceImpl.java | 7 +- .../validator/impl/ValidatorServiceImpl.java | 7 +- .../impl/ValidatorManagerServiceImpl.java | 6 +- .../net/helix/pendulum/utils/KeyfileUtil.java | 93 ++++++++ .../pendulum/utils/bundle/BundleUtils.java | 11 +- 14 files changed, 513 insertions(+), 227 deletions(-) delete mode 100644 src/main/java/net/helix/pendulum/crypto/Merkle.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleFactory.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java create mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java create mode 100644 src/main/java/net/helix/pendulum/utils/KeyfileUtil.java diff --git a/src/main/java/net/helix/pendulum/SignedFiles.java b/src/main/java/net/helix/pendulum/SignedFiles.java index 415ba60f..f036bf82 100644 --- a/src/main/java/net/helix/pendulum/SignedFiles.java +++ b/src/main/java/net/helix/pendulum/SignedFiles.java @@ -1,10 +1,12 @@ package net.helix.pendulum; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.Sha3; import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.MerkleTree; import net.helix.pendulum.model.Hash; import org.apache.commons.lang3.ArrayUtils; import org.bouncycastle.util.encoders.Hex; @@ -46,7 +48,7 @@ private static boolean validateSignature(String signatureFilename, String public if ((line = reader.readLine()) != null) { byte[] lineBytes = Hex.decode(line); - root = Merkle.getMerkleRoot(mode, Winternitz.address(mode, digests), lineBytes, 0, index, depth); + root = MerkleFactory.create(MerkleFactory.MerkleTree, new MerkleOptions(mode)).getMerkleRoot(Winternitz.address(mode, digests), lineBytes, 0, index, depth); } else { root = Winternitz.address(mode, digests); @@ -75,7 +77,7 @@ private static byte[] digestFile(String filename, Sponge sha3) throws IOExceptio messageBytes = Hash.NULL_HASH.bytes(); } int requiredLength = (int) Math.ceil(messageBytes.length / 32.0) * 32; - byte[] finalizedMessage = Merkle.padding(messageBytes, requiredLength); + byte[] finalizedMessage = MerkleTree.padding(messageBytes, requiredLength); // crypto snapshot message sha3.absorb(finalizedMessage, 0, finalizedMessage.length); byte[] signature = new byte[Sha3.HASH_LENGTH]; diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 8d72113d..9e0dd42a 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -2,7 +2,8 @@ import net.helix.pendulum.TransactionValidator; import net.helix.pendulum.conf.BasePendulumConfig; -import net.helix.pendulum.crypto.Merkle; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.IntegerIndex; @@ -12,7 +13,6 @@ import net.helix.pendulum.storage.Persistable; import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Pair; -import net.helix.pendulum.utils.PendulumUtils; import net.helix.pendulum.utils.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -255,7 +255,7 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); + List> merkleTree = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(new ArrayList<>(prevMilestones)); if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { if (prevMilestones.isEmpty()) { trunk.add(Hash.NULL_HASH); @@ -279,7 +279,7 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); + List> merkleTree = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(new ArrayList<>(confirmedTips)); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { if (confirmedTips.isEmpty()){ branch.add(Hash.NULL_HASH); @@ -297,7 +297,7 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); + List> merkleTree = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(new ArrayList<>(prevMilestones)); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { if (prevMilestones.isEmpty()) { branch.add(Hash.NULL_HASH); @@ -475,7 +475,7 @@ public Integer index() { } public Hash getMerkleRoot() { - List> merkleTree = Merkle.buildMerkleTree(new LinkedList<>(getHashes())); + List> merkleTree = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(new LinkedList<>(getHashes())); Hash root = merkleTree.get(merkleTree.size()-1).get(0); return root; } diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java deleted file mode 100644 index cc2c40fc..00000000 --- a/src/main/java/net/helix/pendulum/crypto/Merkle.java +++ /dev/null @@ -1,199 +0,0 @@ -package net.helix.pendulum.crypto; - -import net.helix.pendulum.controllers.RoundViewModel; -import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.model.Hash; -import net.helix.pendulum.model.HashFactory; -import org.bouncycastle.util.encoders.Hex; - -import java.io.*; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; - -public class Merkle { - - public static byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { - int index = indexIn; - final Sponge sha3 = SpongeFactory.create(mode); - for (int i = 0; i < size; i++) { - sha3.reset(); - if ((index & 1) == 0) { - sha3.absorb(hash, 0, hash.length); - sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); - } else { - sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); - sha3.absorb(hash, 0, hash.length); - } - sha3.squeeze(hash, 0, hash.length); - index >>= 1; - } - if(index != 0) { - return Hash.NULL_HASH.bytes(); - } - return hash; - } - - public static List getMerklePath(List> merkleTree, int keyIndex){ - List merklePath = new ArrayList<>((merkleTree.size()-1) * 32); - for (int i = 0; i < merkleTree.size()-1; i++) { - Hash subkey = merkleTree.get(i).get(keyIndex ^ 1); - merklePath.add(subkey == null ? Hash.NULL_HASH : subkey); - keyIndex /= 2; - } - return merklePath; - } - - public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security){ - List keys = new ArrayList<>(1 << pubkeyDepth); - for (int i = 0; i < pubkeyCount; i++) { - int idx = firstIndex + i; - keys.add(HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security))); - } - return buildMerkleTree(keys); - } - - - public static List> buildMerkleTree(List leaves){ - if (leaves.isEmpty()) { - leaves.add(Hash.NULL_HASH); - } - sortLeaves(leaves); - byte[] buffer; - Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); - int depth = (int) Math.ceil(Math.sqrt(leaves.size())); - List> merkleTree = new ArrayList<>(depth + 1); - merkleTree.add(0, leaves); - int row = 1; - // hash two following keys together until only one is left -> merkle tree - while (leaves.size() > 1) { - // Take two following keys (i=0: (k0,k1), i=1: (k2,k3), ...) and get one crypto of them - List nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]); - for (int i = 0; i < nextKeys.size(); i++) { - if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { - // leave the combined key null as well - continue; - } - sha3.reset(); - Hash k1 = leaves.get(i * 2); - Hash k2 = leaves.get(i * 2 + 1); - buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32); - sha3.absorb(buffer, 0, buffer.length); - buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32); - sha3.absorb(buffer, 0, buffer.length); - sha3.squeeze(buffer, 0, buffer.length); - nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); - } - leaves = nextKeys; - merkleTree.add(row++, leaves); - } - return merkleTree; - } - - public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) { - - final TransactionViewModel merkleTx = bundleTransactionViewModels.get(securityLevel); - int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex - - //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) - //TODO: check if its okay here to use bundle hash instead of tx hash - byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); - - //validate leaf signature - ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * securityLevel); - - for (int i = 0; i < securityLevel; i++) { - byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i+1)); - byte[] digest = Winternitz.digest(mode, bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); - bb.put(digest); - } - - byte[] digests = bb.array(); - byte[] address = Winternitz.address(mode, digests); - - //validate Merkle path - byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, - merkleTx.getSignature(), 0, keyIndex, depth); - return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); - } - - public static List> readKeyfile(File keyfile) throws IOException { - try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { - String[] fields = br.readLine().split(" "); - int depth = Integer.parseInt(fields[0]); - List> result = new ArrayList<>(depth + 1); - for (int i = 0; i <= depth; i++) { - fields = br.readLine().split(" "); - int leadingNulls = Integer.parseInt(fields[0]); - List row = new ArrayList<>(); - for (int j = 0; j < leadingNulls; j++) { - row.add(Hash.NULL_HASH); - } - for (int j = 0; j < fields[1].length() / 64; j++) { - row.add(HashFactory.TRANSACTION.create(fields[1].substring(j * 64, (j+1) * 64))); - } - result.add(row); - } - return result; - } - } - - public static String getSeed(File keyfile) throws IOException { - StringBuilder seedBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { - String[] fields = br.readLine().split(" "); - seedBuilder.append(fields[1]); - } - return seedBuilder.toString(); - } - - - public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { - // fill buffer - try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { - // write pubkey depth and seed into buffer - bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); - bw.newLine(); - writeKeys(bw, merkleTree.get(0)); - for (int i = 1; i < merkleTree.size(); i++) { - writeKeys(bw, merkleTree.get(i)); - } - } - } - - private static void writeKeys(BufferedWriter bw, List keys) throws IOException { - int leadingNulls = 0; - while (keys.get(leadingNulls) == null) { - leadingNulls++; - } - bw.write(leadingNulls + " "); - for (int i = leadingNulls; i < keys.size(); i++) { - if (keys.get(i) == null) { - break; - } - bw.write(keys.get(i).toString()); - } - bw.newLine(); - } - - public static byte[] padding(byte[] input, int length){ - if (input.length < length) { - byte[] output = new byte[length]; - System.arraycopy(input, 0, output, length - input.length, input.length); - return output; - } else { - if (input.length > length) { - return Arrays.copyOfRange(input, 0, length); - } else { - return Arrays.copyOfRange(input, 0, input.length); - } - - } - } - - static private void sortLeaves(List leaves) { - leaves.sort(Comparator.comparing((Hash m) -> m.toString())); - } -} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleFactory.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleFactory.java new file mode 100644 index 00000000..c7c732a0 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleFactory.java @@ -0,0 +1,29 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Constructor; + +public enum MerkleFactory { + MerkleTree(MerkleTreeImpl.class); + + private static final Logger log = LoggerFactory.getLogger(MerkleFactory.class); + + Class clazz; + + MerkleFactory(Class clazz) { + this.clazz = clazz; + } + + public static MerkleTree create(MerkleFactory type, MerkleOptions options) { + try { + Constructor constructor = type.clazz.getConstructor(MerkleOptions.class); + return (MerkleTree) constructor.newInstance(options); + } catch (Exception e) { + log.error("Could not instantiate merkle tree object! ", e); + } + return null; + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java new file mode 100644 index 00000000..f5d0fe3a --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java @@ -0,0 +1,72 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.model.Hash; + +public class MerkleOptions { + + private SpongeFactory.Mode mode; + private int securityLevel; + private int depth; + private Hash address; + + public MerkleOptions(SpongeFactory.Mode mode) { + this.mode = mode; + } + + /** + * Merkle option parameters + * + * @param mode + * @param securityLevel + * @param depth + */ + public MerkleOptions(SpongeFactory.Mode mode, int securityLevel, int depth) { + this.mode = mode; + this.securityLevel = securityLevel; + this.depth = depth; + } + + public MerkleOptions(SpongeFactory.Mode mode, int securityLevel, int depth, Hash address) { + this.mode = mode; + this.securityLevel = securityLevel; + this.depth = depth; + this.address = address; + } + + public static MerkleOptions getDefault() { + return new MerkleOptions(SpongeFactory.Mode.S256, 2, 0, Hash.NULL_HASH); + } + + public SpongeFactory.Mode getMode() { + return mode; + } + + public void setMode(SpongeFactory.Mode mode) { + this.mode = mode; + } + + public int getSecurityLevel() { + return securityLevel; + } + + public void setSecurityLevel(int securityLevel) { + this.securityLevel = securityLevel; + } + + public int getDepth() { + return depth; + } + + public void setDepth(int depth) { + this.depth = depth; + } + + public Hash getAddress() { + return address; + } + + public void setAddress(Hash address) { + this.address = address; + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java new file mode 100644 index 00000000..47505330 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java @@ -0,0 +1,71 @@ +package net.helix.pendulum.crypto.merkle; + +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.model.Hash; + +import java.util.Arrays; +import java.util.List; + +public interface MerkleTree { + + /** + * Gets merkle path from the specified key index (Used in merkle transaction message + * + * @param merkleTree + * @param keyIndex + * @return + */ + List getMerklePath(List> merkleTree, int keyIndex); + + /** + * Build merkle tree as lists of lists + * + * @param leaves + * @return + */ + List> buildMerkleTree(List leaves); + + /** + * Gets merkle tree root for given leaves + * + * @param leaves + * @return + */ + byte[] getMerkleRoot(List leaves); + + /** + * Get merkle tree root, based on merkle path + * + * @param mode + * @param hash + * @param bytes merkle path bytes + * @param offset + * @param indexIn + * @param size of elements in path + * @return + */ + byte[] getMerkleRoot(byte[] hash, byte[] bytes, int offset, int indexIn, int size); + + /** + * Validate bundles which contains merkle transaction + * First it validates sender transactions signature + * + * @param bundleTransactionViewModels + * @return + */ + boolean validateMerkleSignature(List bundleTransactionViewModels); + + static byte[] padding(byte[] input, int length) { + if (input.length < length) { + byte[] output = new byte[length]; + System.arraycopy(input, 0, output, length - input.length, input.length); + return output; + } else { + if (input.length > length) { + return Arrays.copyOfRange(input, 0, length); + } else { + return Arrays.copyOfRange(input, 0, input.length); + } + } + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java new file mode 100644 index 00000000..eae109d6 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java @@ -0,0 +1,198 @@ +package net.helix.pendulum.crypto.merkle.impl; + +import net.helix.pendulum.controllers.RoundViewModel; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Sha3; +import net.helix.pendulum.crypto.Sponge; +import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.merkle.MerkleTree; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; + +public class MerkleTreeImpl implements MerkleTree { + + private MerkleOptions options; + + public MerkleTreeImpl(MerkleOptions options) { + this.options = options; + } + + protected List createHashs(List leaves) { + List result = new ArrayList<>(); + result.addAll(leaves); + return result; + } + + protected Hash createHash(Hash hashParent, Hash h1, Hash h2, int row, long index) { + return hashParent; + } + + @Override + public byte[] getMerkleRoot(byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { + int index = indexIn; + final Sponge sha3 = SpongeFactory.create(options.getMode()); + for (int i = 0; i < size; i++) { + sha3.reset(); + if ((index & 1) == 0) { + sha3.absorb(hash, 0, hash.length); + sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); + } else { + sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); + sha3.absorb(hash, 0, hash.length); + } + sha3.squeeze(hash, 0, hash.length); + index >>= 1; + } + if (index != 0) { + return getDefaultMerkleHash().bytes(); + } + return hash; + } + + @Override + public List getMerklePath(List> merkleTree, int keyIndex) { + List merklePath = new ArrayList<>((merkleTree.size() - 1) * Hash.SIZE_IN_BYTES); + for (int i = 0; i < merkleTree.size() - 1; i++) { + Hash subkey = merkleTree.get(i).get(keyIndex ^ 1); + merklePath.add(subkey == null ? getDefaultMerkleHash() : subkey); + keyIndex /= 2; + } + return merklePath; + } + + @Override + public byte[] getMerkleRoot(List leaves) { + List> merkleTree = buildMerkleTree(leaves); + return (merkleTree.get(merkleTree.size() - 1).get(0)).bytes(); + } + + + private void sortLeaves(List leaves) { + leaves.sort(Comparator.comparing((Hash m) -> m.toString())); + } + + @Override + public List> buildMerkleTree(List leaves) { + if (leaves.isEmpty()) { + leaves.add(getDefaultMerkleHash()); + } + byte[] buffer; + addPaddingLeaves(leaves); + sortLeaves(leaves); + Sponge sha3 = SpongeFactory.create(options.getMode()); + List> merkleTree = new ArrayList<>(); + merkleTree.add(0, leaves); + int row = 1; + while (leaves.size() > 1) { + List nextKeys = Arrays.asList(new Hash[getParentNodesSize(leaves)]); + for (int i = 0; i < nextKeys.size(); i++) { + if (areLeavesNull(leaves, i)) { + continue; + } + sha3.reset(); + Hash k1 = getLeaves(leaves, i * 2); + Hash k2 = getLeaves(leaves, i * 2 + 1); + buffer = computeParentHash(sha3, k1, k2); + nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); + } + leaves = nextKeys; + merkleTree.add(row++, createHashs(leaves)); + } + return merkleTree; + } + + public boolean validateMerkleSignature(List bundleTransactionViewModels) { + + final TransactionViewModel merkleTx = bundleTransactionViewModels.get(options.getSecurityLevel()); + int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex + + //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) + //TODO: check if its okay here to use bundle hash instead of tx hash + byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); + + //validate leaf signature + ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * options.getSecurityLevel()); + + for (int i = 0; i < options.getSecurityLevel(); i++) { + byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i + 1)); + byte[] digest = Winternitz.digest(options.getMode(), bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); + bb.put(digest); + } + + byte[] digests = bb.array(); + byte[] address = Winternitz.address(options.getMode(), digests); + + return validateMerklePath(merkleTx.getSignature(), keyIndex, address); + } + + private boolean validateMerklePath(byte[] path, int keyIndex, byte[] address) { + byte[] merkleRoot = getMerkleRoot(address, + path, 0, keyIndex, options.getDepth()); + return HashFactory.ADDRESS.create(merkleRoot).equals(options.getAddress()); + } + + protected void addPaddingLeaves(List leaves) { + int closestPow = (int) Math.pow(2.0, getClosestPow(leaves.size())); + int leaveSize = leaves.size(); + while (leaveSize < closestPow) { + leaves.add(leaveSize++, Hash.NULL_HASH); + } + } + + protected int getTreeDepth(int leavesNumber) { + return getClosestPow(leavesNumber); + } + + protected int getClosestPow(int i) { + int j = 1; + int power = 0; + while (j < i) { + j = j << 1; + power++; + } + return power; + } + + protected Hash getLeaves(List leaves, int index) { + return index < leaves.size() ? leaves.get(index) : getDefaultMerkleHash(); + } + + protected int getParentNodesSize(List leaves) { + return leaves.size() % 2 == 0 ? (leaves.size() / 2) : (leaves.size() / 2 + 1); + } + + private boolean areLeavesNull(List leaves, int i) { + if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { + return true; + } + return false; + } + + private byte[] computeParentHash(Sponge sha3, byte[] k1, byte[] k2) { + byte[] buffer = new byte[Hash.SIZE_IN_BYTES]; + sha3.absorb(k1, 0, k1.length); + sha3.absorb(k2, 0, k2.length); + sha3.squeeze(buffer, 0, buffer.length); + return buffer; + } + + private byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { + return computeParentHash(sha3, copyHash(k1), copyHash(k2)); + } + + private byte[] copyHash(Hash k2) { + return Arrays.copyOfRange(k2 == null ? getDefaultMerkleHash().bytes() : k2.bytes(), 0, Hash.SIZE_IN_BYTES); + } + + private Hash getDefaultMerkleHash() { + return Hash.NULL_HASH; + } +} diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index e6442417..2dfcf108 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -12,6 +12,8 @@ import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.*; import net.helix.pendulum.crypto.*; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.persistables.Transaction; @@ -1681,7 +1683,7 @@ private List addMilestoneReferences(List confirmedTips, int roundInd txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones } //branch - List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); + List> merkleTreeTips = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips } return txToApprove; diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java index 6f518758..9e08990b 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java @@ -1,12 +1,14 @@ package net.helix.pendulum.service.milestone.impl; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.crypto.Merkle; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.service.API; import net.helix.pendulum.service.utils.RoundIndexUtil; import net.helix.pendulum.service.validatormanager.CandidateTracker; +import net.helix.pendulum.utils.KeyfileUtil; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -73,8 +75,9 @@ private void initSeed(PendulumConfig configuration) { } private void writeKeyIndex() throws IOException { - List> merkleTree = Merkle.readKeyfile(new File(keyfile)); - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); + KeyfileUtil keyfileUtil = new KeyfileUtil(); + List> merkleTree = keyfileUtil.readKeyfile(new File(keyfile)); + keyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); } private void readKeyfileMetadata() throws IOException { @@ -87,7 +90,8 @@ private void readKeyfileMetadata() throws IOException { seed = fields[1]; } } - List> merkleTree = Merkle.readKeyfile(new File(keyfile)); + KeyfileUtil keyfileUtil = new KeyfileUtil(); + List> merkleTree = keyfileUtil.readKeyfile(new File(keyfile)); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size() - 1).get(0).bytes()); } @@ -106,7 +110,9 @@ private void doKeyChange() throws Exception { // generate new keyfile int newKeyfileIndex = keyfileIndex + 1; log.debug("Generating Keyfile (idx: " + newKeyfileIndex + ")"); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getValidatorSecurity()); + + KeyfileUtil keyfileUtil = new KeyfileUtil(); + List> merkleTree = keyfileUtil.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getValidatorSecurity()); Hash newAddress = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); // send keyChange bundle to register new address api.publishKeyChange(address.toString(), newAddress, mwm, sign, currentKeyIndex, maxKeyIndex); @@ -114,13 +120,14 @@ private void doKeyChange() throws Exception { keyfileIndex = newKeyfileIndex; address = newAddress; currentKeyIndex = maxKeyIndex * keyfileIndex; - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + keyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); } private void generateKeyfile(String seed) throws Exception { log.debug("Generating Keyfile (idx: " + keyfileIndex + ")"); - List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getValidatorSecurity()); - Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + KeyfileUtil keyfileUtil = new KeyfileUtil(); + List> merkleTree = keyfileUtil.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getValidatorSecurity()); + keyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java index cae1f9a4..86250137 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java @@ -6,8 +6,9 @@ import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.controllers.RoundViewModel; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.IntegerIndex; import net.helix.pendulum.model.StateDiff; @@ -183,7 +184,9 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); + MerkleOptions options = new MerkleOptions(mode, securityLevel,config.getMilestoneKeyDepth(), senderAddress); + + boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); log.trace("valid signature: {}", validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { diff --git a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java index 36780497..9c9953df 100644 --- a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java @@ -3,8 +3,9 @@ import net.helix.pendulum.BundleValidator; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; @@ -58,7 +59,9 @@ public ValidatorValidity validateValidators(TransactionViewModel transactionView // validate signature Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getValidatorManagerKeyDepth()); + MerkleOptions options = new MerkleOptions(mode, securityLevel,config.getMilestoneKeyDepth(), senderAddress); + + boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getValidatorManagerAddress().equals(senderAddress) && validSignature)) { return VALID; diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java index ca25e285..f1d982a7 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java @@ -3,8 +3,9 @@ import net.helix.pendulum.BundleValidator; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; @@ -74,7 +75,8 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM if (tail.getHash().equals(transactionViewModel.getHash()) && isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); + MerkleOptions options = new MerkleOptions(mode, securityLevel,config.getMilestoneKeyDepth(), senderAddress); + boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validator.contains(senderAddress)) && validSignature) { return VALID; diff --git a/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java b/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java new file mode 100644 index 00000000..8b53b18a --- /dev/null +++ b/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java @@ -0,0 +1,93 @@ +package net.helix.pendulum.utils; + +import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; +import org.bouncycastle.util.encoders.Hex; + +import java.io.*; +import java.util.ArrayList; +import java.util.List; + +/** + * It contains the required methods to manage key files (read, write, build) + */ +public class KeyfileUtil { + + private static int ADDRESS_STRING_LENGTH = Hash.SIZE_IN_BYTES * 2; + + public String getSeed(File keyfile) throws IOException { + StringBuilder seedBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + seedBuilder.append(fields[1]); + } + return seedBuilder.toString(); + } + + public List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security) { + List keys = new ArrayList<>(1 << pubkeyDepth); + for (int i = 0; i < pubkeyCount; i++) { + int idx = firstIndex + i; + keys.add(createKeyHash(seed, security, idx)); + } + return MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(keys); + } + + public void writeKeys(BufferedWriter bw, List keys) throws IOException { + int leadingNulls = 0; + while (keys.get(leadingNulls) == null) { + leadingNulls++; + } + bw.write(leadingNulls + " "); + for (int i = leadingNulls; i < keys.size(); i++) { + if (keys.get(i) == null) { + break; + } + bw.write(writeMerkleNode(keys.get(i))); + } + bw.newLine(); + } + + public void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { + try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { + bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); + bw.newLine(); + writeKeys(bw, merkleTree.get(0)); + for (int i = 1; i < merkleTree.size(); i++) { + writeKeys(bw, merkleTree.get(i)); + } + } + } + + public List> readKeyfile(File keyfile) throws IOException { + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + int depth = Integer.parseInt(fields[0]); + List> result = new ArrayList<>(depth + 1); + for (int i = 0; i <= depth; i++) { + fields = br.readLine().split(" "); + int leadingNulls = Integer.parseInt(fields[0]); + List row = new ArrayList<>(); + for (int j = 0; j < leadingNulls; j++) { + row.add(Hash.NULL_HASH); + } + for (int j = 0; j < fields[1].length() / ADDRESS_STRING_LENGTH; j++) { + row.add(HashFactory.ADDRESS.create(fields[1].substring(j * ADDRESS_STRING_LENGTH, (j + 1) * ADDRESS_STRING_LENGTH))); + } + result.add(row); + } + return result; + } + } + + private Hash createKeyHash(String seed, int security, int idx) { + return HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security)); + } + + private String writeMerkleNode(Hash key) { + return key.toString(); + } +} diff --git a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java index df8134fa..bc21c6a5 100644 --- a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java @@ -1,11 +1,13 @@ package net.helix.pendulum.utils.bundle; import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; +import net.helix.pendulum.crypto.merkle.MerkleFactory; +import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; +import net.helix.pendulum.utils.KeyfileUtil; import net.helix.pendulum.utils.Serializer; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -159,12 +161,13 @@ private byte[] addBundleHash(List bundle, SpongeFactory.Mode mode) { private void signBundle(String filepath, byte[] merkleTransaction, List senderTransactions, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { // Get merkle path and store in signatureMessageFragment of Sibling Transaction File keyfile = new File(filepath); - List> merkleTree = Merkle.readKeyfile(keyfile); - String seed = Merkle.getSeed(keyfile); + KeyfileUtil keyfileUtil = new KeyfileUtil(); + List> merkleTree = keyfileUtil.readKeyfile(keyfile); + String seed = keyfileUtil.getSeed(keyfile); int security = senderTransactions.size(); // create merkle path from keyfile - List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); + List merklePath = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).getMerklePath(merkleTree, keyIndex % maxKeyIndex); byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); System.arraycopy(path, 0, merkleTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); From 9e81d2149a345a572333bc0a5ba89add353f35d5 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Thu, 31 Oct 2019 15:25:26 +0300 Subject: [PATCH 05/36] give the user a chance to change the spent addresses db path --- docker/docker-compose-dev.yml | 66 +++++++++++++++ docker/run-dev.sh | 83 +++++++++++++++++++ .../impl/SpentAddressesProviderImpl.java | 7 +- 3 files changed, 154 insertions(+), 2 deletions(-) create mode 100644 docker/docker-compose-dev.yml create mode 100644 docker/run-dev.sh diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml new file mode 100644 index 00000000..60dd4933 --- /dev/null +++ b/docker/docker-compose-dev.yml @@ -0,0 +1,66 @@ +version: "3.1" + +networks: + helix_network: + +secrets: + seed: + file: .seed # + +services: + relayer: + container_name: relayer + # build first + image: ${DOCKER_IMAGE} + hostname: relayer + restart: on-failure + volumes: + - ../target/pendulum-1.0.3.jar:/pendulum/target/pendulum-dev.jar + # uncomment if you need persistent data + # - /docker-volumes/relayer/data:/pendulum/data + environment: + - DOCKER_PLM_JAR_FILE=pendulum-dev.jar + - DOCKER_PLM_REMOTE_LIMIT_API="interruptAttachToTangle" + - JAVA_MAX_MEMORY=500m + - JAVA_MIN_MEMORY=256m + - DOCKER_PLM_LOGGING_LEVEL=debug + - JAVA_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5044 + ports: + - "6550:6550" + - "8085:8085" + - "5044:5044" + expose: + - "4100" + command: ["-p","8085","-n", "udp://backend:4100", "--testnet"] + networks: + helix_network: + + backend: + container_name: backend + image: ${DOCKER_IMAGE} + hostname: backend + restart: on-failure + volumes: + - ../target/pendulum-1.0.3.jar:/pendulum/target/pendulum-dev.jar + #volumes: + # - /docker-volumes/backend/data:/pendulum/data + # - ./backend/seed.txt:/pendulum/conf/seed.txt:ro + environment: + - DOCKER_PLM_REMOTE_LIMIT_API="interruptAttachToTangle" + - DOCKER_PLM_JAR_FILE=pendulum-dev.jar + - JAVA_MAX_MEMORY=500m + - JAVA_MIN_MEMORY=256m + - DOCKER_PLM_LOGGING_LEVEL=debug + - JAVA_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5045 + ports: + - "6551:6550" + - "8086:8085" + - "5045:5045" + expose: + - "4100" + secrets: + - seed + command: ["-p","8085","-n", "udp://relayer:4100", "--testnet", "--validator", "/run/secrets/seed"] + networks: + helix_network: + diff --git a/docker/run-dev.sh b/docker/run-dev.sh new file mode 100644 index 00000000..7a25aeaa --- /dev/null +++ b/docker/run-dev.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +usage() { + echo "Usage (local build + run): run-dev --build --image " + echo "Usage (git build + run): run-dev --git --image " + echo "Usage (pull + run): run-dev --image " + echo "Usage (local jar): run-dev --jar " + echo " (optional) with validator seed: --seed " +} + +run() { + echo "killing compose" + docker-compose -f docker-compose-dev.yml down + echo "spinning up compose" + docker-compose -f docker-compose-dev.yml up -d +} + +restart() { + echo "restarting" + docker-compose -f docker-compose-dev.yml restart relayer + docker-compose -f docker-compose-dev.yml restart backend +} + +# if no arguments supplied, display usage +if [ $# -le 0 ] +then + usage + exit 1 +fi + +build= +DOCKER_IMAGE= + +while [ "$1" != "" ]; do + case $1 in + -i | --image ) shift + DOCKER_IMAGE=$1 + ;; + -b | --build ) build=1 + ;; + --restart ) restart + exit + ;; + --local ) jar=1 + ;; + --git ) git=$1 + ;; + --seed ) echo "$1" > .seed + ;; + -h | --help ) usage + exit + ;; + * ) usage + exit 1 + esac + shift +done + +if [ "$jar" = "1" ]; then + echo "Building jar only, skipping tests" + mvn jar:jar -Dmaven.test.skip=true + run +fi + +if [ $DOCKER_IMAGE = "" ] +then + usage + exit 1 +fi + +export DOCKER_IMAGE=$DOCKER_IMAGE + +if [ "$build" = "1" ]; then + echo "building $DOCKER_IMAGE" + docker build -t $DOCKER_IMAGE ../ +fi + +if [ "$git" != "" ]; then + echo "building from github: $git" + docker build -t $DOCKER_IMAGE $git +fi + +run \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 57ede2e6..8f6c090e 100644 --- a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -14,6 +14,7 @@ import net.helix.pendulum.utils.Pair; import java.io.BufferedReader; +import java.io.File; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Collection; @@ -41,8 +42,10 @@ public class SpentAddressesProviderImpl implements SpentAddressesProvider { public SpentAddressesProviderImpl() { Map> columnFamilies = new HashMap<>(); columnFamilies.put("spent-addresses", SpentAddress.class); - this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider(SPENT_ADDRESSES_DB, - SPENT_ADDRESSES_LOG, 1000, + String addressDBPath = System.getProperty("spent.addresses.db", ""); + this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + addressDBPath + SPENT_ADDRESSES_DB, + addressDBPath + SPENT_ADDRESSES_LOG, 1000, columnFamilies, null); } From 44bd27cabc8b80633770bbca964bb31af00858eb Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Fri, 15 Nov 2019 17:56:51 +0300 Subject: [PATCH 06/36] implement thread-safe BoundedLinkedSet add push method evicting the last queue element --- .../impl/BoundedLinkedListImpl.java | 236 ++++++++++++++++++ .../interfaces/BoundedLinkedSet.java | 52 ++++ 2 files changed, 288 insertions(+) create mode 100644 src/main/java/net/helix/pendulum/utils/collections/impl/BoundedLinkedListImpl.java create mode 100644 src/main/java/net/helix/pendulum/utils/collections/interfaces/BoundedLinkedSet.java diff --git a/src/main/java/net/helix/pendulum/utils/collections/impl/BoundedLinkedListImpl.java b/src/main/java/net/helix/pendulum/utils/collections/impl/BoundedLinkedListImpl.java new file mode 100644 index 00000000..cc154543 --- /dev/null +++ b/src/main/java/net/helix/pendulum/utils/collections/impl/BoundedLinkedListImpl.java @@ -0,0 +1,236 @@ +package net.helix.pendulum.utils.collections.impl; + +import net.helix.pendulum.utils.collections.interfaces.BoundedLinkedSet; +import org.apache.commons.collections4.set.ListOrderedSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Date: 2019-11-15 + * Author: zhelezov + */ +public class BoundedLinkedListImpl implements BoundedLinkedSet { + + private static final Logger log = LoggerFactory.getLogger(BoundedLinkedListImpl.class); + + + private ListOrderedSet queue = ListOrderedSet.listOrderedSet(new LinkedList()); + private ReentrantLock lock = new ReentrantLock(true); + private int maxCapacity; + + public BoundedLinkedListImpl(int maxCapacity) { + this.maxCapacity = maxCapacity; + } + + @Override + public LinkedList drain() { + LinkedList drainedTo = new LinkedList<>(); + lock.lock(); + try { + drainedTo.addAll(queue); + queue.clear(); + } finally { + lock.unlock(); + } + return drainedTo; + } + + @Override + public E poll() { + lock.lock(); + E first = null; + try { + Iterator it = queue.iterator(); + if (it.hasNext()) { + first = it.next(); + it.remove(); + } + } finally { + lock.unlock(); + } + return first; + } + + @Override + public E peek() { + lock.lock(); + E first = null; + try { + Iterator it = queue.iterator(); + if (it.hasNext()) { + first = it.next(); + } + } finally { + lock.unlock(); + } + return first; + } + + @Override + public boolean push(E element) { + lock.lock(); + try { + if (queue.contains(element)) { + return false; + } + + if (queue.size() >= maxCapacity) { + // TODO: different eviction policies + log.warn("The queue reached it max capacity, dropping the last element"); + queue.remove(queue.size()-1); + } + + queue.add(0, element); + } finally { + lock.unlock(); + } + return false; + } + + @Override + public List batchPoll(int size) { + lock.lock(); + List popped = new LinkedList<>(); + try { + Iterator it = queue.iterator(); + int counter = 0; + while ((counter < size) && it.hasNext()) { + popped.add(it.next()); + it.remove(); + counter++; + } + return popped; + } finally { + lock.unlock(); + } + } + + @Override + public int getMaxSize() { + return maxCapacity; + } + + @Override + public int size() { + lock.lock(); + try { + return queue.size(); + } finally { + lock.unlock(); + } + } + + @Override + public boolean isEmpty() { + lock.lock(); + try { + return queue.isEmpty(); + } finally { + lock.unlock(); + } + } + + @Override + public boolean contains(Object o) { + lock.lock(); + try { + return queue.contains(o); + } finally { + lock.unlock(); + } + } + + @Override + public Iterator iterator() { + lock.lock(); + try { + List copy = new ArrayList<>(queue); + return copy.iterator(); + } finally { + lock.unlock(); + } + } + + @Override + public Object[] toArray() { + lock.lock(); + try { + return queue.toArray(); + } finally { + lock.unlock(); + } + } + + @Override + public T[] toArray(T[] a) { + lock.lock(); + try { + return queue.toArray(a); + } finally { + lock.unlock(); + } + } + + @Override + public boolean add(E e) { + lock.lock(); + try { + if (queue.size() >= maxCapacity) { + // TODO: different eviction policies + log.warn("The queue reached it max capacity, dropping the element"); + return false; + } + return queue.add(e); + } finally { + lock.unlock(); + } + } + + @Override + public boolean remove(Object o) { + lock.lock(); + try { + return queue.remove(o); + } finally { + lock.unlock(); + } + } + + @Override + public boolean containsAll(Collection c) { + lock.lock(); + try { + return queue.containsAll(c); + } finally { + lock.unlock(); + } + } + + @Override + public boolean addAll(Collection c) { + throw new UnsupportedOperationException("Add all not supported"); + } + + @Override + public boolean removeAll(Collection c) { + throw new UnsupportedOperationException("Remove all not supported"); + } + + @Override + public boolean retainAll(Collection c) { + throw new UnsupportedOperationException("Retain all not supported"); + } + + @Override + public void clear() { + lock.lock(); + try { + queue.clear(); + } finally { + lock.unlock(); + } + } +} diff --git a/src/main/java/net/helix/pendulum/utils/collections/interfaces/BoundedLinkedSet.java b/src/main/java/net/helix/pendulum/utils/collections/interfaces/BoundedLinkedSet.java new file mode 100644 index 00000000..fc57c9f1 --- /dev/null +++ b/src/main/java/net/helix/pendulum/utils/collections/interfaces/BoundedLinkedSet.java @@ -0,0 +1,52 @@ +package net.helix.pendulum.utils.collections.interfaces; + +import java.util.LinkedList; +import java.util.List; + +/** + * A double-sided queue (e.g. can work as a stack and as a queue) + * with bounded capacity and without duplicates. Similar to LinkedHashSet + * but with following differences: + * + * -- thread-safe + * -- A copy iterator never throws ConcurrentModificationException + * as a new copy is created + * + * Date: 2019-11-15 + * Author: zhelezov + */ +public interface BoundedLinkedSet extends BoundedCollection { + + /** + * Drains the queue into a copy + * @return List of the elements in the same order as they were in the queue + */ + LinkedList drain(); + + /** + * Retrieves the first element + * @return null if empty + */ + E poll(); + + /** + * Returns the first element without removal + * @return null if empty + */ + E peek(); + + + /** + * Adds the top of the queue + * @return true if added + */ + boolean push(E element); + + /** + * Retrieves up to size top elements from the queue + * @param size the maximal number of elements to retrieve + * @return the list containing the elements + */ + List batchPoll(int size); + +} From aee8887da4519de461b7065da5c9e6d6ae584836 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Thu, 21 Nov 2019 18:17:30 +0300 Subject: [PATCH 07/36] log all tangle published messages into a separate file for processing --- .../net/helix/pendulum/storage/Tangle.java | 1 + src/main/resources/logback-save.xml | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/main/java/net/helix/pendulum/storage/Tangle.java b/src/main/java/net/helix/pendulum/storage/Tangle.java index d118e3e5..4462efb9 100644 --- a/src/main/java/net/helix/pendulum/storage/Tangle.java +++ b/src/main/java/net/helix/pendulum/storage/Tangle.java @@ -146,6 +146,7 @@ public void publish(String message, Object... objects) { for(MessageQProvider provider: this.messageQProviders) { provider.publish(message, objects); } + log.trace(String.format(message, objects)); } public Set keysWithMissingReferences(Class modelClass, Class referencedClass) throws Exception { diff --git a/src/main/resources/logback-save.xml b/src/main/resources/logback-save.xml index f74267b8..b2cfae52 100644 --- a/src/main/resources/logback-save.xml +++ b/src/main/resources/logback-save.xml @@ -58,6 +58,24 @@ + + ${user.dir}/logs/tangle-${instance.name}.log + + + + ${user.dir}/logs/tangle-${instance.name}.%d{yyyy-MM-dd}.log + + + 30 + 10GB + + + + + %d %msg%n + + + System.err @@ -80,4 +98,8 @@ + + + + From 7e4474b0e740da50dd2db96c4899fe6074cc6d33 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Thu, 21 Nov 2019 18:17:30 +0300 Subject: [PATCH 08/36] log all tangle published messages into a separate file for processing --- .../net/helix/pendulum/storage/Tangle.java | 1 + src/main/resources/logback-save.xml | 22 +++++++++++++++++++ 2 files changed, 23 insertions(+) diff --git a/src/main/java/net/helix/pendulum/storage/Tangle.java b/src/main/java/net/helix/pendulum/storage/Tangle.java index d118e3e5..4462efb9 100644 --- a/src/main/java/net/helix/pendulum/storage/Tangle.java +++ b/src/main/java/net/helix/pendulum/storage/Tangle.java @@ -146,6 +146,7 @@ public void publish(String message, Object... objects) { for(MessageQProvider provider: this.messageQProviders) { provider.publish(message, objects); } + log.trace(String.format(message, objects)); } public Set keysWithMissingReferences(Class modelClass, Class referencedClass) throws Exception { diff --git a/src/main/resources/logback-save.xml b/src/main/resources/logback-save.xml index f74267b8..b2cfae52 100644 --- a/src/main/resources/logback-save.xml +++ b/src/main/resources/logback-save.xml @@ -58,6 +58,24 @@ + + ${user.dir}/logs/tangle-${instance.name}.log + + + + ${user.dir}/logs/tangle-${instance.name}.%d{yyyy-MM-dd}.log + + + 30 + 10GB + + + + + %d %msg%n + + + System.err @@ -80,4 +98,8 @@ + + + + From 52228a54bb007d22d33d4db01227cb27f0cde3d2 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Thu, 21 Nov 2019 18:32:20 +0300 Subject: [PATCH 09/36] publish to tangle when transaction is solidified --- src/main/java/net/helix/pendulum/TransactionValidator.java | 1 + .../net/helix/pendulum/controllers/TransactionViewModel.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/net/helix/pendulum/TransactionValidator.java b/src/main/java/net/helix/pendulum/TransactionValidator.java index 66890dc2..adb93aa8 100644 --- a/src/main/java/net/helix/pendulum/TransactionValidator.java +++ b/src/main/java/net/helix/pendulum/TransactionValidator.java @@ -479,6 +479,7 @@ private boolean quickSetSolid(final TransactionViewModel transactionViewModel) t if(solid) { transactionViewModel.updateSolid(true); transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); + tangle.publish("sldf %s", transactionViewModel.getHash()); return true; } } diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index 5079801c..06ae87de 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -611,6 +611,7 @@ public static void updateSolidTransactions(Tangle tangle, Snapshot initialSnapsh if(!transactionViewModel.isSolid()) { transactionViewModel.updateSolid(true); transactionViewModel.update(tangle, initialSnapshot, "solid|height"); + tangle.publish("sldf %s", transactionViewModel.getHash()); } } } From 780c1486d5b38726bb83c0c8e010ea861777906a Mon Sep 17 00:00:00 2001 From: oliverfn <17864291+oliverfn@users.noreply.github.com> Date: Thu, 21 Nov 2019 16:38:17 +0100 Subject: [PATCH 10/36] Update TransactionViewModel.java Remove unused import --- .../net/helix/pendulum/controllers/TransactionViewModel.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index 7af1db1b..702e24c1 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -3,7 +3,6 @@ import net.helix.pendulum.model.*; import net.helix.pendulum.model.persistables.*; import net.helix.pendulum.service.milestone.MilestoneTracker; -import net.helix.pendulum.service.milestone.impl.MilestoneTrackerImpl; import net.helix.pendulum.service.snapshot.Snapshot; import net.helix.pendulum.storage.Indexable; import net.helix.pendulum.storage.Persistable; From 1b602b63d28522b67d14fe1849f36c452b8e4b85 Mon Sep 17 00:00:00 2001 From: oliverfn <17864291+oliverfn@users.noreply.github.com> Date: Thu, 21 Nov 2019 16:39:04 +0100 Subject: [PATCH 11/36] Bump to 1.0.6-pre-release --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 170fdcbf..ae43f2d1 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Pendulum is a quorum based [Tangle](https://github.com/iotaledger/iri/) implementation designed towards reliable timekeeping and high-throughput messaging. -- **Latest release:** 1.0.5 pre-release +- **Latest release:** 1.0.6 pre-release - **License:** GPLv3 Special thanks to all of the [IOTA Contributors](https://github.com/iotaledger/iri/graphs/contributors)! From cd64eed1a6e19535f6f7386e0f6ff005dc07f745 Mon Sep 17 00:00:00 2001 From: oliverfn <17864291+oliverfn@users.noreply.github.com> Date: Thu, 21 Nov 2019 16:39:36 +0100 Subject: [PATCH 12/36] Bump to 1.0.6-pre-release --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 2190b68d..c96603b4 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ pendulum - 1.0.5 + 1.0.6 Pendulum From 64c79d6bff5c99070c51f8b54928b5ec369ed5ad Mon Sep 17 00:00:00 2001 From: oliverfn <17864291+oliverfn@users.noreply.github.com> Date: Thu, 21 Nov 2019 16:40:31 +0100 Subject: [PATCH 13/36] Bump to 1.0.6-pre-release --- src/main/java/net/helix/pendulum/Main.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/pendulum/Main.java b/src/main/java/net/helix/pendulum/Main.java index e78443af..3a0770bc 100644 --- a/src/main/java/net/helix/pendulum/Main.java +++ b/src/main/java/net/helix/pendulum/Main.java @@ -46,7 +46,7 @@ public class Main { public static final String MAINNET_NAME = "Pendulum"; public static final String TESTNET_NAME = "Pendulum Testnet"; - public static final String VERSION = "1.0.5"; + public static final String VERSION = "1.0.6"; /** From 459b3f9306bc0c31afa15b456a91fac6bc29ca6c Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Mon, 25 Nov 2019 18:53:48 +0300 Subject: [PATCH 14/36] break down complex conditions into independent checks --- .../impl/LatestSolidMilestoneTrackerImpl.java | 62 ++++++++++++++----- 1 file changed, 46 insertions(+), 16 deletions(-) diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 05675696..53ac6b6d 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -143,40 +143,70 @@ public void shutdown() { @Override public void trackLatestSolidMilestones() throws MilestoneException { try { + int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + int latestMilestoneIndex = milestoneTracker.getCurrentRoundIndex(); + log.delegate().trace("Current Solid Round = " + currentSolidRoundIndex + ", Current Milestone index = " + latestMilestoneIndex + + ", Still in Round = " + (currentSolidRoundIndex == latestMilestoneIndex - 1) + + ", Round active = " + milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime())); + + RoundViewModel nextRound; - if (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) { - log.delegate().trace("Current Solid Round = " + currentSolidRoundIndex + ", Current Round = " + milestoneTracker.getCurrentRoundIndex() + ", Still in Round = " + (currentSolidRoundIndex == milestoneTracker.getCurrentRoundIndex() - 1) + ", Round active = " + milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime())); - } - while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) - && (currentSolidRoundIndex != milestoneTracker.getCurrentRoundIndex() - 1 || !milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime()))) { + //if (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) { + // log.delegate().trace("Current Solid Round = " + currentSolidRoundIndex + ", Current Round = " + milestoneTracker.getCurrentRoundIndex() + ", Still in Round = " + (currentSolidRoundIndex == milestoneTracker.getCurrentRoundIndex() - 1) + ", Round active = " + milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime())); + //} + while (!Thread.currentThread().isInterrupted()) { + currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + latestMilestoneIndex = milestoneTracker.getCurrentRoundIndex(); + + if (currentSolidRoundIndex >= latestMilestoneIndex) { + tracer.trace("Latest milestone index is behind current solid round"); + break; + } + + if ((currentSolidRoundIndex == latestMilestoneIndex - 1) && milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime())) { + tracer.trace("The round is still active, nothing to update"); + break; + } + nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); if (nextRound == null) { // round has finished without milestones RoundViewModel latest = RoundViewModel.latest(tangle); - if (latest != null && latest.index() > currentSolidRoundIndex + 1 && isRoundSolid(latest)) { - log.delegate().trace("Round #" + (currentSolidRoundIndex + 1) + " has finished without milestones"); - nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>()); - nextRound.store(tangle); - tracer.trace("created and stored empty round: {}", nextRound.index()); + tracer.trace("Latest round: {}", latest); + if (latest == null) { + tracer.trace("Latest round is null"); + break; + } + + if (latest.index() <= currentSolidRoundIndex + 1) { + tracer.trace("Latest round is not ahead current solid round "); + break; } - // round hasn't finished yet - else { - tracer.trace("round has not finished: {}", latest); + + if (!isRoundSolid(latest)) { + tracer.trace("Latest round is not solid"); break; } + + log.delegate().trace("Round #" + (currentSolidRoundIndex + 1) + " has finished without milestones"); + nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>()); + nextRound.store(tangle); + tracer.trace("created and stored empty round: {}", nextRound.index()); } - log.delegate().trace("Round = " + nextRound.index() + ", solid = " + isRoundSolid(nextRound)); - if (isRoundSolid(nextRound)) { + + boolean nextRoundSolid = isRoundSolid(nextRound); + + log.delegate().trace("Round = " + nextRound.index() + ", solid = " + nextRoundSolid); + if (nextRoundSolid) { tracer.trace("solid round: {}", nextRound.toString()); // TODO: Ask Oliver about these classes? //syncValidatorTracker(); //syncLatestMilestoneTracker(nextRound.index()); applyRoundToLedger(nextRound); logChange(currentSolidRoundIndex); - currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); tangle.publish("ctx %s %d", nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)), nextRound.index()); log.delegate().trace("ctx= " + nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)) + ", round=" + nextRound.index()); } From 997d46776f66493d724e8d1f06507af456452d8c Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Tue, 26 Nov 2019 19:52:35 +0300 Subject: [PATCH 15/36] CandidateTrackerImpl tries to recover the latest persisted validator set on init --- .../impl/CandidateTrackerImpl.java | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/CandidateTrackerImpl.java index abc11964..f628c3f9 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/CandidateTrackerImpl.java @@ -1,10 +1,7 @@ package net.helix.pendulum.service.validatormanager.impl; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.controllers.AddressViewModel; -import net.helix.pendulum.controllers.BundleViewModel; -import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.controllers.ValidatorViewModel; +import net.helix.pendulum.controllers.*; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; @@ -143,6 +140,7 @@ public CandidateTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide validators = config.getInitialValidators(); startRound = RoundIndexUtil.getRound(RoundIndexUtil.getCurrentTime(), config.getGenesisTime(), config.getRoundDuration(), 2); + recoverValidators(); return this; } @@ -303,6 +301,31 @@ public boolean processCandidate(TransactionViewModel transaction) throws Validat } } + /** + * Try to recover the latest persisted validator set. + * + */ + private void recoverValidators() { + try { + RoundViewModel latest = RoundViewModel.latest(tangle); + if (latest == null) { + log.debug("Latest round is null"); + return; + } + + ValidatorViewModel vvm = ValidatorViewModel.findClosestPrevValidators(tangle, latest.index()); + Set recovered = vvm.getHashes(); + if (recovered == null || recovered.isEmpty()) { + log.debug("The recovered set of validators is empty"); + return; + } + + validators = new HashSet<>(recovered); + } catch (Exception e) { + log.error("Could not recover the latest validator set, using the initial set", e); + } + } + /** * {@inheritDoc} * Adds a candidate to the {@code candidatesToNominate} From c38ff94ff265ec57a5e4fe4de1a04aaaa6e43dd1 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Tue, 26 Nov 2019 20:08:07 +0300 Subject: [PATCH 16/36] Update README.MD add `sldf` ZMQ topic --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index ae43f2d1..af551eff 100644 --- a/README.md +++ b/README.md @@ -196,6 +196,7 @@ Currently the following topics are covered: | `nav` | newly added validators | | `nrv` | newly removed validators | | `cvs` | current validators | +| `sldf` | transactions recently solidified by the node | | `
` | Watching all traffic on a specified address | From 1f51ee6c21fbaf169b9f79936b505e87816eb146 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Wed, 27 Nov 2019 16:20:42 +0300 Subject: [PATCH 17/36] Revert "Sort merkle leaves and sort consistent tips" --- .../java/net/helix/pendulum/SignedFiles.java | 8 +- .../pendulum/controllers/RoundViewModel.java | 12 +- .../net/helix/pendulum/crypto/Merkle.java | 193 +++++++++++++++++ .../pendulum/crypto/merkle/MerkleFactory.java | 29 --- .../pendulum/crypto/merkle/MerkleOptions.java | 72 ------- .../pendulum/crypto/merkle/MerkleTree.java | 71 ------- .../crypto/merkle/impl/MerkleTreeImpl.java | 198 ------------------ .../java/net/helix/pendulum/service/API.java | 7 +- .../milestone/impl/MilestonePublisher.java | 23 +- .../milestone/impl/MilestoneServiceImpl.java | 7 +- .../validator/impl/ValidatorServiceImpl.java | 7 +- .../impl/ValidatorManagerServiceImpl.java | 6 +- .../net/helix/pendulum/utils/KeyfileUtil.java | 93 -------- .../pendulum/utils/bundle/BundleUtils.java | 11 +- 14 files changed, 221 insertions(+), 516 deletions(-) create mode 100644 src/main/java/net/helix/pendulum/crypto/Merkle.java delete mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleFactory.java delete mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java delete mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java delete mode 100644 src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java delete mode 100644 src/main/java/net/helix/pendulum/utils/KeyfileUtil.java diff --git a/src/main/java/net/helix/pendulum/SignedFiles.java b/src/main/java/net/helix/pendulum/SignedFiles.java index f036bf82..415ba60f 100644 --- a/src/main/java/net/helix/pendulum/SignedFiles.java +++ b/src/main/java/net/helix/pendulum/SignedFiles.java @@ -1,12 +1,10 @@ package net.helix.pendulum; +import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.Sha3; import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; -import net.helix.pendulum.crypto.merkle.MerkleTree; import net.helix.pendulum.model.Hash; import org.apache.commons.lang3.ArrayUtils; import org.bouncycastle.util.encoders.Hex; @@ -48,7 +46,7 @@ private static boolean validateSignature(String signatureFilename, String public if ((line = reader.readLine()) != null) { byte[] lineBytes = Hex.decode(line); - root = MerkleFactory.create(MerkleFactory.MerkleTree, new MerkleOptions(mode)).getMerkleRoot(Winternitz.address(mode, digests), lineBytes, 0, index, depth); + root = Merkle.getMerkleRoot(mode, Winternitz.address(mode, digests), lineBytes, 0, index, depth); } else { root = Winternitz.address(mode, digests); @@ -77,7 +75,7 @@ private static byte[] digestFile(String filename, Sponge sha3) throws IOExceptio messageBytes = Hash.NULL_HASH.bytes(); } int requiredLength = (int) Math.ceil(messageBytes.length / 32.0) * 32; - byte[] finalizedMessage = MerkleTree.padding(messageBytes, requiredLength); + byte[] finalizedMessage = Merkle.padding(messageBytes, requiredLength); // crypto snapshot message sha3.absorb(finalizedMessage, 0, finalizedMessage.length); byte[] signature = new byte[Sha3.HASH_LENGTH]; diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 9e0dd42a..8d72113d 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -2,8 +2,7 @@ import net.helix.pendulum.TransactionValidator; import net.helix.pendulum.conf.BasePendulumConfig; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.IntegerIndex; @@ -13,6 +12,7 @@ import net.helix.pendulum.storage.Persistable; import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Pair; +import net.helix.pendulum.utils.PendulumUtils; import net.helix.pendulum.utils.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -255,7 +255,7 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(new ArrayList<>(prevMilestones)); + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { if (prevMilestones.isEmpty()) { trunk.add(Hash.NULL_HASH); @@ -279,7 +279,7 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); - List> merkleTree = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(new ArrayList<>(confirmedTips)); + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { if (confirmedTips.isEmpty()){ branch.add(Hash.NULL_HASH); @@ -297,7 +297,7 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(new ArrayList<>(prevMilestones)); + List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { if (prevMilestones.isEmpty()) { branch.add(Hash.NULL_HASH); @@ -475,7 +475,7 @@ public Integer index() { } public Hash getMerkleRoot() { - List> merkleTree = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(new LinkedList<>(getHashes())); + List> merkleTree = Merkle.buildMerkleTree(new LinkedList<>(getHashes())); Hash root = merkleTree.get(merkleTree.size()-1).get(0); return root; } diff --git a/src/main/java/net/helix/pendulum/crypto/Merkle.java b/src/main/java/net/helix/pendulum/crypto/Merkle.java new file mode 100644 index 00000000..aea7ffc2 --- /dev/null +++ b/src/main/java/net/helix/pendulum/crypto/Merkle.java @@ -0,0 +1,193 @@ +package net.helix.pendulum.crypto; + +import net.helix.pendulum.controllers.RoundViewModel; +import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.model.Hash; +import net.helix.pendulum.model.HashFactory; +import org.bouncycastle.util.encoders.Hex; + +import java.io.*; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class Merkle { + + public static byte[] getMerkleRoot(SpongeFactory.Mode mode, byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { + int index = indexIn; + final Sponge sha3 = SpongeFactory.create(mode); + for (int i = 0; i < size; i++) { + sha3.reset(); + if ((index & 1) == 0) { + sha3.absorb(hash, 0, hash.length); + sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); + } else { + sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); + sha3.absorb(hash, 0, hash.length); + } + sha3.squeeze(hash, 0, hash.length); + index >>= 1; + } + if(index != 0) { + return Hash.NULL_HASH.bytes(); + } + return hash; + } + + public static List getMerklePath(List> merkleTree, int keyIndex){ + List merklePath = new ArrayList<>((merkleTree.size()-1) * 32); + for (int i = 0; i < merkleTree.size()-1; i++) { + Hash subkey = merkleTree.get(i).get(keyIndex ^ 1); + merklePath.add(subkey == null ? Hash.NULL_HASH : subkey); + keyIndex /= 2; + } + return merklePath; + } + + public static List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security){ + List keys = new ArrayList<>(1 << pubkeyDepth); + for (int i = 0; i < pubkeyCount; i++) { + int idx = firstIndex + i; + keys.add(HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security))); + } + return buildMerkleTree(keys); + } + + + public static List> buildMerkleTree(List leaves){ + if (leaves.isEmpty()) { + leaves.add(Hash.NULL_HASH); + } + byte[] buffer; + Sponge sha3 = SpongeFactory.create(SpongeFactory.Mode.S256); + int depth = (int) Math.ceil(Math.sqrt(leaves.size())); + List> merkleTree = new ArrayList<>(depth + 1); + merkleTree.add(0, leaves); + int row = 1; + // hash two following keys together until only one is left -> merkle tree + while (leaves.size() > 1) { + // Take two following keys (i=0: (k0,k1), i=1: (k2,k3), ...) and get one crypto of them + List nextKeys = Arrays.asList(new Hash[(leaves.size() / 2)]); + for (int i = 0; i < nextKeys.size(); i++) { + if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { + // leave the combined key null as well + continue; + } + sha3.reset(); + Hash k1 = leaves.get(i * 2); + Hash k2 = leaves.get(i * 2 + 1); + buffer = Arrays.copyOfRange(k1 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k1.bytes(), 0, 32); + sha3.absorb(buffer, 0, buffer.length); + buffer = Arrays.copyOfRange(k2 == null ? Hex.decode("0000000000000000000000000000000000000000000000000000000000000000") : k2.bytes(), 0, 32); + sha3.absorb(buffer, 0, buffer.length); + sha3.squeeze(buffer, 0, buffer.length); + nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); + } + leaves = nextKeys; + merkleTree.add(row++, leaves); + } + return merkleTree; + } + + public static boolean validateMerkleSignature(List bundleTransactionViewModels, SpongeFactory.Mode mode, Hash validationAddress, int securityLevel, int depth) { + + final TransactionViewModel merkleTx = bundleTransactionViewModels.get(securityLevel); + int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex + + //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) + //TODO: check if its okay here to use bundle hash instead of tx hash + byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); + + //validate leaf signature + ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * securityLevel); + + for (int i = 0; i < securityLevel; i++) { + byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i+1)); + byte[] digest = Winternitz.digest(mode, bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); + bb.put(digest); + } + + byte[] digests = bb.array(); + byte[] address = Winternitz.address(mode, digests); + + //validate Merkle path + byte[] merkleRoot = Merkle.getMerkleRoot(mode, address, + merkleTx.getSignature(), 0, keyIndex, depth); + return HashFactory.ADDRESS.create(merkleRoot).equals(validationAddress); + } + + public static List> readKeyfile(File keyfile) throws IOException { + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + int depth = Integer.parseInt(fields[0]); + List> result = new ArrayList<>(depth + 1); + for (int i = 0; i <= depth; i++) { + fields = br.readLine().split(" "); + int leadingNulls = Integer.parseInt(fields[0]); + List row = new ArrayList<>(); + for (int j = 0; j < leadingNulls; j++) { + row.add(Hash.NULL_HASH); + } + for (int j = 0; j < fields[1].length() / 64; j++) { + row.add(HashFactory.TRANSACTION.create(fields[1].substring(j * 64, (j+1) * 64))); + } + result.add(row); + } + return result; + } + } + + public static String getSeed(File keyfile) throws IOException { + StringBuilder seedBuilder = new StringBuilder(); + try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { + String[] fields = br.readLine().split(" "); + seedBuilder.append(fields[1]); + } + return seedBuilder.toString(); + } + + + public static void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { + // fill buffer + try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { + // write pubkey depth and seed into buffer + bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); + bw.newLine(); + writeKeys(bw, merkleTree.get(0)); + for (int i = 1; i < merkleTree.size(); i++) { + writeKeys(bw, merkleTree.get(i)); + } + } + } + + private static void writeKeys(BufferedWriter bw, List keys) throws IOException { + int leadingNulls = 0; + while (keys.get(leadingNulls) == null) { + leadingNulls++; + } + bw.write(leadingNulls + " "); + for (int i = leadingNulls; i < keys.size(); i++) { + if (keys.get(i) == null) { + break; + } + bw.write(keys.get(i).toString()); + } + bw.newLine(); + } + + public static byte[] padding(byte[] input, int length){ + if (input.length < length) { + byte[] output = new byte[length]; + System.arraycopy(input, 0, output, length - input.length, input.length); + return output; + } else { + if (input.length > length) { + return Arrays.copyOfRange(input, 0, length); + } else { + return Arrays.copyOfRange(input, 0, input.length); + } + + } + } +} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleFactory.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleFactory.java deleted file mode 100644 index c7c732a0..00000000 --- a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleFactory.java +++ /dev/null @@ -1,29 +0,0 @@ -package net.helix.pendulum.crypto.merkle; - -import net.helix.pendulum.crypto.merkle.impl.MerkleTreeImpl; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.reflect.Constructor; - -public enum MerkleFactory { - MerkleTree(MerkleTreeImpl.class); - - private static final Logger log = LoggerFactory.getLogger(MerkleFactory.class); - - Class clazz; - - MerkleFactory(Class clazz) { - this.clazz = clazz; - } - - public static MerkleTree create(MerkleFactory type, MerkleOptions options) { - try { - Constructor constructor = type.clazz.getConstructor(MerkleOptions.class); - return (MerkleTree) constructor.newInstance(options); - } catch (Exception e) { - log.error("Could not instantiate merkle tree object! ", e); - } - return null; - } -} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java deleted file mode 100644 index f5d0fe3a..00000000 --- a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleOptions.java +++ /dev/null @@ -1,72 +0,0 @@ -package net.helix.pendulum.crypto.merkle; - -import net.helix.pendulum.crypto.SpongeFactory; -import net.helix.pendulum.model.Hash; - -public class MerkleOptions { - - private SpongeFactory.Mode mode; - private int securityLevel; - private int depth; - private Hash address; - - public MerkleOptions(SpongeFactory.Mode mode) { - this.mode = mode; - } - - /** - * Merkle option parameters - * - * @param mode - * @param securityLevel - * @param depth - */ - public MerkleOptions(SpongeFactory.Mode mode, int securityLevel, int depth) { - this.mode = mode; - this.securityLevel = securityLevel; - this.depth = depth; - } - - public MerkleOptions(SpongeFactory.Mode mode, int securityLevel, int depth, Hash address) { - this.mode = mode; - this.securityLevel = securityLevel; - this.depth = depth; - this.address = address; - } - - public static MerkleOptions getDefault() { - return new MerkleOptions(SpongeFactory.Mode.S256, 2, 0, Hash.NULL_HASH); - } - - public SpongeFactory.Mode getMode() { - return mode; - } - - public void setMode(SpongeFactory.Mode mode) { - this.mode = mode; - } - - public int getSecurityLevel() { - return securityLevel; - } - - public void setSecurityLevel(int securityLevel) { - this.securityLevel = securityLevel; - } - - public int getDepth() { - return depth; - } - - public void setDepth(int depth) { - this.depth = depth; - } - - public Hash getAddress() { - return address; - } - - public void setAddress(Hash address) { - this.address = address; - } -} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java b/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java deleted file mode 100644 index 47505330..00000000 --- a/src/main/java/net/helix/pendulum/crypto/merkle/MerkleTree.java +++ /dev/null @@ -1,71 +0,0 @@ -package net.helix.pendulum.crypto.merkle; - -import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.model.Hash; - -import java.util.Arrays; -import java.util.List; - -public interface MerkleTree { - - /** - * Gets merkle path from the specified key index (Used in merkle transaction message - * - * @param merkleTree - * @param keyIndex - * @return - */ - List getMerklePath(List> merkleTree, int keyIndex); - - /** - * Build merkle tree as lists of lists - * - * @param leaves - * @return - */ - List> buildMerkleTree(List leaves); - - /** - * Gets merkle tree root for given leaves - * - * @param leaves - * @return - */ - byte[] getMerkleRoot(List leaves); - - /** - * Get merkle tree root, based on merkle path - * - * @param mode - * @param hash - * @param bytes merkle path bytes - * @param offset - * @param indexIn - * @param size of elements in path - * @return - */ - byte[] getMerkleRoot(byte[] hash, byte[] bytes, int offset, int indexIn, int size); - - /** - * Validate bundles which contains merkle transaction - * First it validates sender transactions signature - * - * @param bundleTransactionViewModels - * @return - */ - boolean validateMerkleSignature(List bundleTransactionViewModels); - - static byte[] padding(byte[] input, int length) { - if (input.length < length) { - byte[] output = new byte[length]; - System.arraycopy(input, 0, output, length - input.length, input.length); - return output; - } else { - if (input.length > length) { - return Arrays.copyOfRange(input, 0, length); - } else { - return Arrays.copyOfRange(input, 0, input.length); - } - } - } -} diff --git a/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java b/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java deleted file mode 100644 index eae109d6..00000000 --- a/src/main/java/net/helix/pendulum/crypto/merkle/impl/MerkleTreeImpl.java +++ /dev/null @@ -1,198 +0,0 @@ -package net.helix.pendulum.crypto.merkle.impl; - -import net.helix.pendulum.controllers.RoundViewModel; -import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.crypto.Sha3; -import net.helix.pendulum.crypto.Sponge; -import net.helix.pendulum.crypto.SpongeFactory; -import net.helix.pendulum.crypto.Winternitz; -import net.helix.pendulum.crypto.merkle.MerkleOptions; -import net.helix.pendulum.crypto.merkle.MerkleTree; -import net.helix.pendulum.model.Hash; -import net.helix.pendulum.model.HashFactory; - -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.List; - -public class MerkleTreeImpl implements MerkleTree { - - private MerkleOptions options; - - public MerkleTreeImpl(MerkleOptions options) { - this.options = options; - } - - protected List createHashs(List leaves) { - List result = new ArrayList<>(); - result.addAll(leaves); - return result; - } - - protected Hash createHash(Hash hashParent, Hash h1, Hash h2, int row, long index) { - return hashParent; - } - - @Override - public byte[] getMerkleRoot(byte[] hash, byte[] bytes, int offset, final int indexIn, int size) { - int index = indexIn; - final Sponge sha3 = SpongeFactory.create(options.getMode()); - for (int i = 0; i < size; i++) { - sha3.reset(); - if ((index & 1) == 0) { - sha3.absorb(hash, 0, hash.length); - sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); - } else { - sha3.absorb(bytes, offset + i * Sha3.HASH_LENGTH, Sha3.HASH_LENGTH); - sha3.absorb(hash, 0, hash.length); - } - sha3.squeeze(hash, 0, hash.length); - index >>= 1; - } - if (index != 0) { - return getDefaultMerkleHash().bytes(); - } - return hash; - } - - @Override - public List getMerklePath(List> merkleTree, int keyIndex) { - List merklePath = new ArrayList<>((merkleTree.size() - 1) * Hash.SIZE_IN_BYTES); - for (int i = 0; i < merkleTree.size() - 1; i++) { - Hash subkey = merkleTree.get(i).get(keyIndex ^ 1); - merklePath.add(subkey == null ? getDefaultMerkleHash() : subkey); - keyIndex /= 2; - } - return merklePath; - } - - @Override - public byte[] getMerkleRoot(List leaves) { - List> merkleTree = buildMerkleTree(leaves); - return (merkleTree.get(merkleTree.size() - 1).get(0)).bytes(); - } - - - private void sortLeaves(List leaves) { - leaves.sort(Comparator.comparing((Hash m) -> m.toString())); - } - - @Override - public List> buildMerkleTree(List leaves) { - if (leaves.isEmpty()) { - leaves.add(getDefaultMerkleHash()); - } - byte[] buffer; - addPaddingLeaves(leaves); - sortLeaves(leaves); - Sponge sha3 = SpongeFactory.create(options.getMode()); - List> merkleTree = new ArrayList<>(); - merkleTree.add(0, leaves); - int row = 1; - while (leaves.size() > 1) { - List nextKeys = Arrays.asList(new Hash[getParentNodesSize(leaves)]); - for (int i = 0; i < nextKeys.size(); i++) { - if (areLeavesNull(leaves, i)) { - continue; - } - sha3.reset(); - Hash k1 = getLeaves(leaves, i * 2); - Hash k2 = getLeaves(leaves, i * 2 + 1); - buffer = computeParentHash(sha3, k1, k2); - nextKeys.set(i, HashFactory.TRANSACTION.create(buffer)); - } - leaves = nextKeys; - merkleTree.add(row++, createHashs(leaves)); - } - return merkleTree; - } - - public boolean validateMerkleSignature(List bundleTransactionViewModels) { - - final TransactionViewModel merkleTx = bundleTransactionViewModels.get(options.getSecurityLevel()); - int keyIndex = RoundViewModel.getRoundIndex(merkleTx); // get keyindex - - //milestones sign the normalized hash of the sibling transaction. (why not bundle hash?) - //TODO: check if its okay here to use bundle hash instead of tx hash - byte[] bundleHash = Winternitz.normalizedBundle(merkleTx.getBundleHash().bytes()); - - //validate leaf signature - ByteBuffer bb = ByteBuffer.allocate(Sha3.HASH_LENGTH * options.getSecurityLevel()); - - for (int i = 0; i < options.getSecurityLevel(); i++) { - byte[] bundleHashFragment = Arrays.copyOfRange(bundleHash, Winternitz.NORMALIZED_FRAGMENT_LENGTH * i, Winternitz.NORMALIZED_FRAGMENT_LENGTH * (i + 1)); - byte[] digest = Winternitz.digest(options.getMode(), bundleHashFragment, bundleTransactionViewModels.get(i).getSignature()); - bb.put(digest); - } - - byte[] digests = bb.array(); - byte[] address = Winternitz.address(options.getMode(), digests); - - return validateMerklePath(merkleTx.getSignature(), keyIndex, address); - } - - private boolean validateMerklePath(byte[] path, int keyIndex, byte[] address) { - byte[] merkleRoot = getMerkleRoot(address, - path, 0, keyIndex, options.getDepth()); - return HashFactory.ADDRESS.create(merkleRoot).equals(options.getAddress()); - } - - protected void addPaddingLeaves(List leaves) { - int closestPow = (int) Math.pow(2.0, getClosestPow(leaves.size())); - int leaveSize = leaves.size(); - while (leaveSize < closestPow) { - leaves.add(leaveSize++, Hash.NULL_HASH); - } - } - - protected int getTreeDepth(int leavesNumber) { - return getClosestPow(leavesNumber); - } - - protected int getClosestPow(int i) { - int j = 1; - int power = 0; - while (j < i) { - j = j << 1; - power++; - } - return power; - } - - protected Hash getLeaves(List leaves, int index) { - return index < leaves.size() ? leaves.get(index) : getDefaultMerkleHash(); - } - - protected int getParentNodesSize(List leaves) { - return leaves.size() % 2 == 0 ? (leaves.size() / 2) : (leaves.size() / 2 + 1); - } - - private boolean areLeavesNull(List leaves, int i) { - if (leaves.get(i * 2) == null && leaves.get(i * 2 + 1) == null) { - return true; - } - return false; - } - - private byte[] computeParentHash(Sponge sha3, byte[] k1, byte[] k2) { - byte[] buffer = new byte[Hash.SIZE_IN_BYTES]; - sha3.absorb(k1, 0, k1.length); - sha3.absorb(k2, 0, k2.length); - sha3.squeeze(buffer, 0, buffer.length); - return buffer; - } - - private byte[] computeParentHash(Sponge sha3, Hash k1, Hash k2) { - return computeParentHash(sha3, copyHash(k1), copyHash(k2)); - } - - private byte[] copyHash(Hash k2) { - return Arrays.copyOfRange(k2 == null ? getDefaultMerkleHash().bytes() : k2.bytes(), 0, Hash.SIZE_IN_BYTES); - } - - private Hash getDefaultMerkleHash() { - return Hash.NULL_HASH; - } -} diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 1b948315..58eb06e3 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -12,8 +12,6 @@ import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.*; import net.helix.pendulum.crypto.*; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.persistables.Transaction; @@ -1642,9 +1640,6 @@ private List getConsistentTips() throws Exception { snapshotProvider.getLatestSnapshot().lockRead(); try { WalkValidatorImpl walkValidator = new WalkValidatorImpl(tangle, snapshotProvider, ledgerService, configuration); - final List transactions = new ArrayList<>(tipsViewModel.getTips()); - transactions.sort(Comparator.comparing((Hash h) -> h.toString())); - for (Hash transaction : tipsViewModel.getTips()) { TransactionViewModel txVM = TransactionViewModel.fromHash(tangle, transaction); if (txVM.getType() != TransactionViewModel.PREFILLED_SLOT && @@ -1682,7 +1677,7 @@ private List addMilestoneReferences(List confirmedTips, int roundInd txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones } //branch - List> merkleTreeTips = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(confirmedTips); + List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips } return txToApprove; diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java index 9e08990b..6f518758 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java @@ -1,14 +1,12 @@ package net.helix.pendulum.service.milestone.impl; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; +import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.service.API; import net.helix.pendulum.service.utils.RoundIndexUtil; import net.helix.pendulum.service.validatormanager.CandidateTracker; -import net.helix.pendulum.utils.KeyfileUtil; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -75,9 +73,8 @@ private void initSeed(PendulumConfig configuration) { } private void writeKeyIndex() throws IOException { - KeyfileUtil keyfileUtil = new KeyfileUtil(); - List> merkleTree = keyfileUtil.readKeyfile(new File(keyfile)); - keyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); + List> merkleTree = Merkle.readKeyfile(new File(keyfile)); + Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, currentKeyIndex, keyfileIndex, keyfile); } private void readKeyfileMetadata() throws IOException { @@ -90,8 +87,7 @@ private void readKeyfileMetadata() throws IOException { seed = fields[1]; } } - KeyfileUtil keyfileUtil = new KeyfileUtil(); - List> merkleTree = keyfileUtil.readKeyfile(new File(keyfile)); + List> merkleTree = Merkle.readKeyfile(new File(keyfile)); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size() - 1).get(0).bytes()); } @@ -110,9 +106,7 @@ private void doKeyChange() throws Exception { // generate new keyfile int newKeyfileIndex = keyfileIndex + 1; log.debug("Generating Keyfile (idx: " + newKeyfileIndex + ")"); - - KeyfileUtil keyfileUtil = new KeyfileUtil(); - List> merkleTree = keyfileUtil.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getValidatorSecurity()); + List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * newKeyfileIndex, maxKeyIndex, config.getValidatorSecurity()); Hash newAddress = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); // send keyChange bundle to register new address api.publishKeyChange(address.toString(), newAddress, mwm, sign, currentKeyIndex, maxKeyIndex); @@ -120,14 +114,13 @@ private void doKeyChange() throws Exception { keyfileIndex = newKeyfileIndex; address = newAddress; currentKeyIndex = maxKeyIndex * keyfileIndex; - keyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); } private void generateKeyfile(String seed) throws Exception { log.debug("Generating Keyfile (idx: " + keyfileIndex + ")"); - KeyfileUtil keyfileUtil = new KeyfileUtil(); - List> merkleTree = keyfileUtil.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getValidatorSecurity()); - keyfileUtil.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); + List> merkleTree = Merkle.buildMerkleKeyTree(seed, pubkeyDepth, maxKeyIndex * keyfileIndex, maxKeyIndex, config.getValidatorSecurity()); + Merkle.createKeyfile(merkleTree, Hex.decode(seed), pubkeyDepth, 0, keyfileIndex, keyfile); address = HashFactory.ADDRESS.create(merkleTree.get(merkleTree.size()-1).get(0).bytes()); } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java index 86250137..cae1f9a4 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java @@ -6,9 +6,8 @@ import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.controllers.RoundViewModel; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.IntegerIndex; import net.helix.pendulum.model.StateDiff; @@ -184,9 +183,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM if (isMilestoneBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - MerkleOptions options = new MerkleOptions(mode, securityLevel,config.getMilestoneKeyDepth(), senderAddress); - - boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); log.trace("valid signature: {}", validSignature); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { diff --git a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java index 9c9953df..36780497 100644 --- a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java @@ -3,9 +3,8 @@ import net.helix.pendulum.BundleValidator; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; @@ -59,9 +58,7 @@ public ValidatorValidity validateValidators(TransactionViewModel transactionView // validate signature Hash senderAddress = tail.getAddressHash(); - MerkleOptions options = new MerkleOptions(mode, securityLevel,config.getMilestoneKeyDepth(), senderAddress); - - boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getValidatorManagerKeyDepth()); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getValidatorManagerAddress().equals(senderAddress) && validSignature)) { return VALID; diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java index f1d982a7..ca25e285 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java @@ -3,9 +3,8 @@ import net.helix.pendulum.BundleValidator; import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.SpongeFactory; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.snapshot.SnapshotService; @@ -75,8 +74,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM if (tail.getHash().equals(transactionViewModel.getHash()) && isCandidateBundleStructureValid(bundleTransactionViewModels, securityLevel)) { Hash senderAddress = tail.getAddressHash(); - MerkleOptions options = new MerkleOptions(mode, securityLevel,config.getMilestoneKeyDepth(), senderAddress); - boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); + boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validator.contains(senderAddress)) && validSignature) { return VALID; diff --git a/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java b/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java deleted file mode 100644 index 8b53b18a..00000000 --- a/src/main/java/net/helix/pendulum/utils/KeyfileUtil.java +++ /dev/null @@ -1,93 +0,0 @@ -package net.helix.pendulum.utils; - -import net.helix.pendulum.crypto.Winternitz; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; -import net.helix.pendulum.model.Hash; -import net.helix.pendulum.model.HashFactory; -import org.bouncycastle.util.encoders.Hex; - -import java.io.*; -import java.util.ArrayList; -import java.util.List; - -/** - * It contains the required methods to manage key files (read, write, build) - */ -public class KeyfileUtil { - - private static int ADDRESS_STRING_LENGTH = Hash.SIZE_IN_BYTES * 2; - - public String getSeed(File keyfile) throws IOException { - StringBuilder seedBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { - String[] fields = br.readLine().split(" "); - seedBuilder.append(fields[1]); - } - return seedBuilder.toString(); - } - - public List> buildMerkleKeyTree(String seed, int pubkeyDepth, int firstIndex, int pubkeyCount, int security) { - List keys = new ArrayList<>(1 << pubkeyDepth); - for (int i = 0; i < pubkeyCount; i++) { - int idx = firstIndex + i; - keys.add(createKeyHash(seed, security, idx)); - } - return MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).buildMerkleTree(keys); - } - - public void writeKeys(BufferedWriter bw, List keys) throws IOException { - int leadingNulls = 0; - while (keys.get(leadingNulls) == null) { - leadingNulls++; - } - bw.write(leadingNulls + " "); - for (int i = leadingNulls; i < keys.size(); i++) { - if (keys.get(i) == null) { - break; - } - bw.write(writeMerkleNode(keys.get(i))); - } - bw.newLine(); - } - - public void createKeyfile(List> merkleTree, byte[] seed, int pubkeyDepth, int keyIndex, int keyfileIndex, String filename) throws IOException { - try (BufferedWriter bw = new BufferedWriter(new FileWriter(filename))) { - bw.write(pubkeyDepth + " " + Hex.toHexString(seed) + " " + keyfileIndex + " " + keyIndex); - bw.newLine(); - writeKeys(bw, merkleTree.get(0)); - for (int i = 1; i < merkleTree.size(); i++) { - writeKeys(bw, merkleTree.get(i)); - } - } - } - - public List> readKeyfile(File keyfile) throws IOException { - try (BufferedReader br = new BufferedReader(new FileReader(keyfile))) { - String[] fields = br.readLine().split(" "); - int depth = Integer.parseInt(fields[0]); - List> result = new ArrayList<>(depth + 1); - for (int i = 0; i <= depth; i++) { - fields = br.readLine().split(" "); - int leadingNulls = Integer.parseInt(fields[0]); - List row = new ArrayList<>(); - for (int j = 0; j < leadingNulls; j++) { - row.add(Hash.NULL_HASH); - } - for (int j = 0; j < fields[1].length() / ADDRESS_STRING_LENGTH; j++) { - row.add(HashFactory.ADDRESS.create(fields[1].substring(j * ADDRESS_STRING_LENGTH, (j + 1) * ADDRESS_STRING_LENGTH))); - } - result.add(row); - } - return result; - } - } - - private Hash createKeyHash(String seed, int security, int idx) { - return HashFactory.ADDRESS.create(Winternitz.generateAddress(Hex.decode(seed), idx, security)); - } - - private String writeMerkleNode(Hash key) { - return key.toString(); - } -} diff --git a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java index bc21c6a5..df8134fa 100644 --- a/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java +++ b/src/main/java/net/helix/pendulum/utils/bundle/BundleUtils.java @@ -1,13 +1,11 @@ package net.helix.pendulum.utils.bundle; import net.helix.pendulum.controllers.TransactionViewModel; +import net.helix.pendulum.crypto.Merkle; import net.helix.pendulum.crypto.Sponge; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; -import net.helix.pendulum.crypto.merkle.MerkleFactory; -import net.helix.pendulum.crypto.merkle.MerkleOptions; import net.helix.pendulum.model.Hash; -import net.helix.pendulum.utils.KeyfileUtil; import net.helix.pendulum.utils.Serializer; import org.bouncycastle.util.encoders.Hex; import org.slf4j.Logger; @@ -161,13 +159,12 @@ private byte[] addBundleHash(List bundle, SpongeFactory.Mode mode) { private void signBundle(String filepath, byte[] merkleTransaction, List senderTransactions, byte[] bundleHash, int keyIndex, int maxKeyIndex) throws IOException { // Get merkle path and store in signatureMessageFragment of Sibling Transaction File keyfile = new File(filepath); - KeyfileUtil keyfileUtil = new KeyfileUtil(); - List> merkleTree = keyfileUtil.readKeyfile(keyfile); - String seed = keyfileUtil.getSeed(keyfile); + List> merkleTree = Merkle.readKeyfile(keyfile); + String seed = Merkle.getSeed(keyfile); int security = senderTransactions.size(); // create merkle path from keyfile - List merklePath = MerkleFactory.create(MerkleFactory.MerkleTree, MerkleOptions.getDefault()).getMerklePath(merkleTree, keyIndex % maxKeyIndex); + List merklePath = Merkle.getMerklePath(merkleTree, keyIndex % maxKeyIndex); byte[] path = Hex.decode(merklePath.stream().map(Hash::toString).collect(Collectors.joining())); System.arraycopy(path, 0, merkleTransaction, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_OFFSET, path.length); From 33ea4a66a1e669f7c148e14e4a58777840f44b2d Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Wed, 27 Nov 2019 16:42:27 +0100 Subject: [PATCH 18/36] Refactor double negative in a config variable name, and rename variables to indicate their explicit purpose --- .gitignore | 1 + .../net/helix/pendulum/conf/APIConfig.java | 14 +- .../pendulum/conf/BasePendulumConfig.java | 145 +++++++++-------- .../helix/pendulum/conf/MilestoneConfig.java | 6 +- .../helix/pendulum/conf/TestnetConfig.java | 14 +- .../java/net/helix/pendulum/service/API.java | 2 +- .../net/helix/pendulum/service/Feature.java | 2 +- .../milestone/impl/MilestonePublisher.java | 2 +- .../milestone/impl/MilestoneServiceImpl.java | 2 +- .../validator/impl/ValidatorServiceImpl.java | 2 +- .../impl/ValidatorManagerServiceImpl.java | 2 +- .../net/helix/pendulum/conf/ConfigTest.java | 151 ++++++++++-------- .../helix/pendulum/conf/ZMQConfigTest.java | 16 +- 13 files changed, 190 insertions(+), 169 deletions(-) diff --git a/.gitignore b/.gitignore index c689f2cb..45a18de7 100644 --- a/.gitignore +++ b/.gitignore @@ -67,3 +67,4 @@ Validator.* # os .DS_Store +Makefile \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/APIConfig.java b/src/main/java/net/helix/pendulum/conf/APIConfig.java index 597987f4..3fb89ff2 100644 --- a/src/main/java/net/helix/pendulum/conf/APIConfig.java +++ b/src/main/java/net/helix/pendulum/conf/APIConfig.java @@ -18,14 +18,14 @@ public interface APIConfig extends Config { String getApiHost(); /** - * @return {@value Descriptions#REMOTE_LIMIT_API} + * @return {@value Descriptions#IGNORED_API_ENDPOINTS} */ - List getRemoteLimitApi(); + List getIgnoredApiEndpoints(); /** - * @return {@value Descriptions#REMOTE_TRUSTED_API_HOSTS} + * @return {@value Descriptions#ALLOWED_API_HOSTS} */ - List getRemoteTrustedApiHosts(); + List getAllowedApiHosts(); /** * @return {@value Descriptions#MAX_FIND_TRANSACTIONS} @@ -60,14 +60,14 @@ public interface APIConfig extends Config { interface Descriptions { String API_PORT = "The port that will be used by the API."; String API_HOST = "The host on which the API will listen to. Set to 0.0.0.0 to accept any host."; - String REMOTE_LIMIT_API = "Commands that should be ignored by API."; - String REMOTE_TRUSTED_API_HOSTS = "Open the API interface to defined hosts. You can specify multiple hosts in a comma separated list \"--remote-trusted-api-hosts 192.168.0.55,10.0.0.10\". You must also provide the \"--remote\" parameter. Warning: \"--remote-limit-api\" will have no effect for these hosts."; + String REMOTE = "Open the API interface to any host. Equivalent to \"--api-host 0.0.0.0\""; + String IGNORED_API_ENDPOINTS = "Commands that should be ignored by API. Note that by default localhost and any allowed api host have access to all endpoints."; + String ALLOWED_API_HOSTS = "Open the API interface to defined hosts. You can specify multiple hosts in a comma separated list \"--allowed_api_hosts 192.168.0.55,10.0.0.10\". You must also provide the \"--remote\" parameter. Warning: \"--ignored_api_endpoints\" will have no effect for these hosts."; String REMOTE_AUTH = "A string in the form of :. Used to access the API"; String MAX_FIND_TRANSACTIONS = "The maximal number of transactions that may be returned by the \"findTransactions\" API call. If the number of transactions found exceeds this number an error will be returned."; String MAX_REQUESTS_LIST = "The maximal number of parameters one can place in an API call. If the number parameters exceeds this number an error will be returned"; String MAX_GET_TRANSACTION_STRINGS = "The maximal number of transaction strings that may be returned by the \"getTransactionStrings\" API call. If the number of transactions found exceeds this number an error will be returned."; String MAX_BODY_LENGTH = "The maximal number of characters the body of an API call may hold. If a request body length exceeds this number an error will be returned."; - String REMOTE = "Open the API interface to any host. Equivalent to \"--api-host 0.0.0.0\""; String RESOURCE_PATH = "Resource path"; } } \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java index b6763176..1d637ff7 100644 --- a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java +++ b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java @@ -32,8 +32,8 @@ public abstract class BasePendulumConfig implements PendulumConfig { //API protected int apiPort = Defaults.API_PORT; protected String apiHost = Defaults.API_HOST; - protected List remoteLimitApi = Defaults.REMOTE_LIMIT_API; - protected List remoteTrustedApiHosts = Defaults.REMOTE_LIMIT_API_HOSTS; + protected List ignoredApiEndpoints = Defaults.IGNORED_API_ENDPOINTS; + protected List allowedApiHosts = Defaults.ALLOWED_API_HOSTS; protected int maxFindTransactions = Defaults.MAX_FIND_TRANSACTIONS; protected int maxRequestsList = Defaults.MAX_REQUESTS_LIST; protected int maxGetTransactionStrings = Defaults.MAX_GET_TRANSACTION_STRINGS; @@ -167,7 +167,7 @@ public int getApiPort() { } @JsonProperty - @Parameter(names = {"--api-port", "-p"}, description = APIConfig.Descriptions.API_PORT) + @Parameter(names = {"--api_port", "-p"}, description = APIConfig.Descriptions.API_PORT) public void setPort(int apiPort) { this.apiPort = apiPort; } @@ -178,7 +178,7 @@ public String getApiHost() { } @JsonProperty - @Parameter(names = {"--api-host"}, description = APIConfig.Descriptions.API_HOST) + @Parameter(names = {"--api_host"}, description = APIConfig.Descriptions.API_HOST) protected void setApiHost(String apiHost) { this.apiHost = apiHost; } @@ -190,38 +190,37 @@ protected void setRemote(boolean remote) { } @Override - public List getRemoteLimitApi() { - return remoteLimitApi; + public List getIgnoredApiEndpoints() { + return ignoredApiEndpoints; } @JsonProperty - @Parameter(names = {"--remote-limit-api"}, description = APIConfig.Descriptions.REMOTE_LIMIT_API) - protected void setRemoteLimitApi(String remoteLimitApi) { - this.remoteLimitApi = PendulumUtils.splitStringToImmutableList(remoteLimitApi, SPLIT_STRING_TO_LIST_REGEX); + @Parameter(names = {"--ignored_api_endpoints"}, description = APIConfig.Descriptions.IGNORED_API_ENDPOINTS) + protected void setIgnoredApiEndpoints(String ignoredApiEndpoints) { + this.ignoredApiEndpoints = PendulumUtils.splitStringToImmutableList(ignoredApiEndpoints, SPLIT_STRING_TO_LIST_REGEX); } @Override - public List getRemoteTrustedApiHosts() { - return remoteTrustedApiHosts; + public List getAllowedApiHosts() { + return allowedApiHosts; } @JsonProperty - @Parameter(names = {"--remote-trusted-api-hosts"}, description = APIConfig.Descriptions.REMOTE_TRUSTED_API_HOSTS) - public void setRemoteTrustedApiHosts(String remoteTrustedApiHosts) { - List addresses = PendulumUtils.splitStringToImmutableList(remoteTrustedApiHosts, SPLIT_STRING_TO_LIST_REGEX); + @Parameter(names = {"--allowed_api_hosts"}, description = APIConfig.Descriptions.ALLOWED_API_HOSTS) + public void setAllowedApiHosts(String allowedApiHosts) { + List addresses = PendulumUtils.splitStringToImmutableList(allowedApiHosts, SPLIT_STRING_TO_LIST_REGEX); List inetAddresses = addresses.stream().map(host -> { try { return InetAddress.getByName(host.trim()); } catch (UnknownHostException e) { - throw new ParameterException("Invalid value for --remote-trusted-api-hosts address: ", e); + throw new ParameterException("Invalid value for --remote_trusted_api_hosts address: ", e); } }).collect(Collectors.toList()); - // always make sure that localhost exists as trusted host - if (!inetAddresses.contains(Defaults.REMOTE_LIMIT_API_DEFAULT_HOST)) { - inetAddresses.add(Defaults.REMOTE_LIMIT_API_DEFAULT_HOST); + if (!inetAddresses.contains(Defaults.ALLOWED_API_DEFAULT_HOST)) { + inetAddresses.add(Defaults.ALLOWED_API_DEFAULT_HOST); } - this.remoteTrustedApiHosts = Collections.unmodifiableList(inetAddresses); + this.allowedApiHosts = Collections.unmodifiableList(inetAddresses); } @Override @@ -230,7 +229,7 @@ public int getMaxFindTransactions() { } @JsonProperty - @Parameter(names = {"--max-find-transactions"}, description = APIConfig.Descriptions.MAX_FIND_TRANSACTIONS) + @Parameter(names = {"--max_find_transactions"}, description = APIConfig.Descriptions.MAX_FIND_TRANSACTIONS) protected void setMaxFindTransactions(int maxFindTransactions) { this.maxFindTransactions = maxFindTransactions; } @@ -241,7 +240,7 @@ public int getMaxRequestsList() { } @JsonProperty - @Parameter(names = {"--max-requests-list"}, description = APIConfig.Descriptions.MAX_REQUESTS_LIST) + @Parameter(names = {"--max_requests_list"}, description = APIConfig.Descriptions.MAX_REQUESTS_LIST) protected void setMaxRequestsList(int maxRequestsList) { this.maxRequestsList = maxRequestsList; } @@ -252,7 +251,7 @@ public int getMaxTransactionStrings() { } @JsonProperty - @Parameter(names = {"--max-get-transaction-strings"}, description = APIConfig.Descriptions.MAX_GET_TRANSACTION_STRINGS) + @Parameter(names = {"--max_get_transaction_strings"}, description = APIConfig.Descriptions.MAX_GET_TRANSACTION_STRINGS) protected void setMaxGetTransactionStrings(int maxGetTransactionStrings) { this.maxGetTransactionStrings = maxGetTransactionStrings; } @@ -263,7 +262,7 @@ public int getMaxBodyLength() { } @JsonProperty - @Parameter(names = {"--max-body-length"}, description = APIConfig.Descriptions.MAX_BODY_LENGTH) + @Parameter(names = {"--max_body_length"}, description = APIConfig.Descriptions.MAX_BODY_LENGTH) protected void setMaxBodyLength(int maxBodyLength) { this.maxBodyLength = maxBodyLength; } @@ -274,7 +273,7 @@ public String getRemoteAuth() { } @JsonProperty - @Parameter(names = {"--remote-auth"}, description = APIConfig.Descriptions.REMOTE_AUTH) + @Parameter(names = {"--remote_auth"}, description = APIConfig.Descriptions.REMOTE_AUTH) protected void setRemoteAuth(String remoteAuth) { this.remoteAuth = remoteAuth; } @@ -285,7 +284,7 @@ public int getUdpReceiverPort() { } @JsonProperty - @Parameter(names = {"-u", "--udp-receiver-port"}, description = NetworkConfig.Descriptions.UDP_RECEIVER_PORT) + @Parameter(names = {"-u", "--udp_receiver_port"}, description = NetworkConfig.Descriptions.UDP_RECEIVER_PORT) public void setUdpReceiverPort(int udpReceiverPort) { this.udpReceiverPort = udpReceiverPort; } @@ -296,7 +295,7 @@ public int getTcpReceiverPort() { } @JsonProperty - @Parameter(names = {"-t", "--tcp-receiver-port"}, description = NetworkConfig.Descriptions.TCP_RECEIVER_PORT) + @Parameter(names = {"-t", "--tcp_receiver_port"}, description = NetworkConfig.Descriptions.TCP_RECEIVER_PORT) protected void setTcpReceiverPort(int tcpReceiverPort) { this.tcpReceiverPort = tcpReceiverPort; } @@ -307,7 +306,7 @@ public double getpRemoveRequest() { } @JsonProperty - @Parameter(names = {"--p-remove-request"}, description = NetworkConfig.Descriptions.P_REMOVE_REQUEST) + @Parameter(names = {"--p_remove_request"}, description = NetworkConfig.Descriptions.P_REMOVE_REQUEST) protected void setpRemoveRequest(double pRemoveRequest) { this.pRemoveRequest = pRemoveRequest; } @@ -318,7 +317,7 @@ public int getSendLimit() { } @JsonProperty - @Parameter(names = {"--send-limit"}, description = NetworkConfig.Descriptions.SEND_LIMIT) + @Parameter(names = {"--send_limit"}, description = NetworkConfig.Descriptions.SEND_LIMIT) protected void setSendLimit(int sendLimit) { this.sendLimit = sendLimit; } @@ -329,7 +328,7 @@ public int getMaxPeers() { } @JsonProperty - @Parameter(names = {"--max-peers"}, description = NetworkConfig.Descriptions.MAX_PEERS) + @Parameter(names = {"--max_peers"}, description = NetworkConfig.Descriptions.MAX_PEERS) protected void setMaxPeers(int maxPeers) { this.maxPeers = maxPeers; } @@ -340,7 +339,7 @@ public boolean isDnsRefresherEnabled() { } @JsonProperty - @Parameter(names = {"--dns-refresher"}, description = NetworkConfig.Descriptions.DNS_REFRESHER_ENABLED, arity = 1) + @Parameter(names = {"--dns_refresher"}, description = NetworkConfig.Descriptions.DNS_REFRESHER_ENABLED, arity = 1) protected void setDnsRefresherEnabled(boolean dnsRefresherEnabled) { this.dnsRefresherEnabled = dnsRefresherEnabled; } @@ -351,7 +350,7 @@ public boolean isDnsResolutionEnabled() { } @JsonProperty - @Parameter(names = {"--dns-resolution"}, description = NetworkConfig.Descriptions.DNS_RESOLUTION_ENABLED, arity = 1) + @Parameter(names = {"--dns_resolution"}, description = NetworkConfig.Descriptions.DNS_RESOLUTION_ENABLED, arity = 1) protected void setDnsResolutionEnabled(boolean dnsResolutionEnabled) { this.dnsResolutionEnabled = dnsResolutionEnabled; } @@ -373,7 +372,7 @@ public String getXiDir() { } @JsonProperty - @Parameter(names = {"--XI-dir"}, description = XIConfig.Descriptions.XI_DIR) + @Parameter(names = {"--XI_dir"}, description = XIConfig.Descriptions.XI_DIR) protected void setXiDir(String xiDir) { this.xiDir = xiDir; } @@ -384,7 +383,7 @@ public String getDbPath() { } @JsonProperty - @Parameter(names = {"--db-path"}, description = DbConfig.Descriptions.DB_PATH) + @Parameter(names = {"--db_path"}, description = DbConfig.Descriptions.DB_PATH) protected void setDbPath(String dbPath) { this.dbPath = dbPath; } @@ -395,7 +394,7 @@ public String getDbLogPath() { } @JsonProperty - @Parameter(names = {"--db-log-path"}, description = DbConfig.Descriptions.DB_LOG_PATH) + @Parameter(names = {"--db_log_path"}, description = DbConfig.Descriptions.DB_LOG_PATH) protected void setDbLogPath(String dbLogPath) { this.dbLogPath = dbLogPath; } @@ -406,7 +405,7 @@ public int getDbCacheSize() { } @JsonProperty - @Parameter(names = {"--db-cache-size"}, description = DbConfig.Descriptions.DB_CACHE_SIZE) + @Parameter(names = {"--db_cache_size"}, description = DbConfig.Descriptions.DB_CACHE_SIZE) protected void setDbCacheSize(int dbCacheSize) { this.dbCacheSize = dbCacheSize; } @@ -465,7 +464,7 @@ public double getpReplyRandomTip() { } @JsonProperty - @Parameter(names = {"--p-reply-random"}, description = ProtocolConfig.Descriptions.P_REPLY_RANDOM_TIP) + @Parameter(names = {"--p_reply_random"}, description = ProtocolConfig.Descriptions.P_REPLY_RANDOM_TIP) protected void setpReplyRandomTip(double pReplyRandomTip) { this.pReplyRandomTip = pReplyRandomTip; } @@ -476,7 +475,7 @@ public double getpDropTransaction() { } @JsonProperty - @Parameter(names = {"--p-drop-transaction"}, description = ProtocolConfig.Descriptions.P_DROP_TRANSACTION) + @Parameter(names = {"--p_drop_transaction"}, description = ProtocolConfig.Descriptions.P_DROP_TRANSACTION) protected void setpDropTransaction(double pDropTransaction) { this.pDropTransaction = pDropTransaction; } @@ -487,7 +486,7 @@ public double getpSelectMilestoneChild() { } @JsonProperty - @Parameter(names = {"--p-select-milestone"}, description = ProtocolConfig.Descriptions.P_SELECT_MILESTONE) + @Parameter(names = {"--p_select_milestone"}, description = ProtocolConfig.Descriptions.P_SELECT_MILESTONE) protected void setpSelectMilestoneChild(double pSelectMilestoneChild) { this.pSelectMilestoneChild = pSelectMilestoneChild; } @@ -498,7 +497,7 @@ public double getpSendMilestone() { } @JsonProperty - @Parameter(names = {"--p-send-milestone"}, description = ProtocolConfig.Descriptions.P_SEND_MILESTONE) + @Parameter(names = {"--p_send_milestone"}, description = ProtocolConfig.Descriptions.P_SEND_MILESTONE) protected void setpSendMilestone(double pSendMilestone) { this.pSendMilestone = pSendMilestone; } @@ -509,7 +508,7 @@ public double getpPropagateRequest() { } @JsonProperty - @Parameter(names = {"--p-propagate-request"}, description = ProtocolConfig.Descriptions.P_PROPAGATE_REQUEST) + @Parameter(names = {"--p_propagate_request"}, description = ProtocolConfig.Descriptions.P_PROPAGATE_REQUEST) protected void setpPropagateRequest(double pPropagateRequest) { this.pPropagateRequest = pPropagateRequest; } @@ -520,7 +519,7 @@ public boolean getLocalSnapshotsEnabled() { } @JsonProperty - @Parameter(names = {"--local-snapshots-enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_ENABLED) + @Parameter(names = {"--local_snapshots_enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_ENABLED) protected void setLocalSnapshotsEnabled(boolean localSnapshotsEnabled) { this.localSnapshotsEnabled = localSnapshotsEnabled; } @@ -531,7 +530,7 @@ public boolean getLocalSnapshotsPruningEnabled() { } @JsonProperty - @Parameter(names = {"--local-snapshots-pruning-enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_ENABLED) + @Parameter(names = {"--local_snapshots_pruning_enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_ENABLED) protected void setLocalSnapshotsPruningEnabled(boolean localSnapshotsPruningEnabled) { this.localSnapshotsPruningEnabled = localSnapshotsPruningEnabled; } @@ -542,7 +541,7 @@ public int getLocalSnapshotsPruningDelay() { } @JsonProperty - @Parameter(names = {"--local-snapshots-pruning-delay"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_DELAY) + @Parameter(names = {"--local_snapshots_pruning_delay"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_DELAY) protected void setLocalSnapshotsPruningDelay(int localSnapshotsPruningDelay) { this.localSnapshotsPruningDelay = localSnapshotsPruningDelay; } @@ -553,7 +552,7 @@ public int getLocalSnapshotsIntervalSynced() { } @JsonProperty - @Parameter(names = {"--local-snapshots-interval-synced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_SYNCED) + @Parameter(names = {"--local_snapshots_interval_synced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_SYNCED) protected void setLocalSnapshotsIntervalSynced(int localSnapshotsIntervalSynced) { this.localSnapshotsIntervalSynced = localSnapshotsIntervalSynced; } @@ -564,7 +563,7 @@ public int getLocalSnapshotsIntervalUnsynced() { } @JsonProperty - @Parameter(names = {"--local-snapshots-interval-unsynced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED) + @Parameter(names = {"--local_snapshots_interval_unsynced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED) protected void setLocalSnapshotsIntervalUnsynced(int localSnapshotsIntervalUnsynced) { this.localSnapshotsIntervalUnsynced = localSnapshotsIntervalUnsynced; } @@ -575,7 +574,7 @@ public int getLocalSnapshotsDepth() { } @JsonProperty - @Parameter(names = {"--local-snapshots-depth"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_DEPTH) + @Parameter(names = {"--local_snapshots_depth"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_DEPTH) protected void setLocalSnapshotsDepth(int localSnapshotsDepth) { this.localSnapshotsDepth = localSnapshotsDepth; } @@ -586,7 +585,7 @@ public String getLocalSnapshotsBasePath() { } @JsonProperty - @Parameter(names = {"--local-snapshots-base-path"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_BASE_PATH) + @Parameter(names = {"--local_snapshots_base_path"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_BASE_PATH) protected void setLocalSnapshotsBasePath(String localSnapshotsBasePath) { this.localSnapshotsBasePath = localSnapshotsBasePath; } @@ -644,7 +643,7 @@ public boolean isZmqEnabled() { */ @Deprecated @JsonProperty - @Parameter(names = "--zmq-enabled", description = ZMQConfig.Descriptions.ZMQ_ENABLED, arity = 1) + @Parameter(names = "--zmq_enabled", description = ZMQConfig.Descriptions.ZMQ_ENABLED, arity = 1) protected void setZmqEnabled(boolean zmqEnabled) { this.zmqEnableTcp = zmqEnabled; this.zmqEnableIpc = zmqEnabled; @@ -656,7 +655,7 @@ public boolean isZmqEnableTcp() { } @JsonProperty - @Parameter(names = "--zmq-enable-tcp", description = ZMQConfig.Descriptions.ZMQ_ENABLE_TCP, arity = 1) + @Parameter(names = "--zmq_enable_tcp", description = ZMQConfig.Descriptions.ZMQ_ENABLE_TCP, arity = 1) public void setZmqEnableTcp(boolean zmqEnableTcp) { this.zmqEnableTcp = zmqEnableTcp; } @@ -667,7 +666,7 @@ public boolean isZmqEnableIpc() { } @JsonProperty - @Parameter(names = "--zmq-enable-ipc", description = ZMQConfig.Descriptions.ZMQ_ENABLE_IPC, arity = 1) + @Parameter(names = "--zmq_enable_ipc", description = ZMQConfig.Descriptions.ZMQ_ENABLE_IPC, arity = 1) public void setZmqEnableIpc(boolean zmqEnableIpc) { this.zmqEnableIpc = zmqEnableIpc; } @@ -678,7 +677,7 @@ public int getZmqPort() { } @JsonProperty - @Parameter(names = "--zmq-port", description = ZMQConfig.Descriptions.ZMQ_PORT) + @Parameter(names = "--zmq_port", description = ZMQConfig.Descriptions.ZMQ_PORT) protected void setZmqPort(int zmqPort) { this.zmqPort = zmqPort; this.zmqEnableTcp = true; @@ -690,7 +689,7 @@ public int getZmqThreads() { } @JsonProperty - @Parameter(names = "--zmq-threads", description = ZMQConfig.Descriptions.ZMQ_THREADS) + @Parameter(names = "--zmq_threads", description = ZMQConfig.Descriptions.ZMQ_THREADS) protected void setZmqThreads(int zmqThreads) { this.zmqThreads = zmqThreads; } @@ -701,7 +700,7 @@ public String getZmqIpc() { } @JsonProperty - @Parameter(names = "--zmq-ipc", description = ZMQConfig.Descriptions.ZMQ_IPC) + @Parameter(names = "--zmq_ipc", description = ZMQConfig.Descriptions.ZMQ_IPC) protected void setZmqIpc(String zmqIpc) { this.zmqIpc = zmqIpc; this.zmqEnableIpc = true; @@ -713,7 +712,7 @@ public int getqSizeNode() { } @JsonProperty - @Parameter(names = "--queue-size", description = NetworkConfig.Descriptions.Q_SIZE_NODE) + @Parameter(names = "--queue_size", description = NetworkConfig.Descriptions.Q_SIZE_NODE) protected void setqSizeNode(int qSizeNode) { this.qSizeNode = qSizeNode; } @@ -724,7 +723,7 @@ public double getpDropCacheEntry() { } @JsonProperty - @Parameter(names = "--p-drop-cache", description = NetworkConfig.Descriptions.P_DROP_CACHE_ENTRY) + @Parameter(names = "--p_drop_cache", description = NetworkConfig.Descriptions.P_DROP_CACHE_ENTRY) protected void setpDropCacheEntry(double pDropCacheEntry) { this.pDropCacheEntry = pDropCacheEntry; } @@ -735,7 +734,7 @@ public int getCacheSizeBytes() { } @JsonProperty - @Parameter(names = "--cache-size", description = NetworkConfig.Descriptions.CACHE_SIZE_BYTES) + @Parameter(names = "--cache_size", description = NetworkConfig.Descriptions.CACHE_SIZE_BYTES) protected void setCacheSizeBytes(int cacheSizeBytes) { this.cacheSizeBytes = cacheSizeBytes; } @@ -746,7 +745,7 @@ public int getMaxDepth() { } @JsonProperty - @Parameter(names = "--max-depth", description = TipSelConfig.Descriptions.MAX_DEPTH) + @Parameter(names = "--max_depth", description = TipSelConfig.Descriptions.MAX_DEPTH) protected void setMaxDepth(int maxDepth) { this.maxDepth = maxDepth; } @@ -756,7 +755,7 @@ public double getAlpha() { return alpha; } - @JsonProperty("TIPSELECTION_ALPHA") + @JsonProperty @Parameter(names = "--alpha", description = TipSelConfig.Descriptions.ALPHA) protected void setAlpha(double alpha) { this.alpha = alpha; @@ -773,7 +772,7 @@ public int getBelowMaxDepthTransactionLimit() { } @JsonProperty - @Parameter(names = "--max-analyzed-transactions", description = TipSelConfig.Descriptions.BELOW_MAX_DEPTH_TRANSACTION_LIMIT) + @Parameter(names = "--max_analyzed_transactions", description = TipSelConfig.Descriptions.BELOW_MAX_DEPTH_TRANSACTION_LIMIT) protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { this.maxAnalyzedTransactions = maxAnalyzedTransactions; } @@ -781,7 +780,7 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { // Validator Manager public boolean getValidatorManagerEnabled() {return validatorManagerEnabled; } @JsonProperty - @Parameter(names = {"--validator-manager"}, description = ValidatorManagerConfig.Descriptions.VALIDATOR_MANAGER_ENABLED, arity = 1) + @Parameter(names = {"--validator_manager"}, description = ValidatorManagerConfig.Descriptions.VALIDATOR_MANAGER_ENABLED, arity = 1) protected void setValidatorManagerEnabled(boolean validatorManagerEnabled) { this.validatorManagerEnabled = validatorManagerEnabled; } @Override @@ -793,13 +792,13 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { @Override public int getUpdateValidatorDelay() {return updateValidatorDelay; } @JsonProperty - @Parameter(names = {"--update-validator"}, description = ValidatorManagerConfig.Descriptions.UPDATE_VALIDATOR_DELAY) + @Parameter(names = {"--update_validator_delay"}, description = ValidatorManagerConfig.Descriptions.UPDATE_VALIDATOR_DELAY) protected void setUpdateValidatorDelay(int updateValidatorDelay) { this.updateValidatorDelay = updateValidatorDelay; } @Override public int getStartRoundDelay() {return startRoundDelay; } @JsonProperty - @Parameter(names = {"--start-validator"}, description = ValidatorManagerConfig.Descriptions.START_ROUND_DELAY) + @Parameter(names = {"--start_validator"}, description = ValidatorManagerConfig.Descriptions.START_ROUND_DELAY) protected void setStartRoundDelay(int startRoundDelay) { this.startRoundDelay = startRoundDelay; } @Override @@ -816,7 +815,7 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { public String getValidatorPath() {return validatorPath; } @JsonProperty - @Parameter(names = {"--validator-path"}, description = MilestoneConfig.Descriptions.VALIDATOR_PATH) + @Parameter(names = {"--validator_path"}, description = MilestoneConfig.Descriptions.VALIDATOR_PATH) protected void setValidatorPath(String validatorPath) { this.validatorPath = validatorPath; } @@ -834,8 +833,8 @@ protected void setValidator(boolean validator) { public Set getInitialValidators() {return initialValidators; } @Override - public boolean isDontValidateTestnetMilestoneSig() { - return false; + public boolean isValidateTestnetMilestoneSig() { + return true; } @Override @@ -859,7 +858,7 @@ public int getRoundDuration() { @Override public int getRoundPause() { return roundPause; } @JsonProperty - @Parameter(names = {"--round-pause"}, description = MilestoneConfig.Descriptions.ROUND_PAUSE) + @Parameter(names = {"--round_pause"}, description = MilestoneConfig.Descriptions.ROUND_PAUSE) protected void setRoundPause(int roundPause) { this.roundPause = roundPause; } @Override @@ -899,7 +898,7 @@ public int getPowThreads() { return powThreads; } @JsonProperty - @Parameter(names = "--pow-threads", description = PoWConfig.Descriptions.POW_THREADS) + @Parameter(names = "--pow_threads", description = PoWConfig.Descriptions.POW_THREADS) protected void setPowThreads(int powThreads) { this.powThreads = powThreads; } @@ -910,7 +909,7 @@ public boolean isSaveLogEnabled() { } @JsonProperty - @Parameter(names = {"--savelog-enabled"}, description = LoggingConfig.Descriptions.SAVELOG_ENABLED) + @Parameter(names = {"--savelog_enabled"}, description = LoggingConfig.Descriptions.SAVELOG_ENABLED) protected void setSaveLogEnabled(boolean saveLogEnabled) { this.saveLogEnabled = saveLogEnabled; } @@ -921,7 +920,7 @@ public String getSaveLogBasePath() { } @JsonProperty - @Parameter(names = {"--savelog-path"}, description = LoggingConfig.Descriptions.SAVELOG_BASE_PATH) + @Parameter(names = {"--savelog_path"}, description = LoggingConfig.Descriptions.SAVELOG_BASE_PATH) protected void setSaveLogBasePath(String saveLogBasePath) { this.saveLogBasePath = saveLogBasePath; } @@ -932,7 +931,7 @@ public String getSaveLogXMLFile() { } @JsonProperty - @Parameter(names = {"--savelog-xml"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE) + @Parameter(names = {"--savelog_xml"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE) protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; } @@ -941,9 +940,9 @@ public interface Defaults { //API int API_PORT = 8085; String API_HOST = "localhost"; - List REMOTE_LIMIT_API = PendulumUtils.createImmutableList(); // "addNeighbors", "getNeighbors", "removeNeighbors", "attachToTangle", "interruptAttachingToTangle" <- TODO: limit these in production! - InetAddress REMOTE_LIMIT_API_DEFAULT_HOST = InetAddress.getLoopbackAddress(); - List REMOTE_LIMIT_API_HOSTS = PendulumUtils.createImmutableList(REMOTE_LIMIT_API_DEFAULT_HOST); + List IGNORED_API_ENDPOINTS = PendulumUtils.createImmutableList(); // "addNeighbors", "getNeighbors", "removeNeighbors", "attachToTangle", "interruptAttachingToTangle" <- TODO: limit these in production! + InetAddress ALLOWED_API_DEFAULT_HOST = InetAddress.getLoopbackAddress(); + List ALLOWED_API_HOSTS = PendulumUtils.createImmutableList(ALLOWED_API_DEFAULT_HOST); int MAX_FIND_TRANSACTIONS = 100_000; int MAX_REQUESTS_LIST = 1_000; int MAX_GET_TRANSACTION_STRINGS = 10_000; diff --git a/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java b/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java index d4426fa7..93bde292 100644 --- a/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java @@ -31,9 +31,9 @@ public interface MilestoneConfig extends Config { */ Set getInitialValidators(); /** - * @return {@value Descriptions#DONT_VALIDATE_TESTNET_MILESTONE_SIG} + * @return {@value Descriptions#VALIDATE_TESTNET_MILESTONE_SIG} */ - boolean isDontValidateTestnetMilestoneSig(); + boolean isValidateTestnetMilestoneSig(); /** * @return {@value Descriptions#GENESIS_TIME} */ @@ -64,7 +64,7 @@ interface Descriptions { String VALIDATOR_PATH = "A path to a file containing the seed / keyfile has to be passed."; String VALIDATOR_SEED_PATH = "A path to a file containing the seed has to be passed."; String INITIAL_VALIDATORS = "The addresses of validators the network starts with"; - String DONT_VALIDATE_TESTNET_MILESTONE_SIG = "Disable validator validation on testnet"; + String VALIDATE_TESTNET_MILESTONE_SIG = "Disable validator validation on testnet"; String GENESIS_TIME = "Time when the ledger started."; String ROUND_DURATION = "Duration of a round in milli secounds."; String ROUND_PAUSE = "Duration of time to finalize the round in milli secounds."; diff --git a/src/main/java/net/helix/pendulum/conf/TestnetConfig.java b/src/main/java/net/helix/pendulum/conf/TestnetConfig.java index 82361cb6..fa207272 100644 --- a/src/main/java/net/helix/pendulum/conf/TestnetConfig.java +++ b/src/main/java/net/helix/pendulum/conf/TestnetConfig.java @@ -8,7 +8,7 @@ public class TestnetConfig extends BasePendulumConfig { - protected boolean dontValidateTestnetMilestoneSig = Defaults.DONT_VALIDATE_MILESTONE_SIG; + protected boolean validateTestnetMilestoneSig = Defaults.VALIDATE_MILESTONE_SIG; protected String snapshotFile = Defaults.SNAPSHOT_FILE; protected String snapshotSignatureFile = Defaults.SNAPSHOT_SIG; protected long snapshotTime = Defaults.SNAPSHOT_TIME; @@ -32,14 +32,14 @@ public boolean isTestnet() { } @Override - public boolean isDontValidateTestnetMilestoneSig() { - return dontValidateTestnetMilestoneSig; + public boolean isValidateTestnetMilestoneSig() { + return validateTestnetMilestoneSig; } @JsonProperty - @Parameter(names = "--testnet-no-milestone-sign-validation", description = MilestoneConfig.Descriptions.DONT_VALIDATE_TESTNET_MILESTONE_SIG) - protected void setDontValidateTestnetMilestoneSig(boolean dontValidateTestnetMilestoneSig) { - this.dontValidateTestnetMilestoneSig = dontValidateTestnetMilestoneSig; + @Parameter(names = "--validate_testnet_milestone_sig", description = MilestoneConfig.Descriptions.VALIDATE_TESTNET_MILESTONE_SIG) + protected void setValidateTestnetMilestoneSig(boolean validateTestnetMilestoneSig) { + this.validateTestnetMilestoneSig = validateTestnetMilestoneSig; } @Override @@ -159,7 +159,7 @@ public long getGenesisTime() { public interface Defaults { long GENESIS_TIME = 1571279107785L; - boolean DONT_VALIDATE_MILESTONE_SIG = false; + boolean VALIDATE_MILESTONE_SIG = true; String LOCAL_SNAPSHOTS_BASE_PATH = "snapshot-testnet"; String SNAPSHOT_FILE = "/snapshotTestnet.txt"; int REQUEST_HASH_SIZE = 32; diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 1b948315..1ae6fb8a 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -248,7 +248,7 @@ private AbstractResponse process(final String requestString, InetAddress netAddr // Is this command allowed to be run from this request address? // We check the remote limit API configuration. - if (configuration.getRemoteLimitApi().contains(command) && !configuration.getRemoteTrustedApiHosts().contains(netAddress)) { + if (configuration.getIgnoredApiEndpoints().contains(command) && !configuration.getAllowedApiHosts().contains(netAddress)) { return AccessLimitedResponse.create("COMMAND " + command + " is not available on this node"); } diff --git a/src/main/java/net/helix/pendulum/service/Feature.java b/src/main/java/net/helix/pendulum/service/Feature.java index b954f3aa..80616010 100644 --- a/src/main/java/net/helix/pendulum/service/Feature.java +++ b/src/main/java/net/helix/pendulum/service/Feature.java @@ -75,7 +75,7 @@ public static Feature[] calculateFeatures(PendulumConfig configuration) { List apiFeatures = new ArrayList(Arrays.asList(PROOF_OF_WORK)); - for (String disabled : configuration.getRemoteLimitApi()) { + for (String disabled : configuration.getIgnoredApiEndpoints()) { switch (disabled) { case "attachToTangle": apiFeatures.remove(PROOF_OF_WORK); diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java index 9e08990b..8cd5279d 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java @@ -49,7 +49,7 @@ public MilestonePublisher(PendulumConfig configuration, API api, CandidateTracke delay = config.getRoundDuration(); mwm = config.getMwm(); - sign = !config.isDontValidateTestnetMilestoneSig(); + sign = config.isValidateTestnetMilestoneSig(); pubkeyDepth = config.getMilestoneKeyDepth(); keyfileIndex = 0; maxKeyIndex = (int) Math.pow(2, pubkeyDepth); diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java index 86250137..a6b59801 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java @@ -188,7 +188,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); log.trace("valid signature: {}", validSignature); - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || + if ((config.isTestnet() && !config.isValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { transactionViewModel.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); diff --git a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java index 9c9953df..73ca2c84 100644 --- a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java @@ -63,7 +63,7 @@ public ValidatorValidity validateValidators(TransactionViewModel transactionView boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getValidatorManagerAddress().equals(senderAddress) && validSignature)) { + if ((config.isTestnet() && !config.isValidateTestnetMilestoneSig()) || (config.getValidatorManagerAddress().equals(senderAddress) && validSignature)) { return VALID; } else { return INVALID; diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java index f1d982a7..afb96cd8 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java @@ -78,7 +78,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM MerkleOptions options = new MerkleOptions(mode, securityLevel,config.getMilestoneKeyDepth(), senderAddress); boolean validSignature = MerkleFactory.create(MerkleFactory.MerkleTree, options).validateMerkleSignature(bundleTransactionViewModels); - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validator.contains(senderAddress)) && validSignature) { + if ((config.isTestnet() && !config.isValidateTestnetMilestoneSig()) || (validator.contains(senderAddress)) && validSignature) { return VALID; } else { return INVALID; diff --git a/src/test/java/net/helix/pendulum/conf/ConfigTest.java b/src/test/java/net/helix/pendulum/conf/ConfigTest.java index 8791aa0c..3e9135ca 100644 --- a/src/test/java/net/helix/pendulum/conf/ConfigTest.java +++ b/src/test/java/net/helix/pendulum/conf/ConfigTest.java @@ -55,57 +55,78 @@ public void argsParsingMainnetTest() { "-u", "4200", "-t", "5200", "-n", "udp://neighbor1 neighbor, tcp://neighbor2", - "--api-host", "1.1.1.1", - "--remote-limit-api", "call1 call2, call3", - "--max-find-transactions", "500", - "--max-requests-list", "1000", - "--max-get-transaction-strings", "4000", - "--max-body-length", "220", - "--remote-auth", "2.2.2.2", - "--p-remove-request", "0.23", - "--send-limit", "1000", - "--max-peers", "10", - "--dns-refresher", "false", - "--dns-resolution", "false", - "--XI-dir", "/XI", - "--db-path", "/db", - "--db-log-path", "/dblog", - "--zmq-enabled", "true", - //we ignore this on mainnet - "--mwm", "4", - "--testnet-coordinator", "TTTTTTTTT", - "--test-no-coo-validation", - //this should be ignored everywhere - "--fake-config" + "--api_host", "1.1.1.1", + "--ignored_api_endpoints", "call1 call2, call3", + "--max_find_transactions", "500", + "--max_requests_list", "1000", + "--max_get_transaction_strings", "4000", + "--max_body_length", "220", + "--remote_auth", "2.2.2.2", + "--p_remove_request", "0.23", + "--send_limit", "1000", + "--max_peers", "10", + "--dns_refresher", "false", + "--dns_resolution", "false", + "--XI_dir", "/XI", + "--db_path", "/db", + "--db_log_path", "/dblog", + "--zmq_enabled", "true", + "--mwm", "4", //we ignore this on mainnet + "--testnet_coordinator", "TTTTTTTTT", + "--test_no_coo_validation", + "--fake_config" //this should be ignored everywhere }; + PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); - Assert.assertThat("wrong config class created", pendulumConfig, CoreMatchers.instanceOf(MainnetConfig.class)); + + Assert.assertThat("wrong config class created", + pendulumConfig, CoreMatchers.instanceOf(MainnetConfig.class)); pendulumConfig.parseConfigFromArgs(args); + Assert.assertEquals("port value", 8089, pendulumConfig.getApiPort()); + Assert.assertEquals("udp port", 4200, pendulumConfig.getUdpReceiverPort()); + Assert.assertEquals("tcp port", 5200, pendulumConfig.getTcpReceiverPort()); + Assert.assertEquals("neighbors", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); + Assert.assertEquals("api host", "1.1.1.1", pendulumConfig.getApiHost()); - Assert.assertEquals("remote limit api", Arrays.asList("call1", "call2", "call3"), - pendulumConfig.getRemoteLimitApi()); + + Assert.assertEquals("ignored api endpoints", Arrays.asList("call1", "call2", "call3"), + pendulumConfig.getIgnoredApiEndpoints()); + Assert.assertEquals("max find transactions", 500, pendulumConfig.getMaxFindTransactions()); + Assert.assertEquals("max requests list", 1000, pendulumConfig.getMaxRequestsList()); + Assert.assertEquals("max get bytes", 4000, pendulumConfig.getMaxTransactionStrings()); + Assert.assertEquals("max body length", 220, pendulumConfig.getMaxBodyLength()); - Assert.assertEquals("remote-auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); + + Assert.assertEquals("remote_auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); + Assert.assertEquals("p remove request", 0.23d, pendulumConfig.getpRemoveRequest(), 0d); + Assert.assertEquals("send limit", 1000, pendulumConfig.getSendLimit()); + Assert.assertEquals("max peers", 10, pendulumConfig.getMaxPeers()); + Assert.assertEquals("dns refresher", false, pendulumConfig.isDnsRefresherEnabled()); + Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); + Assert.assertEquals("XI-dir", "/XI", pendulumConfig.getXiDir()); + Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); + Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); + Assert.assertNotEquals("mwm", 4, pendulumConfig.getMwm()); + Assert.assertNotEquals("coo", pendulumConfig.getValidatorManagerAddress(), "TTTTTTTTT"); - Assert.assertEquals("--testnet-no-milestone-sign-validation", false, pendulumConfig.isDontValidateTestnetMilestoneSig()); } @Test @@ -123,28 +144,28 @@ public void argsParsingTestnetTest() { "-u", "4200", "-t", "5200", "-n", "udp://neighbor1 neighbor, tcp://neighbor2", - "--api-host", "1.1.1.1", - "--remote-limit-api", "call1 call2, call3", - "--max-find-transactions", "500", - "--max-requests-list", "1000", - "--max-get-transaction-strings", "4000", - "--max-body-length", "220", - "--remote-auth", "2.2.2.2", - "--p-remove-request", "0.23", - "--send-limit", "1000", - "--max-peers", "10", - "--dns-refresher", "false", - "--dns-resolution", "false", - "--XI-dir", "/XI", - "--db-path", "/db", - "--db-log-path", "/dblog", - "--zmq-enabled", "true", + "--api_host", "1.1.1.1", + "--ignored_api_endpoints", "call1 call2, call3", + "--max_find_transactions", "500", + "--max_requests_list", "1000", + "--max_get_transaction_strings", "4000", + "--max_body_length", "220", + "--remote_auth", "2.2.2.2", + "--p_remove_request", "0.23", + "--send_limit", "1000", + "--max_peers", "10", + "--dns_refresher", "false", + "--dns_resolution", "false", + "--XI_dir", "/XI", + "--db_path", "/db", + "--db_log_path", "/dblog", + "--zmq_enabled", "true", //we ignore this on mainnet "--mwm", "4", - "--testnet-coordinator", "TTTTTTTTT", - "--testnet-no-milestone-sign-validation", + "--testnet_coordinator", "TTTTTTTTT", + "--testnet_no_milestone_sign_validation", //this should be ignored everywhere - "--fake-config" + "--fake_config" }; PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(true); Assert.assertThat("wrong config class created", pendulumConfig, CoreMatchers.instanceOf(TestnetConfig.class)); @@ -156,31 +177,31 @@ public void argsParsingTestnetTest() { Assert.assertEquals("neighbors", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); Assert.assertEquals("api host", "1.1.1.1", pendulumConfig.getApiHost()); - Assert.assertEquals("remote limit api", Arrays.asList("call1", "call2", "call3"), - pendulumConfig.getRemoteLimitApi()); + Assert.assertEquals("ignored api endpoints", Arrays.asList("call1", "call2", "call3"), + pendulumConfig.getIgnoredApiEndpoints()); Assert.assertEquals("max find transactions", 500, pendulumConfig.getMaxFindTransactions()); Assert.assertEquals("max requests list", 1000, pendulumConfig.getMaxRequestsList()); Assert.assertEquals("max get tx strings", 4000, pendulumConfig.getMaxTransactionStrings()); Assert.assertEquals("max body length", 220, pendulumConfig.getMaxBodyLength()); - Assert.assertEquals("remote-auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); + Assert.assertEquals("remote_auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); Assert.assertEquals("p remove request", 0.23d, pendulumConfig.getpRemoveRequest(), 0d); Assert.assertEquals("send limit", 1000, pendulumConfig.getSendLimit()); Assert.assertEquals("max peers", 10, pendulumConfig.getMaxPeers()); Assert.assertEquals("dns refresher", false, pendulumConfig.isDnsRefresherEnabled()); Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); - Assert.assertEquals("XI-dir", "/XI", pendulumConfig.getXiDir()); + Assert.assertEquals("XI_dir", "/XI", pendulumConfig.getXiDir()); Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); Assert.assertEquals("mwm", 4, pendulumConfig.getMwm()); //Assert.assertEquals("coo", "TTTTTTTTT", pendulumConfig.getValidatorManagerAddress()); - Assert.assertEquals("--testnet-no-milestone-sign-validation", true, - pendulumConfig.isDontValidateTestnetMilestoneSig()); + Assert.assertEquals("validate testnet milestone signatures", true, + pendulumConfig.isValidateTestnetMilestoneSig()); } @Test public void iniParsingMainnetTest() throws Exception { String iniContent = new StringBuilder() - .append("[HLX]").append(System.lineSeparator()) + .append("[PENDULUM]").append(System.lineSeparator()) .append("PORT = 8088").append(System.lineSeparator()) .append("NEIGHBORS = udp://neighbor1 neighbor, tcp://neighbor2").append(System.lineSeparator()) .append("ZMQ_ENABLED = true").append(System.lineSeparator()) @@ -207,7 +228,7 @@ public void iniParsingMainnetTest() throws Exception { @Test public void iniParsingTestnetTest() throws Exception { String iniContent = new StringBuilder() - .append("[HLX]").append(System.lineSeparator()) + .append("[PENDULUM]").append(System.lineSeparator()) .append("PORT = 8088").append(System.lineSeparator()) .append("NEIGHBORS = udp://neighbor1 neighbor, tcp://neighbor2").append(System.lineSeparator()) .append("ZMQ_ENABLED = true").append(System.lineSeparator()) @@ -216,7 +237,7 @@ public void iniParsingTestnetTest() throws Exception { .append("MWM = 4").append(System.lineSeparator()) .append("NUMBER_OF_KEYS_IN_A_MILESTONE = 3").append(System.lineSeparator()) .append("DONT_VALIDATE_TESTNET_MILESTONE_SIG = true").append(System.lineSeparator()) - .append("TIPSELECTION_ALPHA = 1.1").append(System.lineSeparator()) + .append("ALPHA = 1.1").append(System.lineSeparator()) //doesn't do anything .append("REMOTE") .append("FAKE").append(System.lineSeparator()) @@ -243,9 +264,9 @@ public void iniParsingTestnetTest() throws Exception { Assert.assertEquals("P_REMOVE_REQUEST", 0.4d, pendulumConfig.getpRemoveRequest(), 0); Assert.assertEquals("MWM", 4, pendulumConfig.getMwm()); Assert.assertEquals("NUMBER_OF_KEYS_IN_A_MILESTONE", 3, pendulumConfig.getNumberOfKeysInMilestone()); - Assert.assertEquals("TIPSELECTION_ALPHA", 1.1d, pendulumConfig.getAlpha(), 0); - Assert.assertEquals("DONT_VALIDATE_TESTNET_MILESTONE_SIG", - pendulumConfig.isDontValidateTestnetMilestoneSig(), true); + Assert.assertEquals("ALPHA", 1.1d, pendulumConfig.getAlpha(), 0); + Assert.assertEquals("VALIDATE_TESTNET_MILESTONE_SIG", + pendulumConfig.isValidateTestnetMilestoneSig(), true); //prove that REMOTE did nothing Assert.assertEquals("API_HOST", pendulumConfig.getApiHost(), "localhost"); } @@ -253,7 +274,7 @@ public void iniParsingTestnetTest() throws Exception { @Test(expected = IllegalArgumentException.class) public void testInvalidIni() throws IOException { String iniContent = new StringBuilder() - .append("[HLX]").append(System.lineSeparator()) + .append("[PENDULUM]").append(System.lineSeparator()) .append("REVALIDATE") .toString(); try (Writer writer = new FileWriter(configFile)) { @@ -278,10 +299,10 @@ public void backwardsIniCompatibilityTest() { } @Test - public void dontValidateMilestoneSigDefaultValueTest() { + public void validateMilestoneSigDefaultValueTest() { PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(true); - Assert.assertFalse("By default testnet should be validating milestones", - pendulumConfig.isDontValidateTestnetMilestoneSig()); + Assert.assertTrue("By default testnet should be validating milestones", + pendulumConfig.isValidateTestnetMilestoneSig()); } private String deriveNameFromSetter(Method setter) { @@ -310,7 +331,7 @@ public enum LegacyDefaultConf { TCP_RECEIVER_PORT, TESTNET, DEBUG, - REMOTE_LIMIT_API, + IGNORED_API_ENDPOINTS, REMOTE_AUTH, NEIGHBORS, XI_DIR, @@ -329,7 +350,7 @@ public enum LegacyDefaultConf { DNS_RESOLUTION_ENABLED, DNS_REFRESHER_ENABLED, COORDINATOR, - DONT_VALIDATE_TESTNET_MILESTONE_SIG, + VALIDATE_TESTNET_MILESTONE_SIG, REVALIDATE, RESCAN_DB, MIN_RANDOM_WALKS, @@ -354,7 +375,7 @@ public enum LegacyDefaultConf { TRANSACTION_PACKET_SIZE, REQUEST_HASH_SIZE, SNAPSHOT_TIME, - TIPSELECTION_ALPHA, + ALPHA, BELOW_MAX_DEPTH_TRANSACTION_LIMIT } } diff --git a/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java b/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java index cc278d2b..37250018 100644 --- a/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java +++ b/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java @@ -10,7 +10,7 @@ public class ZMQConfigTest { @Test public void isZmqEnabledLegacy() { String[] args = { - "--zmq-enabled", "true", + "--zmq_enabled", "true", }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -22,8 +22,8 @@ public void isZmqEnabledLegacy() { @Test public void isZmqEnabled() { String[] args = { - "--zmq-enable-tcp", "true", - "--zmq-enable-ipc", "true", + "--zmq_enable_tcp", "true", + "--zmq_enable_ipc", "true", }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -35,7 +35,7 @@ public void isZmqEnabled() { @Test public void isZmqEnableTcp() { String[] args = { - "--zmq-enable-tcp", "true" + "--zmq_enable_tcp", "true" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -46,7 +46,7 @@ public void isZmqEnableTcp() { @Test public void isZmqEnableIpc() { String[] args = { - "--zmq-enable-ipc", "true" + "--zmq_enable_ipc", "true" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -57,7 +57,7 @@ public void isZmqEnableIpc() { @Test public void getZmqPort() { String[] args = { - "--zmq-port", "8899" + "--zmq_port", "8899" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -68,7 +68,7 @@ public void getZmqPort() { @Test public void getZmqThreads() { String[] args = { - "--zmq-threads", "5" + "--zmq_threads", "5" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -78,7 +78,7 @@ public void getZmqThreads() { @Test public void getZmqIpc() { String[] args = { - "--zmq-ipc", "ipc://test" + "--zmq_ipc", "ipc://test" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); From b39bcf00b750cb31f51c24f664c720e4d66a00ee Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Wed, 27 Nov 2019 16:48:13 +0100 Subject: [PATCH 19/36] Remove Makefile and Jenkinsfile --- Jenkinsfile | 28 ---------------------------- Makefile | 25 ------------------------- 2 files changed, 53 deletions(-) delete mode 100644 Jenkinsfile delete mode 100644 Makefile diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index efe304ba..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,28 +0,0 @@ -pipeline { - agent { - docker { - image 'maven:3.6-jdk-8-slim' - args '-v /root/.m2:/root/.m2' - } - } - options { - skipStagesAfterUnstable() - } - stages { - stage('Build') { - steps { - sh 'mvn -B -DskipTests clean package' - } - } - stage('Test') { - steps { - sh 'mvn integration-test -Dlogging-level=INFO' - } - post { - always { - junit 'target/surefire-reports/*.xml' - } - } - } - } -} diff --git a/Makefile b/Makefile deleted file mode 100644 index 398147ef..00000000 --- a/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -clean: - @echo "Removing all data from pendulum for fresh start!" - @rm -rf db - @rm -rf mainnet_snapshots - @rm -rf testnet_snapshots - @rm -rf logs - @rm -rf snapshots - @rm -rf local_snapshots - @rm -rf modules - @rm -rf snapshot - @rm -rf mainnet - @rm -rf testnet - @rm -rf spent-addresses-db - @rm -rf spent-addresses-log - @rm -rf mainnetdb - @rm -rf mainnet.log - @rm -rf testnetdb - @rm -rf testnet.log - @rm -rf snapshot-mainnet - @rm -rf snapshot-testnet - @rm -rf logs -start-node: - @echo "Starting pendulum" - @java -jar target/pen*.jar --config config.ini - From 8a6c2299176aadadbe5ff2e58e86f04ed71b0d8a Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Wed, 27 Nov 2019 16:56:00 +0100 Subject: [PATCH 20/36] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6261b36a..ed8e0cf8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.0.6 +- Configuration flag names used on command line (by Jcommander) were changed from kabab-case to snake_case to match the config.ini style (used by Jackson). +- Renaming/Refactoring config variables to identify explicit purpose for devops +- Renaming/Refactoring double negative variable names for reader sanity + ## 1.0.5 - Fixed `getBalance`: `RoundViewModel.get()` returns null on `index`=0, thus NPE was thrown when `references` were not passed and the first round hadn't been completed. In our implementation the snapshot is already constructed based on relative confirmations, thus it suffices for `getBalances` to respond with balance according to `latestSnapshot`. - Updated `previousEpochsSpentAddresses` resource files From cf9c71bf56abc163c5892c6718722ebbabdcd5ec Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 28 Nov 2019 17:38:33 +0100 Subject: [PATCH 21/36] sort milestones and tips before building merkle tree --- .../pendulum/controllers/RoundViewModel.java | 16 ++++++++++++---- .../java/net/helix/pendulum/service/API.java | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 8d72113d..f61db623 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -255,7 +255,9 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); + List prevMilestonesList = new ArrayList<>(prevMilestones); + Collections.sort(prevMilestonesList); + List> merkleTree = Merkle.buildMerkleTree(prevMilestonesList); if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { if (prevMilestones.isEmpty()) { trunk.add(Hash.NULL_HASH); @@ -279,7 +281,9 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); + List confirmedTipsList = new ArrayList<>(confirmedTips); + Collections.sort(confirmedTipsList); + List> merkleTree = Merkle.buildMerkleTree(confirmedTipsList); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { if (confirmedTips.isEmpty()){ branch.add(Hash.NULL_HASH); @@ -297,7 +301,9 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); + List prevMilestonesList = new ArrayList<>(prevMilestones); + Collections.sort(prevMilestonesList); + List> merkleTree = Merkle.buildMerkleTree(prevMilestonesList); if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { if (prevMilestones.isEmpty()) { branch.add(Hash.NULL_HASH); @@ -475,7 +481,9 @@ public Integer index() { } public Hash getMerkleRoot() { - List> merkleTree = Merkle.buildMerkleTree(new LinkedList<>(getHashes())); + List milestoneHashes = new ArrayList<>(getHashes()); + Collections.sort(milestoneHashes); + List> merkleTree = Merkle.buildMerkleTree(milestoneHashes); Hash root = merkleTree.get(merkleTree.size()-1).get(0); return root; } diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 58eb06e3..6f35bd86 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -1677,6 +1677,7 @@ private List addMilestoneReferences(List confirmedTips, int roundInd txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones } //branch + Collections.sort(confirmedTips); List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips } From d86cbd12b66b766ecbe7445095e8b7f5a92faab1 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Thu, 28 Nov 2019 18:10:47 +0100 Subject: [PATCH 22/36] add tests for config parsing from command line for all known flags --- .../pendulum/conf/BasePendulumConfig.java | 9 +- .../net/helix/pendulum/conf/ConfigTest.java | 214 +++++++++++++++--- 2 files changed, 183 insertions(+), 40 deletions(-) diff --git a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java index 1d637ff7..bc1887e9 100644 --- a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java +++ b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java @@ -142,7 +142,7 @@ public JCommander parseConfigFromArgs(String[] args) throws ParameterException { .acceptUnknownOptions(true) .allowParameterOverwriting(true) //This is the first line of JCommander Usage - .programName("java -jar hlx-" + Main.VERSION + ".jar") + .programName("java -jar pen-" + Main.VERSION + ".jar") .build(); if (ArrayUtils.isNotEmpty(args)) { jCommander.parse(args); @@ -213,7 +213,7 @@ public void setAllowedApiHosts(String allowedApiHosts) { try { return InetAddress.getByName(host.trim()); } catch (UnknownHostException e) { - throw new ParameterException("Invalid value for --remote_trusted_api_hosts address: ", e); + throw new ParameterException("Invalid value for --allowed_api_hosts address: ", e); } }).collect(Collectors.toList()); // always make sure that localhost exists as trusted host @@ -344,6 +344,7 @@ protected void setDnsRefresherEnabled(boolean dnsRefresherEnabled) { this.dnsRefresherEnabled = dnsRefresherEnabled; } + @Override public boolean isDnsResolutionEnabled() { return dnsResolutionEnabled; @@ -372,7 +373,7 @@ public String getXiDir() { } @JsonProperty - @Parameter(names = {"--XI_dir"}, description = XIConfig.Descriptions.XI_DIR) + @Parameter(names = {"--xi_dir"}, description = XIConfig.Descriptions.XI_DIR) protected void setXiDir(String xiDir) { this.xiDir = xiDir; } @@ -844,7 +845,7 @@ public long getGenesisTime() { @JsonProperty @Parameter(names = {"--genesis"}, description = MilestoneConfig.Descriptions.GENESIS_TIME) - protected void setGenesisTime(int genesisTime) { this.genesisTime = genesisTime; } + protected void setGenesisTime(long genesisTime) { this.genesisTime = genesisTime; } @Override public int getRoundDuration() { diff --git a/src/test/java/net/helix/pendulum/conf/ConfigTest.java b/src/test/java/net/helix/pendulum/conf/ConfigTest.java index 3e9135ca..bb156c59 100644 --- a/src/test/java/net/helix/pendulum/conf/ConfigTest.java +++ b/src/test/java/net/helix/pendulum/conf/ConfigTest.java @@ -46,9 +46,180 @@ public static void tearDownAfterClass() throws IOException { } /* - Test that iterates over common configs. It also attempts to check different types of types (double, boolean, string) + test that --remote returns true without specifically giving true on cmd line + */ + //@Test + public void remoteFlagTest() { + String[] args = {"--remote"}; + PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); + pendulumConfig.parseConfigFromArgs(args); + Assert.assertEquals("The api interface should be open to the public", + "0.0.0.0", pendulumConfig.getApiHost()); + } + + /* + The order of flags provided on cmd line makes a difference. + Test that specifying --remote after any --api_host flag will set the value for api_host. */ @Test + public void testFlagPrecedentRule(){ + String[] args = { + "--api_host", "localhost", + "--remote" + }; + PendulumConfig penConfig = ConfigFactory.createPendulumConfig(false); + penConfig.parseConfigFromArgs(args); + Assert.assertNotEquals(penConfig.getApiHost(), "localhost"); + Assert.assertEquals(penConfig.getApiHost(), "0.0.0.0"); + } + @Test + public void testIgnoredApiHosts(){ + String[] args = { + "--ignored_api_endpoints", + "addNeighbor, getNeighbors, removeNeighbors, attachToTangle, interruptAttachingToTangle" + }; + PendulumConfig penConfig = ConfigFactory.createPendulumConfig(false); + penConfig.parseConfigFromArgs(args); + Assert.assertEquals(penConfig.getIgnoredApiEndpoints(), + Arrays.asList( + "addNeighbor", "getNeighbors", "removeNeighbors", "attachToTangle", "interruptAttachingToTangle") + ); + } + /* + Test all known command line arguments with non-default value to make sure they are being parsed and set correctly. + */ + @Test + public void testCommandLineArgs(){ + String[] args = { + "--xi_dir", "not_here", + "--remote", "true", + //"--api_host", "localhost", + "--api_port", "1234", + //"--allowed_api_hosts", "0.0.0.0", + "--pow_threads", "2", + //"--remote_auth", "pendulum:swings", + //"--ignored_api_endpoints", "addNeighbor, getNeighbors, removeNeighbors, attachToTangle, interruptAttachingToTangle", + "--max_body_length", "123456", + "--max_find_transactions", "123456", + "--max_get_transaction_strings", "123456", + "--max_requests_list", "123456", + "--db", "mongodb", //not available, but we're just making sure the cmd line flags work as expected + "--db_cache_size", "123456", + "--db_log_path", "somewhere_fancy.log", + "--db_path", "some_dir", + "--local_snapshots_base_path", "./snapshot-go-here", + "--local_snapshots_depth", "123456", + "--local_snapshots_enabled", "false", + "--local_snapshots_interval_synced", "123456", + "--local_snapshots_interval_unsynced", "123456", + "--local_snapshots_pruning_delay", "123456", + "--local_snapshots_pruning_enabled", "true", + "--neighbors", "udp://localhost:4101", + "--tcp_receiver_port", "1234", + "--udp_receiver_port", "1234", + "--max_peers", "1234", + "--p_drop_cache", "1.0", + "--p_drop_transaction", "1.0", + "--p_propagate_request", "1.0", + "--p_remove_request", "1.0", + "--p_reply_random", "1.0", + "--p_select_milestone", "1.0", + "--p_send_milestone", "1.0", + "--queue_size", "1234", + "--dns_refresher", "false", + "--dns_resolution", "false", + "--send_limit", "2", + "--cache_size", "1234", + "--update_validator_delay", "1234", + "--validator", "true", + "--validator_manager", "true", + "--validator_path", "./somewhere.txt", + "--start_validator", "10", + "--round", "60000", + "--round_pause", "60000", + "--max_analyzed_transactions", "60000", + "--max_depth", "10", + "--rescan", "true", + "--revalidate", "true", + "--remote_auth", "pendulum:swings", + "--alpha", "1.0", + "--genesis", "2345024001000", + "--zmq_enable_ipc", "true", + "--zmq_enable_tcp", "true", + "--zmq_enabled", "true", + "--zmq_ipc", "ipc://pendulum", + "--zmq_port", "1234", + "--zmq_threads", "10", + "--savelog_enabled", "true", + "--savelog_path", "./somewhere/", + "--savelog_xml", "/save.xml" + }; + + PendulumConfig penConfig = ConfigFactory.createPendulumConfig(false); + penConfig.parseConfigFromArgs(args); + Assert.assertEquals("xi_dir ", "not_here", penConfig.getXiDir()); + Assert.assertEquals("api_port ", 1234, penConfig.getApiPort()); + Assert.assertEquals("pow_threads ", 2, penConfig.getPowThreads()); + Assert.assertEquals("max_body_length ", 123456, penConfig.getMaxBodyLength()); + Assert.assertEquals("max_find_transactions ", 123456, penConfig.getMaxFindTransactions()); + Assert.assertEquals("max_get_transaction_strings ", 123456, penConfig.getMaxTransactionStrings()); + Assert.assertEquals("max_requests_list ", 123456, penConfig.getMaxRequestsList()); + Assert.assertEquals("db ", "mongodb" ,penConfig.getMainDb()); + Assert.assertEquals("db_cache_size ", 123456, penConfig.getDbCacheSize()); + Assert.assertEquals("db_log_path ", "somewhere_fancy.log" , penConfig.getDbLogPath()); + Assert.assertEquals("db_path ", "some_dir", penConfig.getDbPath()); + Assert.assertEquals("local_snapshots_base_path ", "./snapshot-go-here", penConfig.getLocalSnapshotsBasePath()); + Assert.assertEquals("local_snapshots_depth ", 123456, penConfig.getLocalSnapshotsDepth()); + Assert.assertEquals("local_snapshots_enabled ", false, penConfig.getLocalSnapshotsEnabled()); + Assert.assertEquals("local_snapshots_interval_synced ", 123456, penConfig.getLocalSnapshotsIntervalSynced()); + Assert.assertEquals("local_snapshots_interval_unsynced ", 123456, penConfig.getLocalSnapshotsIntervalUnsynced()); + Assert.assertEquals("local_snapshots_pruning_delay ", 123456, penConfig.getLocalSnapshotsPruningDelay()); + Assert.assertEquals("local_snapshots_pruning_enabled ", true, penConfig.getLocalSnapshotsPruningEnabled()); + Assert.assertEquals("neighbors ", Arrays.asList("udp://localhost:4101"), penConfig.getNeighbors()); + Assert.assertEquals("tcp_receiver_port ", 1234, penConfig.getTcpReceiverPort()); + Assert.assertEquals("udp_receiver_port ", 1234, penConfig.getUdpReceiverPort()); + Assert.assertEquals("max_peers ", 1234, penConfig.getMaxPeers()); + Assert.assertEquals("p_drop_cache ", 1.0, penConfig.getpDropCacheEntry(), 0d); + Assert.assertEquals("p_drop_transaction ", 1.0, penConfig.getpDropTransaction(), 0d); + Assert.assertEquals("p_propagate_request ", 1.0, penConfig.getpPropagateRequest(), 0d); + Assert.assertEquals("p_remove_request ", 1.0, penConfig.getpRemoveRequest(), 0d); + Assert.assertEquals("p_reply_random ", 1.0, penConfig.getpReplyRandomTip(), 0d); + Assert.assertEquals("p_select_milestone ", 1.0, penConfig.getpSelectMilestoneChild(), 0d); + Assert.assertEquals("p_send_milestone ",1.0, penConfig.getpSendMilestone(), 0d); + Assert.assertEquals("queue_size ", 1234, penConfig.getqSizeNode()); + Assert.assertEquals("dns_refresher ", false, penConfig.isDnsRefresherEnabled()); + Assert.assertEquals("dns_resolution ", false, penConfig.isDnsResolutionEnabled()); + Assert.assertEquals("send_limit ", 2, penConfig.getSendLimit()); + Assert.assertEquals("cache_size ", 1234, penConfig.getCacheSizeBytes()); + Assert.assertEquals("update_validator_delay ", 1234, penConfig.getUpdateValidatorDelay()); + Assert.assertEquals("validator ", true, penConfig.isValidator()); + Assert.assertEquals("validator_manager ", true, penConfig.getValidatorManagerEnabled()); + Assert.assertEquals("validator_path ", "./somewhere.txt", penConfig.getValidatorPath()); + Assert.assertEquals("start_validator ", 10, penConfig.getStartRoundDelay()); + Assert.assertEquals("round ", 60000, penConfig.getRoundDuration()); + Assert.assertEquals("round_pause ", 60000, penConfig.getRoundPause()); + Assert.assertEquals("max_analyzed_transactions ", 60000, penConfig.getBelowMaxDepthTransactionLimit()); + Assert.assertEquals("max_depth ", 10, penConfig.getMaxDepth()); + Assert.assertEquals("rescan ", true, penConfig.isRescanDb()); + Assert.assertEquals("revalidate ", true, penConfig.isRevalidate()); + Assert.assertEquals("remote_auth ", "pendulum:swings", penConfig.getRemoteAuth()); + Assert.assertEquals("alpha ", 1.0, penConfig.getAlpha(), 0d); + Assert.assertEquals("genesis ", 2345024001000L, penConfig.getGenesisTime()); + Assert.assertEquals("zmq_enable_ipc ", true, penConfig.isZmqEnableIpc()); + Assert.assertEquals("zmq_enable_tcp ", true, penConfig.isZmqEnableTcp()); + Assert.assertEquals("zmq_enabled ", true, penConfig.isZmqEnabled()); + Assert.assertEquals("zmq_ipc ", "ipc://pendulum", penConfig.getZmqIpc()); + Assert.assertEquals("zmq_port ", 1234, penConfig.getZmqPort()); + Assert.assertEquals("zmq_threads ", 10, penConfig.getZmqThreads()); + Assert.assertEquals("savelog_enabled ", true, penConfig.isSaveLogEnabled()); + Assert.assertEquals("savelog_path ", "./somewhere/", penConfig.getSaveLogBasePath()); + Assert.assertEquals("savelog_xml ", "/save.xml", penConfig.getSaveLogXMLFile()); + } + + /* + Test that iterates over common configs. It also attempts to check different types of types (double, boolean, string) + */ + //@Test public void argsParsingMainnetTest() { String[] args = { "-p", "8089", @@ -78,66 +249,37 @@ public void argsParsingMainnetTest() { }; PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); - Assert.assertThat("wrong config class created", pendulumConfig, CoreMatchers.instanceOf(MainnetConfig.class)); - pendulumConfig.parseConfigFromArgs(args); - Assert.assertEquals("port value", 8089, pendulumConfig.getApiPort()); - Assert.assertEquals("udp port", 4200, pendulumConfig.getUdpReceiverPort()); - Assert.assertEquals("tcp port", 5200, pendulumConfig.getTcpReceiverPort()); - Assert.assertEquals("neighbors", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); - Assert.assertEquals("api host", "1.1.1.1", pendulumConfig.getApiHost()); - Assert.assertEquals("ignored api endpoints", Arrays.asList("call1", "call2", "call3"), pendulumConfig.getIgnoredApiEndpoints()); - Assert.assertEquals("max find transactions", 500, pendulumConfig.getMaxFindTransactions()); - Assert.assertEquals("max requests list", 1000, pendulumConfig.getMaxRequestsList()); - Assert.assertEquals("max get bytes", 4000, pendulumConfig.getMaxTransactionStrings()); - Assert.assertEquals("max body length", 220, pendulumConfig.getMaxBodyLength()); - Assert.assertEquals("remote_auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); - Assert.assertEquals("p remove request", 0.23d, pendulumConfig.getpRemoveRequest(), 0d); - Assert.assertEquals("send limit", 1000, pendulumConfig.getSendLimit()); - Assert.assertEquals("max peers", 10, pendulumConfig.getMaxPeers()); - Assert.assertEquals("dns refresher", false, pendulumConfig.isDnsRefresherEnabled()); - Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); - Assert.assertEquals("XI-dir", "/XI", pendulumConfig.getXiDir()); - Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); - Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); - Assert.assertNotEquals("mwm", 4, pendulumConfig.getMwm()); - Assert.assertNotEquals("coo", pendulumConfig.getValidatorManagerAddress(), "TTTTTTTTT"); } - @Test - public void remoteFlagTest() { - String[] args = {"--remote"}; - PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); - pendulumConfig.parseConfigFromArgs(args); - Assert.assertEquals("The api interface should be open to the public", "0.0.0.0", pendulumConfig.getApiHost()); - } - @Test + + //@Test public void argsParsingTestnetTest() { String[] args = { "-p", "8089", @@ -198,7 +340,7 @@ public void argsParsingTestnetTest() { pendulumConfig.isValidateTestnetMilestoneSig()); } - @Test + //@Test public void iniParsingMainnetTest() throws Exception { String iniContent = new StringBuilder() .append("[PENDULUM]").append(System.lineSeparator()) @@ -225,7 +367,7 @@ public void iniParsingMainnetTest() throws Exception { Assert.assertNotEquals("MWM", 4, pendulumConfig.getMwm()); } - @Test + //@Test public void iniParsingTestnetTest() throws Exception { String iniContent = new StringBuilder() .append("[PENDULUM]").append(System.lineSeparator()) @@ -271,7 +413,7 @@ public void iniParsingTestnetTest() throws Exception { Assert.assertEquals("API_HOST", pendulumConfig.getApiHost(), "localhost"); } - @Test(expected = IllegalArgumentException.class) + //@Test(expected = IllegalArgumentException.class) public void testInvalidIni() throws IOException { String iniContent = new StringBuilder() .append("[PENDULUM]").append(System.lineSeparator()) @@ -298,7 +440,7 @@ public void backwardsIniCompatibilityTest() { Assert.assertThat(configNames, IsCollectionContaining.hasItem(config))); } - @Test + //@Test public void validateMilestoneSigDefaultValueTest() { PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(true); Assert.assertTrue("By default testnet should be validating milestones", From afce16c7fc8d216f920b81cc36dc265688909e50 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Thu, 28 Nov 2019 18:20:26 +0100 Subject: [PATCH 23/36] remove time check for milestones --- .../milestone/impl/MilestoneTrackerImpl.java | 50 ++++++++----------- 1 file changed, 22 insertions(+), 28 deletions(-) diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java index 99d5ac00..e43f6acd 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java @@ -303,38 +303,32 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw // - senderAddress must be part of validator addresses // - signature belongs to senderAddress // - index is bigger than snapshot index - // - attachment timestamp is in correct time window for the index + // - attachment timestamp is in correct time window for the index (removed) // - there doesn't already exist a milestone with the same address for that round - long calculatedRoundIndex = getRound(transaction.getAttachmentTimestamp()); - tracer.trace("ri, calculated ri= {}, {}", roundIndex, calculatedRoundIndex); - if (roundIndex == calculatedRoundIndex && isRoundActive(transaction.getAttachmentTimestamp())) { - tracer.trace("round is active"); - RoundViewModel currentRoundViewModel; - - // a milestone already arrived for that round, just update - if ((currentRoundViewModel = RoundViewModel.get(tangle, roundIndex)) != null) { - // check if there is already a milestone with the same address - if (RoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { - currentRoundViewModel.addMilestone(transaction.getHash()); - currentRoundViewModel.update(tangle); - tracer.trace("updated the round with mstn: {}", transaction.getHash()); - } + RoundViewModel currentRoundViewModel; + // a milestone already arrived for that round, just update + if ((currentRoundViewModel = RoundViewModel.get(tangle, roundIndex)) != null) { + // check if there is already a milestone with the same address + if (RoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { + currentRoundViewModel.addMilestone(transaction.getHash()); + currentRoundViewModel.update(tangle); + tracer.trace("updated the round with mstn: {}", transaction.getHash()); + } else { + tracer.trace("already exists a milestone from this validator: {}", transaction.getAddressHash()); } - // this is the first milestone for that round, make new database entry - else { - tracer.trace("current rvm is null"); - Set milestones = new HashSet<>(); - milestones.add(transaction.getHash()); - currentRoundViewModel = new RoundViewModel(roundIndex, milestones); - currentRoundViewModel.store(tangle); - } - addMilestoneToRoundLog(transaction.getHash(), roundIndex, currentRoundViewModel.size(), validators.size()); - setRoundIndexAndConfirmations(currentRoundViewModel, transaction, roundIndex); - publishMilestoneRefs(transaction); - } else { - tracer.trace("round is not active"); } + // this is the first milestone for that round, make new database entry + else { + tracer.trace("current rvm is null"); + Set milestones = new HashSet<>(); + milestones.add(transaction.getHash()); + currentRoundViewModel = new RoundViewModel(roundIndex, milestones); + currentRoundViewModel.store(tangle); + } + addMilestoneToRoundLog(transaction.getHash(), roundIndex, currentRoundViewModel.size(), validators.size()); + setRoundIndexAndConfirmations(currentRoundViewModel, transaction, roundIndex); + publishMilestoneRefs(transaction); if (!transaction.isSolid()) { tracer.trace("non solid: {}", transaction.getHash()); From 27a981d55836f9f21c9429a329440e42de9b6271 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Thu, 28 Nov 2019 21:11:04 +0300 Subject: [PATCH 24/36] log if merkle root mismatches tips confirmed by a milestone branch --- .../java/net/helix/pendulum/controllers/RoundViewModel.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 8d72113d..353cc007 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -12,7 +12,6 @@ import net.helix.pendulum.storage.Persistable; import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Pair; -import net.helix.pendulum.utils.PendulumUtils; import net.helix.pendulum.utils.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -256,12 +255,15 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr } else { Set prevMilestones = prevMilestone.getHashes(); List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + Hash root = merkleTree.get(merkleTree.size() - 1).get(0); + if (transaction.getBranchTransactionHash().equals(root)) { if (prevMilestones.isEmpty()) { trunk.add(Hash.NULL_HASH); } else { trunk.addAll(prevMilestones); } + } else { + log.debug("Branch does not match merkle root {} {}", transaction.getBranchTransactionHash(), root); } } } From db6aa8d0268728955e3486ff770ab6e716b5a0ea Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Fri, 29 Nov 2019 11:19:18 +0100 Subject: [PATCH 25/36] Uncomment all config tests and make explicit the renaming and removal of legacy config params --- .../net/helix/pendulum/conf/ConfigTest.java | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/src/test/java/net/helix/pendulum/conf/ConfigTest.java b/src/test/java/net/helix/pendulum/conf/ConfigTest.java index bb156c59..2d23d2f0 100644 --- a/src/test/java/net/helix/pendulum/conf/ConfigTest.java +++ b/src/test/java/net/helix/pendulum/conf/ConfigTest.java @@ -48,7 +48,7 @@ public static void tearDownAfterClass() throws IOException { /* test that --remote returns true without specifically giving true on cmd line */ - //@Test + @Test public void remoteFlagTest() { String[] args = {"--remote"}; PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); @@ -219,7 +219,7 @@ public void testCommandLineArgs(){ /* Test that iterates over common configs. It also attempts to check different types of types (double, boolean, string) */ - //@Test + @Test public void argsParsingMainnetTest() { String[] args = { "-p", "8089", @@ -238,7 +238,7 @@ public void argsParsingMainnetTest() { "--max_peers", "10", "--dns_refresher", "false", "--dns_resolution", "false", - "--XI_dir", "/XI", + "--xi_dir", "/XI", "--db_path", "/db", "--db_log_path", "/dblog", "--zmq_enabled", "true", @@ -270,7 +270,7 @@ public void argsParsingMainnetTest() { Assert.assertEquals("max peers", 10, pendulumConfig.getMaxPeers()); Assert.assertEquals("dns refresher", false, pendulumConfig.isDnsRefresherEnabled()); Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); - Assert.assertEquals("XI-dir", "/XI", pendulumConfig.getXiDir()); + Assert.assertEquals("xi-dir", "/XI", pendulumConfig.getXiDir()); Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); Assert.assertNotEquals("mwm", 4, pendulumConfig.getMwm()); @@ -279,7 +279,7 @@ public void argsParsingMainnetTest() { - //@Test + @Test public void argsParsingTestnetTest() { String[] args = { "-p", "8089", @@ -298,7 +298,7 @@ public void argsParsingTestnetTest() { "--max_peers", "10", "--dns_refresher", "false", "--dns_resolution", "false", - "--XI_dir", "/XI", + "--xi_dir", "/XI", "--db_path", "/db", "--db_log_path", "/dblog", "--zmq_enabled", "true", @@ -331,7 +331,7 @@ public void argsParsingTestnetTest() { Assert.assertEquals("max peers", 10, pendulumConfig.getMaxPeers()); Assert.assertEquals("dns refresher", false, pendulumConfig.isDnsRefresherEnabled()); Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); - Assert.assertEquals("XI_dir", "/XI", pendulumConfig.getXiDir()); + Assert.assertEquals("xi_dir", "/XI", pendulumConfig.getXiDir()); Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); Assert.assertEquals("mwm", 4, pendulumConfig.getMwm()); @@ -340,7 +340,7 @@ public void argsParsingTestnetTest() { pendulumConfig.isValidateTestnetMilestoneSig()); } - //@Test + @Test public void iniParsingMainnetTest() throws Exception { String iniContent = new StringBuilder() .append("[PENDULUM]").append(System.lineSeparator()) @@ -367,7 +367,7 @@ public void iniParsingMainnetTest() throws Exception { Assert.assertNotEquals("MWM", 4, pendulumConfig.getMwm()); } - //@Test + @Test public void iniParsingTestnetTest() throws Exception { String iniContent = new StringBuilder() .append("[PENDULUM]").append(System.lineSeparator()) @@ -413,7 +413,7 @@ public void iniParsingTestnetTest() throws Exception { Assert.assertEquals("API_HOST", pendulumConfig.getApiHost(), "localhost"); } - //@Test(expected = IllegalArgumentException.class) + @Test(expected = IllegalArgumentException.class) public void testInvalidIni() throws IOException { String iniContent = new StringBuilder() .append("[PENDULUM]").append(System.lineSeparator()) @@ -425,22 +425,26 @@ public void testInvalidIni() throws IOException { ConfigFactory.createFromFile(configFile, false); } - //@Test + @Test public void backwardsIniCompatibilityTest() { Collection configNames = PendulumUtils.getAllSetters(TestnetConfig.class) .stream() .map(this::deriveNameFromSetter) .collect(Collectors.toList()); + Stream.of(LegacyDefaultConf.values()) .map(Enum::name) // make it explicit that we have removed some configs + // in some cases, we have renamed the config param (to e.g. fix double negative variable names) .filter(config -> !ArrayUtils.contains(new String[]{"CONFIG", "TESTNET", "DEBUG", - "MIN_RANDOM_WALKS", "MAX_RANDOM_WALKS"}, config)) + "MIN_RANDOM_WALKS", "MAX_RANDOM_WALKS", "DONT_VALIDATE_TESTNET_MILESTONE_SIG", + "COORDINATOR", "REMOTE_LIMIT_API", "MWM", "TIPSELECTION_ALPHA"}, config)) .forEach(config -> - Assert.assertThat(configNames, IsCollectionContaining.hasItem(config))); + Assert.assertThat(configNames, IsCollectionContaining.hasItem(config)) + ); } - //@Test + @Test public void validateMilestoneSigDefaultValueTest() { PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(true); Assert.assertTrue("By default testnet should be validating milestones", @@ -473,7 +477,7 @@ public enum LegacyDefaultConf { TCP_RECEIVER_PORT, TESTNET, DEBUG, - IGNORED_API_ENDPOINTS, + REMOTE_LIMIT_API, REMOTE_AUTH, NEIGHBORS, XI_DIR, @@ -492,7 +496,7 @@ public enum LegacyDefaultConf { DNS_RESOLUTION_ENABLED, DNS_REFRESHER_ENABLED, COORDINATOR, - VALIDATE_TESTNET_MILESTONE_SIG, + DONT_VALIDATE_TESTNET_MILESTONE_SIG, REVALIDATE, RESCAN_DB, MIN_RANDOM_WALKS, @@ -517,7 +521,7 @@ public enum LegacyDefaultConf { TRANSACTION_PACKET_SIZE, REQUEST_HASH_SIZE, SNAPSHOT_TIME, - ALPHA, + TIPSELECTION_ALPHA, BELOW_MAX_DEPTH_TRANSACTION_LIMIT } } From 90297135b522a7940fafa17920c0c0b6eb0bec53 Mon Sep 17 00:00:00 2001 From: spangin <> Date: Sat, 30 Nov 2019 22:59:26 +0300 Subject: [PATCH 26/36] fix db lock in SpentAddressesProviderImpl ('dev' branch) --- .../java/net/helix/pendulum/Pendulum.java | 1 + .../impl/SpentAddressesProviderImpl.java | 30 +++++++++++-------- .../pendulum/service/APIIntegrationTest.java | 5 ---- .../impl/SpentAddressesProviderImplTest.java | 4 +-- 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/src/main/java/net/helix/pendulum/Pendulum.java b/src/main/java/net/helix/pendulum/Pendulum.java index bd1cb26b..ba93e64f 100644 --- a/src/main/java/net/helix/pendulum/Pendulum.java +++ b/src/main/java/net/helix/pendulum/Pendulum.java @@ -294,6 +294,7 @@ public void shutdown() throws Exception { transactionValidator.shutdown(); tangle.shutdown(); + spentAddressesProvider.shutdown(); // free the resources of the snapshot provider last because all other instances need it snapshotProvider.shutdown(); } diff --git a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 8f6c090e..dd0327ba 100644 --- a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -14,7 +14,6 @@ import net.helix.pendulum.utils.Pair; import java.io.BufferedReader; -import java.io.File; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Collection; @@ -31,8 +30,7 @@ */ public class SpentAddressesProviderImpl implements SpentAddressesProvider { - //@VisibleForTesting - public RocksDBPersistenceProvider rocksDBPersistenceProvider; + private RocksDBPersistenceProvider rocksDBPersistenceProvider; private SnapshotConfig config; @@ -40,13 +38,6 @@ public class SpentAddressesProviderImpl implements SpentAddressesProvider { * Creates a new instance of SpentAddressesProvider */ public SpentAddressesProviderImpl() { - Map> columnFamilies = new HashMap<>(); - columnFamilies.put("spent-addresses", SpentAddress.class); - String addressDBPath = System.getProperty("spent.addresses.db", ""); - this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider( - addressDBPath + SPENT_ADDRESSES_DB, - addressDBPath + SPENT_ADDRESSES_LOG, 1000, - columnFamilies, null); } /** @@ -56,11 +47,18 @@ public SpentAddressesProviderImpl() { * @return the current instance * @throws SpentAddressesException if we failed to create a file at the designated location */ - public SpentAddressesProviderImpl init(SnapshotConfig config) - throws SpentAddressesException { + public SpentAddressesProviderImpl init(SnapshotConfig config) throws SpentAddressesException { + Map> columnFamilies = new HashMap<>(); + columnFamilies.put("spent-addresses", SpentAddress.class); + String addressDBPath = System.getProperty("spent.addresses.db", ""); + rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + addressDBPath + SPENT_ADDRESSES_DB, + addressDBPath + SPENT_ADDRESSES_LOG, 1000, + columnFamilies, null); + this.config = config; try { - this.rocksDBPersistenceProvider.init(); + rocksDBPersistenceProvider.init(); readPreviousEpochsSpentAddresses(); } catch (Exception e) { @@ -68,6 +66,12 @@ public SpentAddressesProviderImpl init(SnapshotConfig config) } return this; } + + public void shutdown() { + if (rocksDBPersistenceProvider != null) { + rocksDBPersistenceProvider.shutdown(); + } + } private void readPreviousEpochsSpentAddresses() throws SpentAddressesException { if (config.isTestnet()) { diff --git a/src/test/java/net/helix/pendulum/service/APIIntegrationTest.java b/src/test/java/net/helix/pendulum/service/APIIntegrationTest.java index c88308e8..b0a7c210 100644 --- a/src/test/java/net/helix/pendulum/service/APIIntegrationTest.java +++ b/src/test/java/net/helix/pendulum/service/APIIntegrationTest.java @@ -117,11 +117,6 @@ public static void shutdown() { Xi.shutdown(); api.shutDown(); pendulum.shutdown(); - // shutdown spentAddressesProvider.rocksDBPersistenceProvider to avoid an exception in other unit-tests - if (pendulum.spentAddressesProvider != null - && pendulum.spentAddressesProvider.rocksDBPersistenceProvider != null) { - pendulum.spentAddressesProvider.rocksDBPersistenceProvider.shutdown(); - } } catch (final Exception e) { log.error("Exception occurred shutting down Pendulum node: ", e); fail("Exception occurred shutting down Pendulum node"); diff --git a/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java b/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java index ec358342..759fa4f9 100644 --- a/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java +++ b/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java @@ -28,9 +28,7 @@ public static void setUp() throws Exception { @AfterClass public static void shutdown() { - if (provider.rocksDBPersistenceProvider != null) { - provider.rocksDBPersistenceProvider.shutdown(); - } + provider.shutdown(); } @Test From 8ab4602f44679814df6cf3101ac5782e15fef7e6 Mon Sep 17 00:00:00 2001 From: spangin <> Date: Sat, 30 Nov 2019 23:04:47 +0300 Subject: [PATCH 27/36] fix getting instance of ledgerService --- src/main/java/net/helix/pendulum/Pendulum.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/helix/pendulum/Pendulum.java b/src/main/java/net/helix/pendulum/Pendulum.java index ba93e64f..f6498e06 100644 --- a/src/main/java/net/helix/pendulum/Pendulum.java +++ b/src/main/java/net/helix/pendulum/Pendulum.java @@ -100,7 +100,7 @@ public class Pendulum { public final CandidateTrackerImpl candidateTracker; public final LatestSolidMilestoneTrackerImpl latestSolidMilestoneTracker; public final SeenMilestonesRetrieverImpl seenMilestonesRetriever; - public final LedgerServiceImpl ledgerService = new LedgerServiceImpl(); + public final LedgerServiceImpl ledgerService; public final AsyncTransactionPruner transactionPruner; public final MilestoneSolidifierImpl milestoneSolidifier; public final CandidateSolidifierImpl candidateSolidifier; @@ -150,6 +150,8 @@ public Pendulum(PendulumConfig configuration) throws TransactionPruningException : null; transactionRequesterWorker = new TransactionRequesterWorkerImpl(); + ledgerService = new LedgerServiceImpl(); + // legacy code bundleValidator = new BundleValidator(); tangle = new Tangle(); From bdf591164b92abc5dd16adf9604b980099ba53b1 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Mon, 2 Dec 2019 10:57:15 +0100 Subject: [PATCH 28/36] comment test --- src/main/java/net/helix/pendulum/service/API.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 6f35bd86..8d3f8cc1 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -1677,7 +1677,7 @@ private List addMilestoneReferences(List confirmedTips, int roundInd txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones } //branch - Collections.sort(confirmedTips); + Collections.sort(confirmedTips); // sort tips before building merkle root List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips } From 9ee932d3a435a3fc520cc307565a815b8d50bc56 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Tue, 3 Dec 2019 14:15:44 +0300 Subject: [PATCH 29/36] roundDuration should be read off from config, not defaults --- .../pendulum/service/milestone/impl/MilestoneTrackerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java index 99d5ac00..dec0c9a8 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java @@ -230,8 +230,8 @@ public int getCurrentRoundIndex() { @Override public int getRound(long time) { return config.isTestnet() ? - RoundIndexUtil.getRound(time, TestnetConfig.Defaults.GENESIS_TIME, BasePendulumConfig.Defaults.ROUND_DURATION) : - RoundIndexUtil.getRound(time, BasePendulumConfig.Defaults.GENESIS_TIME, BasePendulumConfig.Defaults.ROUND_DURATION); + RoundIndexUtil.getRound(time, TestnetConfig.Defaults.GENESIS_TIME, roundDuration) : + RoundIndexUtil.getRound(time, BasePendulumConfig.Defaults.GENESIS_TIME, roundDuration); } @Override From ff80e209ca320f7ed6e77a40c271398d49e40aaf Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Tue, 3 Dec 2019 19:37:37 +0300 Subject: [PATCH 30/36] refactor BundleValidator.validate --- .../net/helix/pendulum/BundleValidator.java | 406 ++++++++++++------ 1 file changed, 286 insertions(+), 120 deletions(-) diff --git a/src/main/java/net/helix/pendulum/BundleValidator.java b/src/main/java/net/helix/pendulum/BundleValidator.java index f5e26430..6bb560e3 100644 --- a/src/main/java/net/helix/pendulum/BundleValidator.java +++ b/src/main/java/net/helix/pendulum/BundleValidator.java @@ -6,15 +6,14 @@ import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; import net.helix.pendulum.model.Hash; +import net.helix.pendulum.service.ValidationException; import net.helix.pendulum.service.snapshot.Snapshot; import net.helix.pendulum.storage.Tangle; +import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Validates bundles. @@ -26,6 +25,8 @@ */ public class BundleValidator { + private final static Logger log = LoggerFactory.getLogger(BundleValidator.class); + /** * Fetches a bundle of transactions identified by the {@code tailHash} and validates the transactions. * Bundle is a group of transactions with the same bundle hash chained by their trunks. @@ -65,129 +66,152 @@ public class BundleValidator { public static List> validate(Tangle tangle, Snapshot initialSnapshot, Hash tailHash) throws Exception { TransactionViewModel tail = TransactionViewModel.fromHash(tangle, tailHash); - if (tail.getCurrentIndex() != 0 || tail.getValidity() == -1) { - return Collections.EMPTY_LIST; + if (tail.getCurrentIndex() != 0) { + log.trace("{} is not a tail", tail); + return Collections.emptyList(); + } + + if (tail.getValidity() == -1) { + log.trace("{} is not valid", tail); + return Collections.emptyList(); } List> transactions = new LinkedList<>(); final Map bundleTransactions = loadTransactionsFromTangle(tangle, tail); - //we don't really iterate, we just pick the tail tx. See the if on the next line - for (TransactionViewModel transactionViewModel : bundleTransactions.values()) { - - if (transactionViewModel.getCurrentIndex() == 0 && transactionViewModel.getValidity() >= 0) { - - final List instanceTransactionViewModels = new LinkedList<>(); - - final long lastIndex = transactionViewModel.lastIndex(); - long bundleValue = 0; - int i = 0; - final Sponge sha3Instance = SpongeFactory.create(SpongeFactory.Mode.S256); - final Sponge addressInstance = SpongeFactory.create(SpongeFactory.Mode.S256); - - final byte[] addressBytes = new byte[TransactionViewModel.ADDRESS_SIZE]; - final byte[] bundleHashBytes = new byte[TransactionViewModel.BUNDLE_SIZE]; - byte[] normalizedBundle = new byte[TransactionViewModel.BUNDLE_SIZE]; - byte[] digestBytes = new byte[Sha3.HASH_LENGTH]; - - //here we iterate over the txs by checking the trunk of the current transaction - MAIN_LOOP: - while (true) { - - instanceTransactionViewModels.add(transactionViewModel); - - //semantic checks - if ( - transactionViewModel.getCurrentIndex() != i - || transactionViewModel.lastIndex() != lastIndex - || ((bundleValue = Math.addExact(bundleValue, transactionViewModel.value())) < -TransactionViewModel.SUPPLY - || bundleValue > TransactionViewModel.SUPPLY) - ) { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - break; - } - - // It's supposed to become -3812798742493 after 3812798742493 and to go "down" to -1 but - // we hope that no one will create such long bundles - if (i++ == lastIndex) { - - if (bundleValue == 0) { - - if (instanceTransactionViewModels.get(0).getValidity() == 0) { - sha3Instance.reset(); - for (final TransactionViewModel transactionViewModel2 : instanceTransactionViewModels) { - sha3Instance.absorb(transactionViewModel2.getBytes(), TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_SIZE); - } - sha3Instance.squeeze(bundleHashBytes, 0, bundleHashBytes.length); - //verify bundle hash is correct - if (Arrays.equals(instanceTransactionViewModels.get(0).getBundleHash().bytes(), bundleHashBytes)) { - //normalizing the bundle in preparation for signature verification - normalizedBundle = Winternitz.normalizedBundle(bundleHashBytes); - - int offset = 0; - for (int j = 0; j < instanceTransactionViewModels.size(); ) { - - transactionViewModel = instanceTransactionViewModels.get(j); - //if it is a spent transaction that should be signed - if (transactionViewModel.value() < 0) { - // let's verify the signature by recalculating the public address - addressInstance.reset(); - do { - digestBytes = Winternitz.digest(SpongeFactory.Mode.S256, Arrays.copyOfRange(normalizedBundle, offset*Winternitz.NORMALIZED_FRAGMENT_LENGTH, (offset+1)*Winternitz.NORMALIZED_FRAGMENT_LENGTH), - Arrays.copyOfRange(instanceTransactionViewModels.get(j).getBytes(), 0, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE)); - addressInstance.absorb(digestBytes,0, Sha3.HASH_LENGTH); - offset++; - } //loop to traverse signature fragments divided between transactions - while (++j < instanceTransactionViewModels.size() - && instanceTransactionViewModels.get(j).getAddressHash().equals(transactionViewModel.getAddressHash()) - && instanceTransactionViewModels.get(j).value() == 0); - - addressInstance.squeeze(addressBytes, 0, addressBytes.length); - //signature verification - if (! Arrays.equals(transactionViewModel.getAddressHash().bytes(), addressBytes)) { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - break MAIN_LOOP; - } - } else { - j++; - } - } - //should only be reached after the above for loop is done - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, 1); - transactions.add(instanceTransactionViewModels); - } - //bundle hash verification failed - else { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - } - } - //bundle validity status is known - else { - transactions.add(instanceTransactionViewModels); - } - } - //total bundle value does not sum to 0 - else { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - } - //break from main loop - break; - - } - //traverse to the next tx in the bundle - else { - transactionViewModel = bundleTransactions.get(transactionViewModel.getTrunkTransactionHash()); - if (transactionViewModel == null) { - //we found all the transactions and we can now return - break; - } - } - } - } + LinkedList sortedTxs; + try { + sortedTxs = validateOrder(bundleTransactions.values()); + validateValue(sortedTxs); + validateBundleHash(sortedTxs); + validateSignatures(sortedTxs); + } catch (ValidationException ve) { + tail.setValidity(tangle, initialSnapshot, -1); + log.warn("Bundle validation exception", ve); + return Collections.emptyList(); } + + tail.setValidity(tangle, initialSnapshot, 1); + transactions.add(sortedTxs); + + //we don't really iterate, we just pick the tail tx. See the if on the next line +// for (TransactionViewModel transactionViewModel : bundleTransactions.values()) { +// +// if (transactionViewModel.getCurrentIndex() == 0 && transactionViewModel.getValidity() >= 0) { +// +// final List instanceTransactionViewModels = new LinkedList<>(); +// +// final long lastIndex = tail.lastIndex(); +// long bundleValue = 0; +// int i = 0; +// final Sponge sha3Instance = SpongeFactory.create(SpongeFactory.Mode.S256); +// final Sponge addressInstance = SpongeFactory.create(SpongeFactory.Mode.S256); +// +// final byte[] addressBytes = new byte[TransactionViewModel.ADDRESS_SIZE]; +// final byte[] bundleHashBytes = new byte[TransactionViewModel.BUNDLE_SIZE]; +// byte[] normalizedBundle = new byte[TransactionViewModel.BUNDLE_SIZE]; +// byte[] digestBytes = new byte[Sha3.HASH_LENGTH]; +// +// //here we iterate over the txs by checking the trunk of the current transaction +// MAIN_LOOP: +// while (true) { +// +// instanceTransactionViewModels.add(transactionViewModel); +// +// //semantic checks +// if ( +// transactionViewModel.getCurrentIndex() != i +// || transactionViewModel.lastIndex() != lastIndex +// || ((bundleValue = Math.addExact(bundleValue, transactionViewModel.value())) < -TransactionViewModel.SUPPLY +// || bundleValue > TransactionViewModel.SUPPLY) +// ) { +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); +// break; +// } +// +// // It's supposed to become -3812798742493 after 3812798742493 and to go "down" to -1 but +// // we hope that no one will create such long bundles +// if (i++ == lastIndex) { +// +// if (bundleValue == 0) { +// +// if (instanceTransactionViewModels.get(0).getValidity() == 0) { +// sha3Instance.reset(); +// for (final TransactionViewModel transactionViewModel2 : instanceTransactionViewModels) { +// sha3Instance.absorb(transactionViewModel2.getBytes(), TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_SIZE); +// } +// sha3Instance.squeeze(bundleHashBytes, 0, bundleHashBytes.length); +// //verify bundle hash is correct +// if (Arrays.equals(instanceTransactionViewModels.get(0).getBundleHash().bytes(), bundleHashBytes)) { +// //normalizing the bundle in preparation for signature verification +// normalizedBundle = Winternitz.normalizedBundle(bundleHashBytes); +// +// int offset = 0; +// for (int j = 0; j < instanceTransactionViewModels.size(); ) { +// +// transactionViewModel = instanceTransactionViewModels.get(j); +// //if it is a spent transaction that should be signed +// if (transactionViewModel.value() < 0) { +// // let's verify the signature by recalculating the public address +// addressInstance.reset(); +// do { +// digestBytes = Winternitz.digest(SpongeFactory.Mode.S256, Arrays.copyOfRange(normalizedBundle, offset*Winternitz.NORMALIZED_FRAGMENT_LENGTH, (offset+1)*Winternitz.NORMALIZED_FRAGMENT_LENGTH), +// Arrays.copyOfRange(instanceTransactionViewModels.get(j).getBytes(), 0, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE)); +// addressInstance.absorb(digestBytes,0, Sha3.HASH_LENGTH); +// offset++; +// } //loop to traverse signature fragments divided between transactions +// while (++j < instanceTransactionViewModels.size() +// && instanceTransactionViewModels.get(j).getAddressHash().equals(transactionViewModel.getAddressHash()) +// && instanceTransactionViewModels.get(j).value() == 0); +// +// addressInstance.squeeze(addressBytes, 0, addressBytes.length); +// //signature verification +// if (! Arrays.equals(transactionViewModel.getAddressHash().bytes(), addressBytes)) { +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); +// break MAIN_LOOP; +// } +// } else { +// j++; +// } +// } +// //should only be reached after the above for loop is done +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, 1); +// transactions.add(instanceTransactionViewModels); +// } +// //bundle hash verification failed +// else { +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); +// } +// } +// //bundle validity status is known +// else { +// transactions.add(instanceTransactionViewModels); +// } +// } +// //total bundle value does not sum to 0 +// else { +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); +// } +// //break from main loop +// break; +// +// } +// //traverse to the next tx in the bundle +// else { +// transactionViewModel = bundleTransactions.get(transactionViewModel.getTrunkTransactionHash()); +// if (transactionViewModel == null) { +// //we found all the transactions and we can now return +// break; +// } +// } +// } +// } +// } return transactions; } + + /** * Checks that the bundle's inputs and outputs are balanced. * @@ -233,4 +257,146 @@ private static Map loadTransactionsFromTangle(Tangle } return bundleTransactions; } + + /** + * Sorts the provided list according to the index + * + * @param bundleTxs unordered list of transactions + * @return List of TransactionViewModel in the bundle, from starting from currentIndex == 0 to + * currentIndex == tail.lastIndex() + * + * @throws ValidationException if there is a missing index or the size does not match the last index + */ + private static LinkedList validateOrder(Collection bundleTxs) { + SortedSet set = new TreeSet<>(Comparator.comparing(TransactionViewModel::getCurrentIndex)); + set.addAll(bundleTxs); + LinkedList sorted = new LinkedList<>(set); + int index = 0; + TransactionViewModel lastTx = sorted.getFirst(); + for (TransactionViewModel txvm : sorted) { + if (txvm.getCurrentIndex() != index) { + throw new ValidationException(String.format("%s does not match index %d", txvm.toString(), index)); + } + if (index > 0 && lastTx.getTrunkTransactionHash() != txvm.getHash()) { + throw new ValidationException(String.format("Trunk of %s should be equal to %s", + lastTx.toString(), txvm.getHash())); + } + lastTx = txvm; + index ++; + } + long expectedSize = sorted.getFirst().lastIndex() + 1; + if (sorted.size() != expectedSize) { + throw new ValidationException(String.format("The bundle is incomplete. Expected bundle size: %d", expectedSize)); + } + return sorted; + } + + /** + * + * @param bundleTxs Ordered list of the transaction bundles + * + * @throws ValidationException if the value transfers are out of bounds or the bundle value is inconsistent + * (i.e. cummulative value transfers don't add up to zero) + */ + private static void validateValue(LinkedList bundleTxs) { + long bundleValue = 0; + for (TransactionViewModel txvm : bundleTxs) { + bundleValue = Math.addExact(bundleValue, txvm.value()); + if (bundleValue < -TransactionViewModel.SUPPLY) { + throw new ValidationException("Bundle value is below the negative max supply"); + } + if (bundleValue > TransactionViewModel.SUPPLY) { + throw new ValidationException("Bundle value is above the max supply"); + } + } + + if (bundleValue != 0) { + throw new ValidationException("Bundle transaction values are inconsistent"); + } + } + + /** + * + * @param bundleTxs Ordered list of the bundle transactions + * + * @throws ValidationException If the bundle hash as stored in the tail does not match the actual bundle hash + * as stored in the essense part of the bundle transactions + */ + private static void validateBundleHash(LinkedList bundleTxs) { + if (bundleTxs == null || bundleTxs.size() == 0) { + throw new ValidationException("Bundle txs list is empty"); + } + + Sponge sha3Instance = SpongeFactory.create(SpongeFactory.Mode.S256); + byte[] bundleHashBytes = new byte[TransactionViewModel.BUNDLE_SIZE]; + sha3Instance.reset(); + for (final TransactionViewModel txvm : bundleTxs) { + sha3Instance.absorb(txvm.getBytes(), TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_SIZE); + } + sha3Instance.squeeze(bundleHashBytes, 0, bundleHashBytes.length); + TransactionViewModel tail = bundleTxs.getFirst(); + Hash bundleHash = tail.getBundleHash(); + if (bundleHash == null) { + throw new ValidationException("Bundle hash of the tail tx is null: " + tail.toString()); + } + + if (!Arrays.equals(bundleHash.bytes(), bundleHashBytes)) { + throw new ValidationException(String.format("Bundle hash %s does not match calculated hash %s", + Hex.toHexString(bundleHash.bytes()), Hex.toHexString(bundleHashBytes))); + } + } + + /** + * After a spendig txs, a few zero-value txs bearing the signature should follow. + * @param bundleTxs Ordered list of the bundle transactions + * + * @throws ValidationException If the address hashes of the spending transactions + * do not match the public address (Winternitz OTS) + */ + private static void validateSignatures(LinkedList bundleTxs) { + TransactionViewModel tail = bundleTxs.getFirst(); + Hash bundleHash = tail.getBundleHash(); + if (bundleHash == null) { + throw new ValidationException("Bundle hash of the tail tx is null: " + tail.toString()); + } + byte[] bundleHashBytes = bundleHash.bytes(); + final Sponge addressInstance = SpongeFactory.create(SpongeFactory.Mode.S256); + final byte[] addressBytes = new byte[TransactionViewModel.ADDRESS_SIZE]; + byte[] normalizedBundle = Winternitz.normalizedBundle(bundleHashBytes); + int offset = 0; + + ArrayList bundleArray = new ArrayList<>(bundleTxs); + + for (int j = 0; j < bundleArray.size(); ) { + + TransactionViewModel transactionViewModel = bundleArray.get(j); + if (transactionViewModel.value() >= 0) { + j++; + log.trace("{} has non-negative value {}", transactionViewModel.getHash(), transactionViewModel.value()); + continue; + } + //if it is a spent transaction that should be signed + // let's verify the signature by recalculating the public address + addressInstance.reset(); + do { + byte[] digestBytes = Winternitz.digest(SpongeFactory.Mode.S256, + Arrays.copyOfRange(normalizedBundle, offset*Winternitz.NORMALIZED_FRAGMENT_LENGTH, (offset+1)*Winternitz.NORMALIZED_FRAGMENT_LENGTH), + Arrays.copyOfRange(bundleArray.get(j).getBytes(), 0, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE)); + addressInstance.absorb(digestBytes,0, Sha3.HASH_LENGTH); + offset++; + } //loop to traverse signature fragments divided between transactions + while (++j < bundleArray.size() + && bundleArray.get(j).getAddressHash().equals(transactionViewModel.getAddressHash()) + && bundleArray.get(j).value() == 0); + + addressInstance.squeeze(addressBytes, 0, addressBytes.length); + //signature verification + if (! Arrays.equals(transactionViewModel.getAddressHash().bytes(), addressBytes)) { + throw new ValidationException( + String.format("Signature verification failed: %s does not match %s", + Hex.toHexString(transactionViewModel.getAddressHash().bytes()), Hex.toHexString(addressBytes))); + } + + } + } } From 86bc83fc6752a151eb7e53d8cc51de4af4696a81 Mon Sep 17 00:00:00 2001 From: fsbbn Date: Wed, 4 Dec 2019 12:44:47 +0100 Subject: [PATCH 31/36] add trace statements and and fix calling branch instead of trunk --- .../pendulum/controllers/RoundViewModel.java | 31 ++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index f61db623..dbca8fcf 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -250,20 +250,29 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr // add previous milestones to non analyzed transactions RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); if (prevMilestone == null) { - if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { + log.trace("prev milestone is null"); + if (transaction.getTrunkTransactionHash().equals(Hash.NULL_HASH)) { + log.trace("prev milestone = trunk = null"); trunk.add(Hash.NULL_HASH); + } else { + log.trace("trunk is not null: {}", transaction.getTrunkTransactionHash()); } } else { Set prevMilestones = prevMilestone.getHashes(); List prevMilestonesList = new ArrayList<>(prevMilestones); Collections.sort(prevMilestonesList); List> merkleTree = Merkle.buildMerkleTree(prevMilestonesList); - if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + Hash merkleRoot = merkleTree.get(merkleTree.size() - 1).get(0); + if (transaction.getTrunkTransactionHash().equals(merkleRoot)) { + log.trace("merkle root = trunk"); if (prevMilestones.isEmpty()) { + log.trace("prev round was empty"); trunk.add(Hash.NULL_HASH); } else { trunk.addAll(prevMilestones); } + } else { + log.trace("merkle root ({}) != trunk ({})", merkleRoot, transaction.getTrunkTransactionHash()); } } } @@ -284,32 +293,46 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t List confirmedTipsList = new ArrayList<>(confirmedTips); Collections.sort(confirmedTipsList); List> merkleTree = Merkle.buildMerkleTree(confirmedTipsList); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { + Hash merkleRoot = merkleTree.get(merkleTree.size()-1).get(0); + if (transaction.getBranchTransactionHash().equals(merkleRoot)) { + log.trace("merkle root = branch"); if (confirmedTips.isEmpty()){ + log.trace("confirmed tips = null"); branch.add(Hash.NULL_HASH); } else { branch.addAll(confirmedTips); } + } else { + log.trace("merkle root ({}) != branch ({})", merkleRoot, transaction.getBranchTransactionHash()); } } else { // add previous milestones to non analyzed transactions RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); if (prevMilestone == null) { + log.trace("prev milestone is null"); if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { + log.trace("prev milestone = branch = null"); branch.add(Hash.NULL_HASH); + } else { + log.trace("branch is not null: {}", transaction.getBranchTransactionHash()); } } else { Set prevMilestones = prevMilestone.getHashes(); List prevMilestonesList = new ArrayList<>(prevMilestones); Collections.sort(prevMilestonesList); List> merkleTree = Merkle.buildMerkleTree(prevMilestonesList); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + Hash merkleRoot = merkleTree.get(merkleTree.size() - 1).get(0); + if (transaction.getBranchTransactionHash().equals(merkleRoot)) { + log.trace("merkle root = branch"); if (prevMilestones.isEmpty()) { + log.trace("prev round was empty"); branch.add(Hash.NULL_HASH); } else { branch.addAll(prevMilestones); } + } else { + log.trace("merkle root ({}) != branch ({})", merkleRoot, transaction.getBranchTransactionHash()); } } } From 5a60783eea98931f042c26436bf3919b14bc8847 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Wed, 4 Dec 2019 13:46:30 +0100 Subject: [PATCH 32/36] Distinguish between db and db-log paths for spent-addresses in mainnet vs. testnet --- .gitignore | 13 +- .../pendulum/conf/BasePendulumConfig.java | 414 ++++++++++-------- .../java/net/helix/pendulum/conf/Config.java | 4 + .../helix/pendulum/conf/ConsensusConfig.java | 9 +- .../net/helix/pendulum/conf/GraphConfig.java | 9 - .../helix/pendulum/conf/MainnetConfig.java | 1 + .../helix/pendulum/conf/MilestoneConfig.java | 65 +-- .../helix/pendulum/conf/PendulumConfig.java | 21 +- .../helix/pendulum/conf/ProtocolConfig.java | 1 + .../net/helix/pendulum/conf/RoundConfig.java | 23 + .../helix/pendulum/conf/SnapshotConfig.java | 16 +- .../pendulum/conf/SpentAddressesConfig.java | 35 ++ .../helix/pendulum/conf/TestnetConfig.java | 37 +- .../helix/pendulum/conf/ValidatorConfig.java | 55 +++ .../pendulum/conf/ValidatorManagerConfig.java | 5 - .../net/helix/pendulum/conf/XIConfig.java | 3 + .../net/helix/pendulum/conf/ZMQConfig.java | 3 +- .../snapshot/impl/SnapshotProviderImpl.java | 9 +- .../impl/SpentAddressesProviderImpl.java | 24 +- .../async/AsyncTransactionPruner.java | 6 +- .../async/MilestonePrunerJobQueue.java | 8 +- .../impl/ValidatorPublisher.java | 2 +- .../net/helix/pendulum/conf/ConfigTest.java | 14 +- .../helix/pendulum/conf/ZMQConfigTest.java | 3 +- .../impl/SpentAddressesProviderImplTest.java | 4 +- 25 files changed, 463 insertions(+), 321 deletions(-) delete mode 100644 src/main/java/net/helix/pendulum/conf/GraphConfig.java create mode 100644 src/main/java/net/helix/pendulum/conf/RoundConfig.java create mode 100644 src/main/java/net/helix/pendulum/conf/SpentAddressesConfig.java create mode 100644 src/main/java/net/helix/pendulum/conf/ValidatorConfig.java diff --git a/.gitignore b/.gitignore index 45a18de7..82853dec 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,17 @@ spent-addresses-log *.snapshot.meta.bkp *.snapshot.state.bkp snapshot-*/* +mainnet-db-log/ +mainnet-db/ +mainnet-snapshot/ +mainnet-spent-addresses-db-log/ +mainnet-spent-addresses-db/ +testnet-db-log/ +testnet-db/ +testnet-snapshot/ +testnet-spent-addresses-db-log/ +testnet-spent-addresses-db/ + # modules XI @@ -67,4 +78,4 @@ Validator.* # os .DS_Store -Makefile \ No newline at end of file +Makefile diff --git a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java index bc1887e9..0dc064ed 100644 --- a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java +++ b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java @@ -18,17 +18,165 @@ import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; -/* - Note: the fields in this class are being deserialized from Jackson so they must follow Java Bean convention. - Meaning that every field must have a getter that is prefixed with `get` unless it is a boolean and then it should be - prefixed with `is`. -*/ + public abstract class BasePendulumConfig implements PendulumConfig { + /* + Pendulum default configuration values + Some of these values are not really "configurable" in the sense that the user can't change them using command + line flags, or params in the pendulum.ini file. Instead, if the user wanted to change them, the user would + need to alter the defaults found in this interface and recompile the source code. This is not recommended + since Pendulum has been tested under different conditions to ensure critical network properties with these + default values with different host machines operating together in cooperation to achieve a common goal. + + On the other hand, if your goal is to break the system, then go a head and change the default values. + However, note that the network will quickly discover your deviation and kick you out of the p2p system. + + Happy hacking! ;) + */ + public interface Defaults { + //API + int API_PORT = 8085; + String API_HOST = "localhost"; + List IGNORED_API_ENDPOINTS = PendulumUtils.createImmutableList(); + InetAddress ALLOWED_API_DEFAULT_HOST = InetAddress.getLoopbackAddress(); + List ALLOWED_API_HOSTS = PendulumUtils.createImmutableList(ALLOWED_API_DEFAULT_HOST); + int MAX_FIND_TRANSACTIONS = 100_000; + int MAX_REQUESTS_LIST = 1_000; + int MAX_GET_TRANSACTION_STRINGS = 10_000; + int MAX_BODY_LENGTH = 1_000_000; + String REMOTE_AUTH = ""; + + //Network + int UDP_RECEIVER_PORT = 4100; + int TCP_RECEIVER_PORT = 5100; + double P_REMOVE_REQUEST = 0.01d; + int SEND_LIMIT = -1; + int MAX_PEERS = 0; + boolean DNS_REFRESHER_ENABLED = true; + boolean DNS_RESOLUTION_ENABLED = true; + + //XI + String XI_DIR = "modules"; + + //DB + String DB_PATH = "mainnet-db"; + String DB_LOG_PATH = "mainnet-db-log"; + int DB_CACHE_SIZE = 100_000; + String ROCKS_DB = "rocksdb"; + boolean REVALIDATE = false; + boolean RESCAN_DB = false; + + //Protocol + double P_REPLY_RANDOM_TIP = 0.66d; + double P_DROP_TRANSACTION = 0d; + double P_SELECT_MILESTONE_CHILD = 0.7d; + double P_SEND_MILESTONE = 0.02d; + double P_PROPAGATE_REQUEST = 0.01d; + int MWM = 1; + int PACKET_SIZE = 800; + int REQ_HASH_SIZE = 32; + int QUEUE_SIZE = 1_000; + double P_DROP_CACHE_ENTRY = 0.02d; + int CACHE_SIZE_BYTES = 150_000; + + //Zmq + int ZMQ_THREADS = 1; + boolean ZMQ_ENABLE_IPC = false; + String ZMQ_IPC = "ipc://hlx"; + boolean ZMQ_ENABLE_TCP = false; + int ZMQ_PORT = 5556; + + //TipSel + int MAX_DEPTH = 15; + double ALPHA = 0.001d; + + //Tip solidification + boolean TIP_SOLIDIFIER_ENABLED = true; + + //PoW + int POW_THREADS = 8; + + //Resource directory: + String RESOUCER_PATH = "./src/main/resources"; + String DEFAULT_RESOUCE_PATH = "./resources"; + + //Validator Manager + boolean VALIDATOR_MANAGER_ENABLED = false; + Hash VALIDATOR_MANAGER_ADDRESS = HashFactory.ADDRESS.create( + "9474289ae28f0ea6e3b8bedf8fc52f14d2fa9528a4eb29d7879d8709fd2f6d37" + ); + int UPDATE_VALIDATOR_DELAY = 30000; + int START_ROUND_DELAY = 2; + String VALIDATOR_MANAGER_KEYFILE = "/ValidatorManager.key"; + int VALIDATOR_MANAGER_KEY_DEPTH = 15; + int VALIDATOR_MANAGER_SECURITY = 2; + + //Milestone + boolean VALIDATOR = false; + String VALIDATOR_PATH = null; + Set INITIAL_VALIDATORS = new HashSet<>(Arrays.asList( + HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832"), + HashFactory.ADDRESS.create("a5afe01e64ae959f266b382bb5927fd07b49e7e3180239535126844aaae9bf93"), + HashFactory.ADDRESS.create("e2debe246b5d1a6e05b57b0fc14edb51d136966a91a803b523586ad032f72f3d"), + HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), + HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), + HashFactory.ADDRESS.create("1c6b0ee311a7ddccf255c1097995714b285cb06628be1cef2080b0bef7700e12"), + HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832") + )); + + long GENESIS_TIME = 1569024001000L; + int ROUND_DURATION = 15000; + int ROUND_PAUSE = 5000; + String VALIDATOR_KEYFILE = "/Validator.key"; + String VALIDATOR_SEED_PATH = "/Validator.txt"; + int MILESTONE_KEY_DEPTH = 10; + int VALIDATOR_SECURITY = 2; + int NUMBER_OF_ACTIVE_VALIDATORS = 1; + double CONFIRMATION_THRESHOLD = 0.66; + + // bootstrap hash for empty round + Hash EMPTY_ROUND_HASH = HashFactory.ADDRESS.create( + "00000000000000000000000000000000000000000000656d707479726f756e64" + ); + + //Snapshot + boolean LOCAL_SNAPSHOTS_ENABLED = true; + boolean LOCAL_SNAPSHOTS_PRUNING_ENABLED = false; + int LOCAL_SNAPSHOTS_PRUNING_DELAY = 50000; + int LOCAL_SNAPSHOTS_INTERVAL_SYNCED = 10; + int LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = 1000; + String LOCAL_SNAPSHOTS_BASE_PATH = "./mainnet-snapshot"; + int LOCAL_SNAPSHOTS_DEPTH = 100; + String SNAPSHOT_FILE = "/snapshotMainnet.txt"; + String SNAPSHOT_SIG_FILE = "/snapshotMainnet.sig"; + long GLOBAL_SNAPSHOT_TIME = 1522235533L; + int MILESTONE_START_INDEX = 0; + int NUM_KEYS_IN_MILESTONE = 10; + int MAX_ANALYZED_TXS = 20_000; + + //Spent-addresses + String SPENT_ADDRESSES_DB_PATH = "mainnet-spent-addresses-db"; + String SPENT_ADDRESSES_DB_LOG_PATH = "mainnet-spent-addresses-db-log"; + String PREVIOUS_EPOCHS_SPENT_ADDRESSES_TXT = "/previousEpochsSpentAddresses.txt"; + String PREVIOUS_EPOCHS_SPENT_ADDRESSES_SIG = "/previousEpochsSpentAddresses.sig"; + + + //Logging + boolean SAVELOG_ENABLED = false; + String SAVELOG_BASE_PATH = "./logs/"; + String SAVELOG_XML_FILE = "/logback-save.xml"; + } + + // ******************************************* + // OUR IMPLEMENTATION OF THE CONFIG INTERFACES + // ******************************************* + protected static final String SPLIT_STRING_TO_LIST_REGEX = ",| "; private boolean help; + // PUBLIC SERVICE RELATED CONFIGURATION //API protected int apiPort = Defaults.API_PORT; protected String apiHost = Defaults.API_HOST; @@ -40,9 +188,13 @@ public abstract class BasePendulumConfig implements PendulumConfig { protected int maxBodyLength = Defaults.MAX_BODY_LENGTH; protected String remoteAuth = Defaults.REMOTE_AUTH; + //XI + protected String xiDir = Defaults.XI_DIR; + //We don't have a REMOTE config but we have a remote flag. We must add a field for JCommander private boolean remote; + // COMMUNICATION RELATED CONFIGURATION //Network protected int udpReceiverPort = Defaults.UDP_RECEIVER_PORT; protected int tcpReceiverPort = Defaults.TCP_RECEIVER_PORT; @@ -54,17 +206,6 @@ public abstract class BasePendulumConfig implements PendulumConfig { protected boolean dnsResolutionEnabled = Defaults.DNS_RESOLUTION_ENABLED; protected List neighbors = new ArrayList<>(); - //XI - protected String xiDir = Defaults.XI_DIR; - - //DB - protected String dbPath = Defaults.DB_PATH; - protected String dbLogPath = Defaults.DB_LOG_PATH; - protected int dbCacheSize = Defaults.DB_CACHE_SIZE; //KB - protected String mainDb = Defaults.ROCKS_DB; - protected boolean revalidate = Defaults.REVALIDATE; - protected boolean rescanDb = Defaults.RESCAN_DB; - //Protocol protected double pReplyRandomTip = Defaults.P_REPLY_RANDOM_TIP; protected double pDropTransaction = Defaults.P_DROP_TRANSACTION; @@ -72,20 +213,11 @@ public abstract class BasePendulumConfig implements PendulumConfig { protected double pSendMilestone = Defaults.P_SEND_MILESTONE; protected double pPropagateRequest = Defaults.P_PROPAGATE_REQUEST; - //ZMQ - protected boolean zmqEnableTcp = Defaults.ZMQ_ENABLE_TCP; - protected boolean zmqEnableIpc = Defaults.ZMQ_ENABLE_IPC; - protected int zmqPort = Defaults.ZMQ_PORT; - protected int zmqThreads = Defaults.ZMQ_THREADS; - protected String zmqIpc = Defaults.ZMQ_IPC; - protected int qSizeNode = Defaults.QUEUE_SIZE; - protected int cacheSizeBytes = Defaults.CACHE_SIZE_BYTES; - /** - * @deprecated This field was replaced by {@link #zmqEnableTcp} and {@link #zmqEnableIpc}. It is only needed - * for backward compatibility to --zmq-enabled parameter with JCommander. - */ - @Deprecated - private boolean zmqEnabled; + // CONSENSUS RELATED CONFIGURATION + //Milestone + protected String resourcePath = Defaults.RESOUCER_PATH; + protected String defaultResourcePath = Defaults.DEFAULT_RESOUCE_PATH; + protected int milestoneKeyDepth = Defaults.MILESTONE_KEY_DEPTH; //Tip Selection protected int maxDepth = Defaults.MAX_DEPTH; @@ -107,32 +239,56 @@ public abstract class BasePendulumConfig implements PendulumConfig { protected int localSnapshotsDepth = Defaults.LOCAL_SNAPSHOTS_DEPTH; protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; - //Logging - protected boolean saveLogEnabled = Defaults.SAVELOG_ENABLED; - protected String saveLogBasePath = Defaults.SAVELOG_BASE_PATH; - protected String saveLogXMLFile = Defaults.SAVELOG_XML_FILE; + //Spent-addresses + protected String spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH; + protected String spentAddressesDbLogPath = Defaults.SPENT_ADDRESSES_DB_LOG_PATH; //Validator Manager protected boolean validatorManagerEnabled = Defaults.VALIDATOR_MANAGER_ENABLED; protected Hash validatorManagerAddress = Defaults.VALIDATOR_MANAGER_ADDRESS; protected int updateValidatorDelay = Defaults.UPDATE_VALIDATOR_DELAY; - protected int startRoundDelay = Defaults.START_ROUND_DELAY; protected String validatorManagerKeyfile = Defaults.VALIDATOR_MANAGER_KEYFILE; protected int validatorManagerKeyDepth = Defaults.VALIDATOR_MANAGER_KEY_DEPTH; protected int validatorManagerSecurity = Defaults.VALIDATOR_MANAGER_SECURITY; - //Milestone - protected String validatorPath = Defaults.VALIDATOR_PATH; - protected boolean validator = Defaults.VALIDATOR; - protected Set initialValidators = Defaults.INITIAL_VALIDATORS; + //Round and Timing protected long genesisTime = Defaults.GENESIS_TIME; protected int roundDuration = Defaults.ROUND_DURATION; protected int roundPause = Defaults.ROUND_PAUSE; - protected String resourcePath = Defaults.RESOUCER_PATH; - protected String defaultResourcePath = Defaults.DEFAULT_RESOUCE_PATH; - protected int milestoneKeyDepth = Defaults.MILESTONE_KEY_DEPTH; + protected int startRoundDelay = Defaults.START_ROUND_DELAY; + + //Validator + protected String validatorPath = Defaults.VALIDATOR_PATH; + protected boolean validator = Defaults.VALIDATOR; + protected Set initialValidators = Defaults.INITIAL_VALIDATORS; protected int validatorSecurity = Defaults.VALIDATOR_SECURITY; + + // ACCOUNTABILITY CONFIGURATION + //DB + protected String dbPath = Defaults.DB_PATH; + protected String dbLogPath = Defaults.DB_LOG_PATH; + protected int dbCacheSize = Defaults.DB_CACHE_SIZE; //KB + protected String mainDb = Defaults.ROCKS_DB; + protected boolean revalidate = Defaults.REVALIDATE; + protected boolean rescanDb = Defaults.RESCAN_DB; + + //Logging + protected boolean saveLogEnabled = Defaults.SAVELOG_ENABLED; + protected String saveLogBasePath = Defaults.SAVELOG_BASE_PATH; + protected String saveLogXMLFile = Defaults.SAVELOG_XML_FILE; + + //ZMQ + protected boolean zmqEnableTcp = Defaults.ZMQ_ENABLE_TCP; + protected boolean zmqEnableIpc = Defaults.ZMQ_ENABLE_IPC; + protected int zmqPort = Defaults.ZMQ_PORT; + protected int zmqThreads = Defaults.ZMQ_THREADS; + protected String zmqIpc = Defaults.ZMQ_IPC; + protected int qSizeNode = Defaults.QUEUE_SIZE; + protected int cacheSizeBytes = Defaults.CACHE_SIZE_BYTES; + + + @Override public JCommander parseConfigFromArgs(String[] args) throws ParameterException { //One can invoke help via INI file (feature/bug) so we always create JCommander even if args is empty @@ -422,6 +578,25 @@ protected void setMainDb(String mainDb) { this.mainDb = mainDb; } + + @JsonProperty + @Parameter(names = {"--spnt_add_db_path"}, description = SpentAddressesConfig.Descriptions.SPENT_ADDRESSES_DB_PATH) + protected void setSpentAddressesDbPath(String spentAddressesDbPath) { + this.spentAddressesDbPath = spentAddressesDbPath; + } + public String getSpentAddressesDbPath() { + return spentAddressesDbPath; + } + + @JsonProperty + @Parameter(names = {"--spnt_add_db_log_path"}, description = SpentAddressesConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH) + protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { + this.spentAddressesDbLogPath = spentAddressesDbLogPath; + } + public String getSpentAddressesDbLogPath() { + return spentAddressesDbLogPath; + } + @Override public boolean isRevalidate() { return revalidate; @@ -626,30 +801,11 @@ public int getNumberOfKeysInMilestone() { return Defaults.NUM_KEYS_IN_MILESTONE; } - /** - * Checks if ZMQ is enabled. - * - * @return true if zmqEnableTcp or zmqEnableIpc is set. - */ @Override public boolean isZmqEnabled() { return zmqEnableTcp || zmqEnableIpc; } - /** - * Activates ZMQ to listen on TCP and IPC. - * - * @param zmqEnabled true if ZMQ should listen in TCP and IPC. - * @deprecated Use {@link #setZmqEnableTcp(boolean) and/or {@link #setZmqEnableIpc(boolean)}} instead. - */ - @Deprecated - @JsonProperty - @Parameter(names = "--zmq_enabled", description = ZMQConfig.Descriptions.ZMQ_ENABLED, arity = 1) - protected void setZmqEnabled(boolean zmqEnabled) { - this.zmqEnableTcp = zmqEnabled; - this.zmqEnableIpc = zmqEnabled; - } - @Override public boolean isZmqEnableTcp() { return zmqEnableTcp; @@ -787,9 +943,6 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { @Override public Hash getValidatorManagerAddress() { return validatorManagerAddress; } - @Override - public boolean isDontValidateTestnetValidatorManagerSig() { return false; } - @Override public int getUpdateValidatorDelay() {return updateValidatorDelay; } @JsonProperty @@ -816,7 +969,7 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { public String getValidatorPath() {return validatorPath; } @JsonProperty - @Parameter(names = {"--validator_path"}, description = MilestoneConfig.Descriptions.VALIDATOR_PATH) + @Parameter(names = {"--validator_path"}, description = ValidatorConfig.Descriptions.VALIDATOR_PATH) protected void setValidatorPath(String validatorPath) { this.validatorPath = validatorPath; } @@ -825,7 +978,7 @@ protected void setValidatorPath(String validatorPath) { public boolean isValidator() {return validator; } @JsonProperty - @Parameter(names = {"--validator"}, description = MilestoneConfig.Descriptions.VALIDATOR) + @Parameter(names = {"--validator"}, description = ValidatorConfig.Descriptions.VALIDATOR) protected void setValidator(boolean validator) { this.validator = validator; } @@ -844,7 +997,7 @@ public long getGenesisTime() { } @JsonProperty - @Parameter(names = {"--genesis"}, description = MilestoneConfig.Descriptions.GENESIS_TIME) + @Parameter(names = {"--genesis"}, description = RoundConfig.Descriptions.GENESIS_TIME) protected void setGenesisTime(long genesisTime) { this.genesisTime = genesisTime; } @Override @@ -853,13 +1006,13 @@ public int getRoundDuration() { } @JsonProperty - @Parameter(names = {"--round"}, description = MilestoneConfig.Descriptions.ROUND_DURATION) + @Parameter(names = {"--round"}, description = RoundConfig.Descriptions.ROUND_DURATION) protected void setRoundDuration(int roundDuration) { this.roundDuration = roundDuration; } @Override public int getRoundPause() { return roundPause; } @JsonProperty - @Parameter(names = {"--round_pause"}, description = MilestoneConfig.Descriptions.ROUND_PAUSE) + @Parameter(names = {"--round_pause"}, description = RoundConfig.Descriptions.ROUND_PAUSE) protected void setRoundPause(int roundPause) { this.roundPause = roundPause; } @Override @@ -937,127 +1090,4 @@ protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; } - public interface Defaults { - //API - int API_PORT = 8085; - String API_HOST = "localhost"; - List IGNORED_API_ENDPOINTS = PendulumUtils.createImmutableList(); // "addNeighbors", "getNeighbors", "removeNeighbors", "attachToTangle", "interruptAttachingToTangle" <- TODO: limit these in production! - InetAddress ALLOWED_API_DEFAULT_HOST = InetAddress.getLoopbackAddress(); - List ALLOWED_API_HOSTS = PendulumUtils.createImmutableList(ALLOWED_API_DEFAULT_HOST); - int MAX_FIND_TRANSACTIONS = 100_000; - int MAX_REQUESTS_LIST = 1_000; - int MAX_GET_TRANSACTION_STRINGS = 10_000; - int MAX_BODY_LENGTH = 1_000_000; - String REMOTE_AUTH = ""; - - //Network - int UDP_RECEIVER_PORT = 4100; - int TCP_RECEIVER_PORT = 5100; - double P_REMOVE_REQUEST = 0.01d; - int SEND_LIMIT = -1; - int MAX_PEERS = 0; - boolean DNS_REFRESHER_ENABLED = true; - boolean DNS_RESOLUTION_ENABLED = true; - - //XI - String XI_DIR = "modules"; - - //DB - String DB_PATH = "mainnetdb"; - String DB_LOG_PATH = "mainnet.log"; - int DB_CACHE_SIZE = 100_000; - String ROCKS_DB = "rocksdb"; - boolean REVALIDATE = false; - boolean RESCAN_DB = false; - - //Protocol - double P_REPLY_RANDOM_TIP = 0.66d; - double P_DROP_TRANSACTION = 0d; - double P_SELECT_MILESTONE_CHILD = 0.7d; - double P_SEND_MILESTONE = 0.02d; - double P_PROPAGATE_REQUEST = 0.01d; - int MWM = 1; - int PACKET_SIZE = 800; - int REQ_HASH_SIZE = 32; - int QUEUE_SIZE = 1_000; - double P_DROP_CACHE_ENTRY = 0.02d; - int CACHE_SIZE_BYTES = 150_000; - - //Zmq - int ZMQ_THREADS = 1; - boolean ZMQ_ENABLE_IPC = false; - String ZMQ_IPC = "ipc://hlx"; - boolean ZMQ_ENABLE_TCP = false; - int ZMQ_PORT = 5556; - - //TipSel - int MAX_DEPTH = 15; - double ALPHA = 0.001d; - - //Tip solidification - boolean TIP_SOLIDIFIER_ENABLED = true; - - //PoW - int POW_THREADS = 8; - - //Resource directory: - String RESOUCER_PATH = "./src/main/resources"; - String DEFAULT_RESOUCE_PATH = "./resources"; - - //Validator Manager - boolean VALIDATOR_MANAGER_ENABLED = false; - Hash VALIDATOR_MANAGER_ADDRESS = HashFactory.ADDRESS.create("9474289ae28f0ea6e3b8bedf8fc52f14d2fa9528a4eb29d7879d8709fd2f6d37"); - int UPDATE_VALIDATOR_DELAY = 30000; - int START_ROUND_DELAY = 2; - String VALIDATOR_MANAGER_KEYFILE = "/ValidatorManager.key"; - int VALIDATOR_MANAGER_KEY_DEPTH = 15; - int VALIDATOR_MANAGER_SECURITY = 2; - - //Milestone - boolean VALIDATOR = false; - String VALIDATOR_PATH = null; - Set INITIAL_VALIDATORS = new HashSet<>(Arrays.asList( - HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832"), - HashFactory.ADDRESS.create("a5afe01e64ae959f266b382bb5927fd07b49e7e3180239535126844aaae9bf93"), - HashFactory.ADDRESS.create("e2debe246b5d1a6e05b57b0fc14edb51d136966a91a803b523586ad032f72f3d"), - HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), - HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), - HashFactory.ADDRESS.create("1c6b0ee311a7ddccf255c1097995714b285cb06628be1cef2080b0bef7700e12"), - HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832") - )); - - long GENESIS_TIME = 1569024001000L; - int ROUND_DURATION = 15000; - int ROUND_PAUSE = 5000; - String VALIDATOR_KEYFILE = "/Validator.key"; - String VALIDATOR_SEED_PATH = "/Validator.txt"; - int MILESTONE_KEY_DEPTH = 10; - int VALIDATOR_SECURITY = 2; - int NUMBER_OF_ACTIVE_VALIDATORS = 1; - double CONFIRMATION_THRESHOLD = 0.66; - - Hash EMPTY_ROUND_HASH = HashFactory.ADDRESS.create("00000000000000000000000000000000000000000000656d707479726f756e64"); // bootstrap hash for empty round - - //Snapshot - boolean LOCAL_SNAPSHOTS_ENABLED = true; - boolean LOCAL_SNAPSHOTS_PRUNING_ENABLED = false; - int LOCAL_SNAPSHOTS_PRUNING_DELAY = 50000; - int LOCAL_SNAPSHOTS_INTERVAL_SYNCED = 10; - int LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = 1000; - String LOCAL_SNAPSHOTS_BASE_PATH = "./snapshot-mainnet"; - int LOCAL_SNAPSHOTS_DEPTH = 100; - String SNAPSHOT_FILE = "/snapshotMainnet.txt"; - String SNAPSHOT_SIG_FILE = "/snapshotMainnet.sig"; - String PREVIOUS_EPOCHS_SPENT_ADDRESSES_TXT = "/previousEpochsSpentAddresses.txt"; - String PREVIOUS_EPOCHS_SPENT_ADDRESSES_SIG = "/previousEpochsSpentAddresses.sig"; - long GLOBAL_SNAPSHOT_TIME = 1522235533L; - int MILESTONE_START_INDEX = 0; - int NUM_KEYS_IN_MILESTONE = 10; - int MAX_ANALYZED_TXS = 20_000; - - //Logging - boolean SAVELOG_ENABLED = false; - String SAVELOG_BASE_PATH = "./logs/"; - String SAVELOG_XML_FILE = "/logback-save.xml"; - } } diff --git a/src/main/java/net/helix/pendulum/conf/Config.java b/src/main/java/net/helix/pendulum/conf/Config.java index 5be42362..9548f8b7 100644 --- a/src/main/java/net/helix/pendulum/conf/Config.java +++ b/src/main/java/net/helix/pendulum/conf/Config.java @@ -1,14 +1,18 @@ package net.helix.pendulum.conf; public interface Config { + String TESTNET_FLAG = "--testnet"; + /** * @return {@value Descriptions#TESTNET} */ boolean isTestnet(); + interface Descriptions { String TESTNET = "Start in testnet mode."; } + class DescriptionHelper { protected static final String PROB_OF = "A number between 0 and 1 that represents the probability of "; } diff --git a/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java b/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java index 1626e3d2..0f79911f 100644 --- a/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java @@ -6,5 +6,12 @@ * @implNote It currently extends two other interfaces. This has been done due to lack of separation of concerns in * the current code base and will be changed in the future */ -public interface ConsensusConfig extends SnapshotConfig, MilestoneConfig, ValidatorManagerConfig { + +public interface ConsensusConfig extends + MilestoneConfig, + RoundConfig, + SnapshotConfig, + SpentAddressesConfig, + ValidatorConfig, + ValidatorManagerConfig { } diff --git a/src/main/java/net/helix/pendulum/conf/GraphConfig.java b/src/main/java/net/helix/pendulum/conf/GraphConfig.java deleted file mode 100644 index c52b06ff..00000000 --- a/src/main/java/net/helix/pendulum/conf/GraphConfig.java +++ /dev/null @@ -1,9 +0,0 @@ -package net.helix.pendulum.conf; - -public interface GraphConfig extends Config{ - boolean isGraphEnabled(); - - interface Descriptions { - String GRAPH_ENABLED = "Enabling Graphstream"; - } -} \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/MainnetConfig.java b/src/main/java/net/helix/pendulum/conf/MainnetConfig.java index 206d4511..137875e2 100644 --- a/src/main/java/net/helix/pendulum/conf/MainnetConfig.java +++ b/src/main/java/net/helix/pendulum/conf/MainnetConfig.java @@ -1,6 +1,7 @@ package net.helix.pendulum.conf; public class MainnetConfig extends BasePendulumConfig { + @Override public boolean isTestnet() { return false; diff --git a/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java b/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java index 93bde292..ec52e411 100644 --- a/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java @@ -1,75 +1,28 @@ package net.helix.pendulum.conf; -import net.helix.pendulum.model.Hash; - -import java.util.Set; /** * Configs that should be used for tracking milestones */ public interface MilestoneConfig extends Config { - /** - * @return {@value Descriptions#VALIDATOR} - */ - boolean isValidator(); - /** - * @return {@value Descriptions#VALIDATOR_PATH} - */ - String getValidatorPath(); - /** - * @return {@value Descriptions#VALIDATOR_SEED_PATH} - */ - String getValidatorSeedfile(); - /** - * Returns is validator is enabled, is dependent on isValidator and ValidatorPath - * @return {@value Descriptions#VALIDATOR} - */ - boolean isValidatorEnabled(); - /** - * @return Descriptions#INITIAL_VALIDATORS - */ - Set getInitialValidators(); - /** - * @return {@value Descriptions#VALIDATE_TESTNET_MILESTONE_SIG} - */ - boolean isValidateTestnetMilestoneSig(); - /** - * @return {@value Descriptions#GENESIS_TIME} - */ - long getGenesisTime(); - /** - * @return {@value Descriptions#ROUND_DURATION} - */ - int getRoundDuration(); - /** - * @return {@value Descriptions#ROUND_PAUSE} - */ - int getRoundPause(); - /** - * @return {@value Descriptions#VALIDATOR_KEYFILE} - */ - String getValidatorKeyfile(); /** * @return {@value Descriptions#MILESTONE_KEY_DEPTH} */ int getMilestoneKeyDepth(); + /** - * @return {@value Descriptions#VALIDATOR_SECURITY} + * @return {@value Descriptions#MILESTONE_START_INDEX} */ - int getValidatorSecurity(); + int getMilestoneStartIndex(); + interface Descriptions { - String VALIDATOR = "Flag that enables applying as a validator in the network."; - String VALIDATOR_PATH = "A path to a file containing the seed / keyfile has to be passed."; - String VALIDATOR_SEED_PATH = "A path to a file containing the seed has to be passed."; - String INITIAL_VALIDATORS = "The addresses of validators the network starts with"; - String VALIDATE_TESTNET_MILESTONE_SIG = "Disable validator validation on testnet"; - String GENESIS_TIME = "Time when the ledger started."; - String ROUND_DURATION = "Duration of a round in milli secounds."; - String ROUND_PAUSE = "Duration of time to finalize the round in milli secounds."; - String VALIDATOR_KEYFILE = "Filepath to validator keyfile"; String MILESTONE_KEY_DEPTH = "Depth of the merkle tree the milestones are signed with."; - String VALIDATOR_SECURITY = "Security level of transactions sent from a validator (milestones, registrations)"; + String MILESTONE_START_INDEX = "The start index of the milestones. This index is encoded in each milestone " + + "transaction by the coordinator."; + String NUMBER_OF_KEYS_IN_A_MILESTONE = "The depth of the Merkle tree which in turn determines the number of" + + "leaves (private keys) that the coordinator can use to sign a message."; + } } diff --git a/src/main/java/net/helix/pendulum/conf/PendulumConfig.java b/src/main/java/net/helix/pendulum/conf/PendulumConfig.java index 7a58c710..77c530d8 100644 --- a/src/main/java/net/helix/pendulum/conf/PendulumConfig.java +++ b/src/main/java/net/helix/pendulum/conf/PendulumConfig.java @@ -6,11 +6,25 @@ import java.io.File; /** - * A container for all possible configuration parameters of SBX. + * A container for all possible configuration parameters of Pendulum. * In charge of how we parse the configuration from given inputs. */ -public interface PendulumConfig extends APIConfig, NodeConfig, XIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PoWConfig, SolidificationConfig, LoggingConfig { - File CONFIG_FILE = new File("hlx.ini"); +public interface PendulumConfig extends + APIConfig, + ConsensusConfig, +// extends MilestoneConfig, RoundConfig, SnapshotConfig, SpentAddressesConfig, ValidatorConfig, ValidatorManagerConfig + DbConfig, + LoggingConfig, + NodeConfig,//extends NetworkConfig, ProtocolConfig + PoWConfig, + SolidificationConfig, + TipSelConfig, + XIConfig, + ZMQConfig + { + + File CONFIG_FILE = new File("pendulum.ini"); + /** * Parses the args to populate the configuration object * @@ -19,5 +33,6 @@ public interface PendulumConfig extends APIConfig, NodeConfig, XIConfig, DbConfi * @throws ParameterException if the parsing failed */ JCommander parseConfigFromArgs(String[] args) throws ParameterException; + boolean isHelp(); } diff --git a/src/main/java/net/helix/pendulum/conf/ProtocolConfig.java b/src/main/java/net/helix/pendulum/conf/ProtocolConfig.java index 4e1091bd..7c6a5bd0 100644 --- a/src/main/java/net/helix/pendulum/conf/ProtocolConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ProtocolConfig.java @@ -37,5 +37,6 @@ interface Descriptions { String P_SEND_MILESTONE = DescriptionHelper.PROB_OF + "sending a milestoneTracker transaction when the node looks for a random transaction to send to a neighbor."; String P_REPLY_RANDOM_TIP = DescriptionHelper.PROB_OF + "replying to a random transaction request, even though your node doesn't have anything to request."; String P_PROPAGATE_REQUEST = DescriptionHelper.PROB_OF + "propagating the request of a transaction to a neighbor node if it can't be found. This should be low since we don't want to propagate non-existing transactions that spam the network."; + } } \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/RoundConfig.java b/src/main/java/net/helix/pendulum/conf/RoundConfig.java new file mode 100644 index 00000000..f9fc9456 --- /dev/null +++ b/src/main/java/net/helix/pendulum/conf/RoundConfig.java @@ -0,0 +1,23 @@ +package net.helix.pendulum.conf; + +public interface RoundConfig extends Config { + + /** + * @return {@value Descriptions#GENESIS_TIME} + */ + long getGenesisTime(); + /** + * @return {@value Descriptions#ROUND_DURATION} + */ + int getRoundDuration(); + /** + * @return {@value Descriptions#ROUND_PAUSE} + */ + int getRoundPause(); + + interface Descriptions { + String GENESIS_TIME = "Time when the ledger started."; + String ROUND_DURATION = "Duration of a round in milli secounds."; + String ROUND_PAUSE = "Duration of time to finalize the round in milli secounds."; + } +} diff --git a/src/main/java/net/helix/pendulum/conf/SnapshotConfig.java b/src/main/java/net/helix/pendulum/conf/SnapshotConfig.java index df45c54d..07278a8a 100644 --- a/src/main/java/net/helix/pendulum/conf/SnapshotConfig.java +++ b/src/main/java/net/helix/pendulum/conf/SnapshotConfig.java @@ -1,7 +1,7 @@ package net.helix.pendulum.conf; /** - * Configurations for handling global snapshot data + * Configurations for handling snapshot data */ public interface SnapshotConfig extends Config { @@ -50,10 +50,7 @@ public interface SnapshotConfig extends Config { */ String getSnapshotSignatureFile(); - /** - * @return {@value Descriptions#MILESTONE_START_INDEX} - */ - int getMilestoneStartIndex(); + /** * @return {@value Descriptions#LOCAL_SNAPSHOTS_BASE_PATH} @@ -87,12 +84,7 @@ interface Descriptions { String SNAPSHOT_TIME = "Epoch time of the last snapshot."; String SNAPSHOT_FILE = "Path of the file that contains the state of the ledger at the last snapshot."; String SNAPSHOT_SIGNATURE_FILE = "Path to the file that contains a signature for the snapshot file."; - String MILESTONE_START_INDEX = "The start index of the milestones. This index is encoded in each milestone " + - "transaction by the coordinator."; - String NUMBER_OF_KEYS_IN_A_MILESTONE = "The depth of the Merkle tree which in turn determines the number of" + - "leaves (private keys) that the coordinator can use to sign a message."; - String PREVIOUS_EPOCH_SPENT_ADDRESSES_FILE = "The file that contains the list of all used addresses " + - "from previous epochs"; - String PREVIOUS_EPOCH_SPENT_ADDRESSES_SIG_FILE = "The file that contains signature of the previous epochs spent addresses file"; + + } } \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/SpentAddressesConfig.java b/src/main/java/net/helix/pendulum/conf/SpentAddressesConfig.java new file mode 100644 index 00000000..0464bd33 --- /dev/null +++ b/src/main/java/net/helix/pendulum/conf/SpentAddressesConfig.java @@ -0,0 +1,35 @@ +package net.helix.pendulum.conf; + +/** + * Configurations for handling global snapshot data + */ +public interface SpentAddressesConfig extends Config { + + /** + * @return {@value Descriptions#SPENT_ADDRESSES_DB_PATH} + */ + String getSpentAddressesDbPath(); + + /** + * @return {@value Descriptions#SPENT_ADDRESSES_DB_LOG_PATH} + */ + String getSpentAddressesDbLogPath(); + + /** + * @return {@value Descriptions#PREVIOUS_EPOCH_SPENT_ADDRESSES_FILE} + */ + String getPreviousEpochSpentAddressesFiles(); + + /** + * @return {@value Descriptions#PREVIOUS_EPOCH_SPENT_ADDRESSES_SIG_FILE} + */ + String getPreviousEpochSpentAddressesSigFile(); + + interface Descriptions { + + String SPENT_ADDRESSES_DB_PATH = "The path where the spent addresses db is stored"; + String SPENT_ADDRESSES_DB_LOG_PATH = "The path where the spent addresses db log is stored"; + String PREVIOUS_EPOCH_SPENT_ADDRESSES_FILE = "The file that contains the list of all used addresses from previous epochs"; + String PREVIOUS_EPOCH_SPENT_ADDRESSES_SIG_FILE = "The file that contains signature of the previous epochs spent addresses file"; + } +} diff --git a/src/main/java/net/helix/pendulum/conf/TestnetConfig.java b/src/main/java/net/helix/pendulum/conf/TestnetConfig.java index fa207272..55367d21 100644 --- a/src/main/java/net/helix/pendulum/conf/TestnetConfig.java +++ b/src/main/java/net/helix/pendulum/conf/TestnetConfig.java @@ -23,6 +23,8 @@ public TestnetConfig() { super(); dbPath = Defaults.DB_PATH; dbLogPath = Defaults.DB_LOG_PATH; + spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH; + spentAddressesDbLogPath= Defaults.SPENT_ADDRESSES_DB_LOG_PATH; localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; } @@ -37,7 +39,7 @@ public boolean isValidateTestnetMilestoneSig() { } @JsonProperty - @Parameter(names = "--validate_testnet_milestone_sig", description = MilestoneConfig.Descriptions.VALIDATE_TESTNET_MILESTONE_SIG) + @Parameter(names = "--validate_testnet_milestone_sig", description = ValidatorConfig.Descriptions.VALIDATE_TESTNET_MILESTONE_SIG) protected void setValidateTestnetMilestoneSig(boolean validateTestnetMilestoneSig) { this.validateTestnetMilestoneSig = validateTestnetMilestoneSig; } @@ -92,7 +94,7 @@ public int getMilestoneStartIndex() { } @JsonProperty - @Parameter(names = "--milestone-start", description = SnapshotConfig.Descriptions.MILESTONE_START_INDEX) + @Parameter(names = "--milestone-start", description = MilestoneConfig.Descriptions.MILESTONE_START_INDEX) protected void setMilestoneStartIndex(int milestoneStartIndex) { this.milestoneStartIndex = milestoneStartIndex; } @@ -103,7 +105,7 @@ public int getNumberOfKeysInMilestone() { } @JsonProperty("NUMBER_OF_KEYS_IN_A_MILESTONE") - @Parameter(names = "--milestone-keys", description = SnapshotConfig.Descriptions.NUMBER_OF_KEYS_IN_A_MILESTONE) + @Parameter(names = "--milestone-keys", description = MilestoneConfig.Descriptions.NUMBER_OF_KEYS_IN_A_MILESTONE) protected void setNumberOfKeysInMilestone(int numberOfKeysInMilestone) { this.numberOfKeysInMilestone = numberOfKeysInMilestone; } @@ -148,19 +150,37 @@ public void setDbLogPath(String dbLogPath) { super.setDbLogPath(dbLogPath); } + @JsonProperty + @Override + public void setSpentAddressesDbPath(String spentAddressesDbPath) { + if (Objects.equals(MainnetConfig.Defaults.SPENT_ADDRESSES_DB_PATH, spentAddressesDbPath)) { + throw new ParameterException("Testnet spent-addresses db folder cannot be configured to mainnet's spent-addresses-db folder"); + } + super.setSpentAddressesDbPath(spentAddressesDbPath); + } + + @JsonProperty + @Override + public void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { + if (Objects.equals(MainnetConfig.Defaults.SPENT_ADDRESSES_DB_LOG_PATH, spentAddressesDbLogPath)) { + throw new ParameterException("Testnet spent-addresses db log folder cannot be configured to mainnet's spent-addresses-db log folder"); + } + super.setSpentAddressesDbLogPath(spentAddressesDbLogPath); + } + @Override public long getGenesisTime() { return genesisTime; } @JsonProperty - @Parameter(names = {"--genesis-testnet"}, description = MilestoneConfig.Descriptions.GENESIS_TIME) + @Parameter(names = {"--genesis-testnet"}, description = RoundConfig.Descriptions.GENESIS_TIME) protected void setGenesisTime(int genesisTime) { this.genesisTime = genesisTime; } public interface Defaults { long GENESIS_TIME = 1571279107785L; boolean VALIDATE_MILESTONE_SIG = true; - String LOCAL_SNAPSHOTS_BASE_PATH = "snapshot-testnet"; + String LOCAL_SNAPSHOTS_BASE_PATH = "testnet-snapshot"; String SNAPSHOT_FILE = "/snapshotTestnet.txt"; int REQUEST_HASH_SIZE = 32; String SNAPSHOT_SIG = "/snapshotTestnet.sig"; @@ -169,8 +189,9 @@ public interface Defaults { int MILESTONE_START_INDEX = 0; int KEYS_IN_MILESTONE = 10; int PACKET_SIZE = 800; - String DB_PATH = "testnetdb"; - String DB_LOG_PATH = "testnetdb.log"; + String DB_PATH = "testnet-db"; + String DB_LOG_PATH = "testnet-db-log"; + String SPENT_ADDRESSES_DB_PATH = "testnet-spent-addresses-db"; + String SPENT_ADDRESSES_DB_LOG_PATH = "testnet-spent-addresses-db-log"; } } - diff --git a/src/main/java/net/helix/pendulum/conf/ValidatorConfig.java b/src/main/java/net/helix/pendulum/conf/ValidatorConfig.java new file mode 100644 index 00000000..1e0f9fe5 --- /dev/null +++ b/src/main/java/net/helix/pendulum/conf/ValidatorConfig.java @@ -0,0 +1,55 @@ +package net.helix.pendulum.conf; + +import net.helix.pendulum.model.Hash; + +import java.util.Set; + +public interface ValidatorConfig extends Config { + + /** + * @return {@value Descriptions#VALIDATOR_SECURITY} + */ + int getValidatorSecurity(); + /** + * @return {@value Descriptions#VALIDATOR} + */ + boolean isValidator(); + /** + * @return {@value Descriptions#VALIDATOR_PATH} + */ + String getValidatorPath(); + /** + * @return {@value Descriptions#VALIDATOR_SEED_PATH} + */ + String getValidatorSeedfile(); + /** + * Returns is validator is enabled, is dependent on isValidator and ValidatorPath + * @return {@value Descriptions#VALIDATOR} + */ + boolean isValidatorEnabled(); + /** + * @return Descriptions#INITIAL_VALIDATORS + */ + Set getInitialValidators(); + /** + * @return {@value Descriptions#VALIDATE_TESTNET_MILESTONE_SIG} + */ + boolean isValidateTestnetMilestoneSig(); + + /** + * @return {@value Descriptions#VALIDATOR_KEYFILE} + */ + String getValidatorKeyfile(); + + interface Descriptions { + String INITIAL_VALIDATORS = "The addresses of validators the network starts with"; + String VALIDATOR = "Flag that enables applying as a validator in the network."; + String VALIDATOR_PATH = "A path to a file containing the seed / keyfile has to be passed."; + String VALIDATOR_SEED_PATH = "A path to a file containing the seed has to be passed."; + String VALIDATE_TESTNET_MILESTONE_SIG = "Disable validator validation on testnet"; + String VALIDATOR_KEYFILE = "Filepath to validator keyfile"; + String VALIDATOR_SECURITY = "Security level of transactions sent from a validator (milestones, registrations)"; + + } + +} diff --git a/src/main/java/net/helix/pendulum/conf/ValidatorManagerConfig.java b/src/main/java/net/helix/pendulum/conf/ValidatorManagerConfig.java index 20ca5656..fd48d95f 100644 --- a/src/main/java/net/helix/pendulum/conf/ValidatorManagerConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ValidatorManagerConfig.java @@ -14,10 +14,6 @@ public interface ValidatorManagerConfig extends Config { * @return Descriptions#VALIDATOR_MANAGER_ADDRESS */ Hash getValidatorManagerAddress(); - /** - * @return {@value Descriptions#DONT_VALIDATE_TESTNET_VALIDATOR_MANAGER_SIG} - */ - boolean isDontValidateTestnetValidatorManagerSig(); /** * @return {@value Descriptions#UPDATE_VALIDATOR_DELAY} */ @@ -43,7 +39,6 @@ public interface ValidatorManagerConfig extends Config { interface Descriptions { String VALIDATOR_MANAGER_ENABLED = "Flag that determines if the node is a validator manager."; String VALIDATOR_MANAGER_ADDRESS = "The address of the node that publishes validators"; - String DONT_VALIDATE_TESTNET_VALIDATOR_MANAGER_SIG = "Disable validatomanager validation on testnet"; String UPDATE_VALIDATOR_DELAY = "The desired delay for updating validators in seconds."; String START_ROUND_DELAY = "The number of rounds between validators are published and the round they start to operate."; String VALIDATOR_MANAGER_KEYFILE = "Filepath to validatomanager keyfile"; diff --git a/src/main/java/net/helix/pendulum/conf/XIConfig.java b/src/main/java/net/helix/pendulum/conf/XIConfig.java index aebbc73a..f5615506 100644 --- a/src/main/java/net/helix/pendulum/conf/XIConfig.java +++ b/src/main/java/net/helix/pendulum/conf/XIConfig.java @@ -5,11 +5,14 @@ */ public interface XIConfig extends Config{ + String XI_DIR = "modules"; + /** * @return Descriptions#XI_DIR */ String getXiDir(); + interface Descriptions { String XI_DIR = "The folder where XI modules should be added for automatic discovery."; } diff --git a/src/main/java/net/helix/pendulum/conf/ZMQConfig.java b/src/main/java/net/helix/pendulum/conf/ZMQConfig.java index 94e62644..e283da98 100644 --- a/src/main/java/net/helix/pendulum/conf/ZMQConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ZMQConfig.java @@ -34,9 +34,8 @@ public interface ZMQConfig extends Config { interface Descriptions { String ZMQ_PORT = "The port used to connect to the ZMQ feed"; String ZMQ_IPC = "The path that is used to communicate with ZMQ in IPC"; - String ZMQ_ENABLED = "Enable zmq channels (deprecated). Use --zmq-enable-tcp or --zmq-enable-ipc instead"; String ZMQ_ENABLE_TCP = "Enable zmq channels on tcp port 5556. Use --zmq-port=[PORT] to override."; - String ZMQ_ENABLE_IPC = "Enable zmq channels on ipc://iri. Use --zmq-ipc=[SOCKET] to override."; + String ZMQ_ENABLE_IPC = "Enable zmq channels on ipc://pendulum. Use --zmq-ipc=[SOCKET] to override."; String ZMQ_THREADS = "The threads used by ZMQ publisher"; } } diff --git a/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java b/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java index 9c6c905d..ed3ec99b 100644 --- a/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java +++ b/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java @@ -1,7 +1,7 @@ package net.helix.pendulum.service.snapshot.impl; import net.helix.pendulum.SignedFiles; -import net.helix.pendulum.conf.SnapshotConfig; +import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.service.snapshot.*; @@ -60,7 +60,7 @@ public class SnapshotProviderImpl implements SnapshotProvider { /** * Holds a cached version of the builtin snapshot. * - * Note: The builtin snapshot is embedded in the iri.jar and will not change. To speed up tests that need the + * Note: The builtin snapshot is embedded in the pendulum.jar and will not change. To speed up tests that need the * snapshot multiple times while creating their own version of the LocalSnapshotManager, we cache the instance * here so they don't have to rebuild it from the scratch every time (massively speeds up the unit tests). */ @@ -70,7 +70,8 @@ public class SnapshotProviderImpl implements SnapshotProvider { /** * Holds Snapshot related configuration parameters. */ - private SnapshotConfig config; + private ConsensusConfig config; + /** * Internal property for the value returned by {@link SnapshotProvider#getInitialSnapshot()}. @@ -99,7 +100,7 @@ public class SnapshotProviderImpl implements SnapshotProvider { * @return the initialized instance itself to allow chaining * */ - public SnapshotProviderImpl init(SnapshotConfig config) throws SnapshotException { + public SnapshotProviderImpl init(ConsensusConfig config) throws SnapshotException { this.config = config; File pathToLocalSnapshotDir = Paths.get(this.config.getLocalSnapshotsBasePath()).toFile(); if (!pathToLocalSnapshotDir.exists() || !pathToLocalSnapshotDir.isDirectory()) { diff --git a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 8f6c090e..f3de7585 100644 --- a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -1,7 +1,7 @@ package net.helix.pendulum.service.spentaddresses.impl; +import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.conf.SnapshotConfig; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.persistables.SpentAddress; @@ -14,7 +14,6 @@ import net.helix.pendulum.utils.Pair; import java.io.BufferedReader; -import java.io.File; import java.io.InputStream; import java.io.InputStreamReader; import java.util.Collection; @@ -34,19 +33,15 @@ public class SpentAddressesProviderImpl implements SpentAddressesProvider { //@VisibleForTesting public RocksDBPersistenceProvider rocksDBPersistenceProvider; - private SnapshotConfig config; + private ConsensusConfig config; + +// private String hello = config.getSpentAddressesDbPath(); +// private String kitty = config.getSpentAddressesDbLogPath(); /** * Creates a new instance of SpentAddressesProvider */ public SpentAddressesProviderImpl() { - Map> columnFamilies = new HashMap<>(); - columnFamilies.put("spent-addresses", SpentAddress.class); - String addressDBPath = System.getProperty("spent.addresses.db", ""); - this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider( - addressDBPath + SPENT_ADDRESSES_DB, - addressDBPath + SPENT_ADDRESSES_LOG, 1000, - columnFamilies, null); } /** @@ -56,9 +51,16 @@ public SpentAddressesProviderImpl() { * @return the current instance * @throws SpentAddressesException if we failed to create a file at the designated location */ - public SpentAddressesProviderImpl init(SnapshotConfig config) + public SpentAddressesProviderImpl init(ConsensusConfig config) throws SpentAddressesException { this.config = config; + Map> columnFamilies = new HashMap<>(); + columnFamilies.put("spent-addresses", SpentAddress.class); + this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + config.getSpentAddressesDbPath(), config.getSpentAddressesDbLogPath(), + 1000, + columnFamilies, + null); try { this.rocksDBPersistenceProvider.init(); readPreviousEpochsSpentAddresses(); diff --git a/src/main/java/net/helix/pendulum/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/net/helix/pendulum/service/transactionpruning/async/AsyncTransactionPruner.java index 2fa3976b..b3d254dd 100644 --- a/src/main/java/net/helix/pendulum/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/net/helix/pendulum/service/transactionpruning/async/AsyncTransactionPruner.java @@ -1,7 +1,7 @@ package net.helix.pendulum.service.transactionpruning.async; import net.helix.pendulum.conf.BasePendulumConfig; -import net.helix.pendulum.conf.SnapshotConfig; +import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.controllers.TipsViewModel; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.spentaddresses.SpentAddressesService; @@ -75,7 +75,7 @@ public class AsyncTransactionPruner implements TransactionPruner { /** * Configuration with important snapshot related parameters. */ - private SnapshotConfig config; + private ConsensusConfig config; /** * Holds a reference to the {@link ThreadIdentifier} for the cleanup thread. @@ -129,7 +129,7 @@ public class AsyncTransactionPruner implements TransactionPruner { */ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesService spentAddressesService, TipsViewModel tipsViewModel, - SnapshotConfig config) { + ConsensusConfig config) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; diff --git a/src/main/java/net/helix/pendulum/service/transactionpruning/async/MilestonePrunerJobQueue.java b/src/main/java/net/helix/pendulum/service/transactionpruning/async/MilestonePrunerJobQueue.java index a0b6c3ba..1a4e2c0c 100644 --- a/src/main/java/net/helix/pendulum/service/transactionpruning/async/MilestonePrunerJobQueue.java +++ b/src/main/java/net/helix/pendulum/service/transactionpruning/async/MilestonePrunerJobQueue.java @@ -1,6 +1,6 @@ package net.helix.pendulum.service.transactionpruning.async; -import net.helix.pendulum.conf.SnapshotConfig; +import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.service.transactionpruning.TransactionPruner; import net.helix.pendulum.service.transactionpruning.TransactionPrunerJobStatus; import net.helix.pendulum.service.transactionpruning.TransactionPruningException; @@ -40,12 +40,12 @@ public class MilestonePrunerJobQueue implements JobQueue { * different way than other jobs and consolidate the queue whenever we add a new job. * * @param transactionPruner reference to the container of this queue - * @param snapshotConfig reference to the config with snapshot related parameters + * @param consensusConfig reference to the consensus related configuration parameters */ - public MilestonePrunerJobQueue(TransactionPruner transactionPruner, SnapshotConfig snapshotConfig) { + public MilestonePrunerJobQueue(TransactionPruner transactionPruner, ConsensusConfig consensusConfig) { this.transactionPruner = transactionPruner; - youngestFullyCleanedMilestoneIndex = snapshotConfig.getMilestoneStartIndex(); + youngestFullyCleanedMilestoneIndex = consensusConfig.getMilestoneStartIndex(); } /** diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorPublisher.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorPublisher.java index d654eacd..b6ff562c 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorPublisher.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorPublisher.java @@ -26,7 +26,7 @@ public ValidatorPublisher(PendulumConfig configuration, API api) { this.api = api; delay = config.getUpdateValidatorDelay(); mwm = config.getMwm(); - sign = !config.isDontValidateTestnetValidatorManagerSig(); + sign = config.isValidateTestnetMilestoneSig(); currentKeyIndex = 0; startRoundDelay = config.getStartRoundDelay(); } diff --git a/src/test/java/net/helix/pendulum/conf/ConfigTest.java b/src/test/java/net/helix/pendulum/conf/ConfigTest.java index 2d23d2f0..168b8b60 100644 --- a/src/test/java/net/helix/pendulum/conf/ConfigTest.java +++ b/src/test/java/net/helix/pendulum/conf/ConfigTest.java @@ -272,7 +272,7 @@ public void argsParsingMainnetTest() { Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); Assert.assertEquals("xi-dir", "/XI", pendulumConfig.getXiDir()); Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); - Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); + //Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); Assert.assertNotEquals("mwm", 4, pendulumConfig.getMwm()); Assert.assertNotEquals("coo", pendulumConfig.getValidatorManagerAddress(), "TTTTTTTTT"); } @@ -333,7 +333,7 @@ public void argsParsingTestnetTest() { Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); Assert.assertEquals("xi_dir", "/XI", pendulumConfig.getXiDir()); Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); - Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); + //Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); Assert.assertEquals("mwm", 4, pendulumConfig.getMwm()); //Assert.assertEquals("coo", "TTTTTTTTT", pendulumConfig.getValidatorManagerAddress()); Assert.assertEquals("validate testnet milestone signatures", true, @@ -346,7 +346,7 @@ public void iniParsingMainnetTest() throws Exception { .append("[PENDULUM]").append(System.lineSeparator()) .append("PORT = 8088").append(System.lineSeparator()) .append("NEIGHBORS = udp://neighbor1 neighbor, tcp://neighbor2").append(System.lineSeparator()) - .append("ZMQ_ENABLED = true").append(System.lineSeparator()) + //.append("ZMQ_ENABLED = true").append(System.lineSeparator()) .append("P_REMOVE_REQUEST = 0.4").append(System.lineSeparator()) .append("MWM = 4").append(System.lineSeparator()) .append("FAKE").append(System.lineSeparator()) @@ -362,7 +362,7 @@ public void iniParsingMainnetTest() throws Exception { Assert.assertEquals("PORT", 8088, pendulumConfig.getApiPort()); Assert.assertEquals("NEIGHBORS", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); - Assert.assertEquals("ZMQ_ENABLED", true, pendulumConfig.isZmqEnabled()); + //Assert.assertEquals("ZMQ_ENABLED", true, pendulumConfig.isZmqEnabled()); Assert.assertEquals("P_REMOVE_REQUEST", 0.4d, pendulumConfig.getpRemoveRequest(), 0); Assert.assertNotEquals("MWM", 4, pendulumConfig.getMwm()); } @@ -373,7 +373,7 @@ public void iniParsingTestnetTest() throws Exception { .append("[PENDULUM]").append(System.lineSeparator()) .append("PORT = 8088").append(System.lineSeparator()) .append("NEIGHBORS = udp://neighbor1 neighbor, tcp://neighbor2").append(System.lineSeparator()) - .append("ZMQ_ENABLED = true").append(System.lineSeparator()) + //.append("ZMQ_ENABLED = true").append(System.lineSeparator()) .append("DNS_RESOLUTION_ENABLED = true").append(System.lineSeparator()) .append("P_REMOVE_REQUEST = 0.4").append(System.lineSeparator()) .append("MWM = 4").append(System.lineSeparator()) @@ -395,7 +395,7 @@ public void iniParsingTestnetTest() throws Exception { Assert.assertEquals("PORT", 8088, pendulumConfig.getApiPort()); Assert.assertEquals("NEIGHBORS", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); - Assert.assertEquals("ZMQ_ENABLED", true, pendulumConfig.isZmqEnabled()); + //Assert.assertEquals("ZMQ_ENABLED", true, pendulumConfig.isZmqEnabled()); Assert.assertEquals("DNS_RESOLUTION_ENABLED", true, pendulumConfig.isDnsResolutionEnabled()); //true by default Assert.assertEquals("DNS_REFRESHER_ENABLED", true, pendulumConfig.isDnsRefresherEnabled()); @@ -437,7 +437,7 @@ public void backwardsIniCompatibilityTest() { // make it explicit that we have removed some configs // in some cases, we have renamed the config param (to e.g. fix double negative variable names) .filter(config -> !ArrayUtils.contains(new String[]{"CONFIG", "TESTNET", "DEBUG", - "MIN_RANDOM_WALKS", "MAX_RANDOM_WALKS", "DONT_VALIDATE_TESTNET_MILESTONE_SIG", + "MIN_RANDOM_WALKS", "MAX_RANDOM_WALKS", "DONT_VALIDATE_TESTNET_MILESTONE_SIG", "ZMQ_ENABLED", "COORDINATOR", "REMOTE_LIMIT_API", "MWM", "TIPSELECTION_ALPHA"}, config)) .forEach(config -> Assert.assertThat(configNames, IsCollectionContaining.hasItem(config)) diff --git a/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java b/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java index 37250018..61197b51 100644 --- a/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java +++ b/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java @@ -7,7 +7,8 @@ public class ZMQConfigTest { - @Test + // We have removed the zmq_enabled param completely. + //@Test public void isZmqEnabledLegacy() { String[] args = { "--zmq_enabled", "true", diff --git a/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java b/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java index ec358342..92658c5d 100644 --- a/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java +++ b/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java @@ -2,6 +2,7 @@ import net.helix.pendulum.TransactionTestUtils; import net.helix.pendulum.conf.ConfigFactory; +import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.model.Hash; import java.util.LinkedList; @@ -22,8 +23,9 @@ public class SpentAddressesProviderImplTest { @BeforeClass public static void setUp() throws Exception { + PendulumConfig config = ConfigFactory.createPendulumConfig(true); provider = new SpentAddressesProviderImpl(); - provider.init(ConfigFactory.createPendulumConfig(true)); + provider.init(config); } @AfterClass From e25f269b85581b63a3ed58ac2ce135e561fcb557 Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Wed, 4 Dec 2019 13:53:27 +0100 Subject: [PATCH 33/36] Update changelog --- CHANGELOG.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed8e0cf8..b66b928c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,21 @@ -## 1.0.6 +## 1.0.6 - Configuration flag names used on command line (by Jcommander) were changed from kabab-case to snake_case to match the config.ini style (used by Jackson). - Renaming/Refactoring config variables to identify explicit purpose for devops -- Renaming/Refactoring double negative variable names for reader sanity +- Renaming/Refactoring double negative variable names for reader sanity +- Previous versions did not distinguish between testnet and mainnet db and db-log paths for spent-addresses. This was fixed. Depending on the net setting (**{test}**net vs. **{main}**net), by default, pendulum state is stored in: +``` +{x}net-db-log/ +{x}net-db/ +{x}net-snapshot/ +{x}net-spent-addresses-db-log/ +{x}net-spent-addresses-db/ +``` +Additional state is stored in the ./logs directory. + ## 1.0.5 - Fixed `getBalance`: `RoundViewModel.get()` returns null on `index`=0, thus NPE was thrown when `references` were not passed and the first round hadn't been completed. In our implementation the snapshot is already constructed based on relative confirmations, thus it suffices for `getBalances` to respond with balance according to `latestSnapshot`. -- Updated `previousEpochsSpentAddresses` resource files +- Updated `previousEpochsSpentAddresses` resource files ## 1.0.4 - Fixed several zmq publish statements in which an incorrect format was specified. From 8df644d771c0409da9fe3bc6d66cd904ef39f77a Mon Sep 17 00:00:00 2001 From: Dan Cook Date: Wed, 4 Dec 2019 14:41:32 +0100 Subject: [PATCH 34/36] remove irrelevant test variable --- .../spentaddresses/impl/SpentAddressesProviderImpl.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java index f3de7585..3477a16f 100644 --- a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -35,9 +35,6 @@ public class SpentAddressesProviderImpl implements SpentAddressesProvider { private ConsensusConfig config; -// private String hello = config.getSpentAddressesDbPath(); -// private String kitty = config.getSpentAddressesDbLogPath(); - /** * Creates a new instance of SpentAddressesProvider */ From f5e0f93891189db8ff4bf08878ebf038cb5d0aa5 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Wed, 4 Dec 2019 18:22:30 +0300 Subject: [PATCH 35/36] read round duration from config --- .../pendulum/service/snapshot/impl/SnapshotServiceImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java index 96fa0f4a..bddd0641 100644 --- a/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java @@ -189,8 +189,8 @@ public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws Sna // todo: unfortunately we need to have getRound in milestoneTracker and here, as RoundIndexUtils is static and we need to check isTestnet. public int getRound(long time) { return config.isTestnet() ? - RoundIndexUtil.getRound(time, TestnetConfig.Defaults.GENESIS_TIME, BasePendulumConfig.Defaults.ROUND_DURATION) : - RoundIndexUtil.getRound(time, BasePendulumConfig.Defaults.GENESIS_TIME, BasePendulumConfig.Defaults.ROUND_DURATION); + RoundIndexUtil.getRound(time, TestnetConfig.Defaults.GENESIS_TIME, config.getRoundDuration()) : + RoundIndexUtil.getRound(time, BasePendulumConfig.Defaults.GENESIS_TIME, config.getRoundDuration()); } /** From ea659043acf28952df9ce515f6b1721042c50d93 Mon Sep 17 00:00:00 2001 From: dzhelezov Date: Thu, 5 Dec 2019 19:15:56 +0300 Subject: [PATCH 36/36] fix unjustified validity check of the tail, causing inconsistent balance errors --- src/main/java/net/helix/pendulum/BundleValidator.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/helix/pendulum/BundleValidator.java b/src/main/java/net/helix/pendulum/BundleValidator.java index 6bb560e3..03e9e76d 100644 --- a/src/main/java/net/helix/pendulum/BundleValidator.java +++ b/src/main/java/net/helix/pendulum/BundleValidator.java @@ -71,10 +71,10 @@ public static List> validate(Tangle tangle, Snapshot return Collections.emptyList(); } - if (tail.getValidity() == -1) { - log.trace("{} is not valid", tail); - return Collections.emptyList(); - } +// if (tail.getValidity() == -1) { +// log.trace("{} is not valid", tail); +// return Collections.emptyList(); +// } List> transactions = new LinkedList<>(); final Map bundleTransactions = loadTransactionsFromTangle(tangle, tail);