diff --git a/.gitignore b/.gitignore index c689f2cb..82853dec 100644 --- a/.gitignore +++ b/.gitignore @@ -46,6 +46,17 @@ spent-addresses-log *.snapshot.meta.bkp *.snapshot.state.bkp snapshot-*/* +mainnet-db-log/ +mainnet-db/ +mainnet-snapshot/ +mainnet-spent-addresses-db-log/ +mainnet-spent-addresses-db/ +testnet-db-log/ +testnet-db/ +testnet-snapshot/ +testnet-spent-addresses-db-log/ +testnet-spent-addresses-db/ + # modules XI @@ -67,3 +78,4 @@ Validator.* # os .DS_Store +Makefile diff --git a/CHANGELOG.md b/CHANGELOG.md index 6261b36a..b66b928c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,21 @@ +## 1.0.6 +- Configuration flag names used on command line (by Jcommander) were changed from kabab-case to snake_case to match the config.ini style (used by Jackson). +- Renaming/Refactoring config variables to identify explicit purpose for devops +- Renaming/Refactoring double negative variable names for reader sanity +- Previous versions did not distinguish between testnet and mainnet db and db-log paths for spent-addresses. This was fixed. Depending on the net setting (**{test}**net vs. **{main}**net), by default, pendulum state is stored in: +``` +{x}net-db-log/ +{x}net-db/ +{x}net-snapshot/ +{x}net-spent-addresses-db-log/ +{x}net-spent-addresses-db/ +``` +Additional state is stored in the ./logs directory. + + ## 1.0.5 - Fixed `getBalance`: `RoundViewModel.get()` returns null on `index`=0, thus NPE was thrown when `references` were not passed and the first round hadn't been completed. In our implementation the snapshot is already constructed based on relative confirmations, thus it suffices for `getBalances` to respond with balance according to `latestSnapshot`. -- Updated `previousEpochsSpentAddresses` resource files +- Updated `previousEpochsSpentAddresses` resource files ## 1.0.4 - Fixed several zmq publish statements in which an incorrect format was specified. diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index efe304ba..00000000 --- a/Jenkinsfile +++ /dev/null @@ -1,28 +0,0 @@ -pipeline { - agent { - docker { - image 'maven:3.6-jdk-8-slim' - args '-v /root/.m2:/root/.m2' - } - } - options { - skipStagesAfterUnstable() - } - stages { - stage('Build') { - steps { - sh 'mvn -B -DskipTests clean package' - } - } - stage('Test') { - steps { - sh 'mvn integration-test -Dlogging-level=INFO' - } - post { - always { - junit 'target/surefire-reports/*.xml' - } - } - } - } -} diff --git a/Makefile b/Makefile deleted file mode 100644 index 398147ef..00000000 --- a/Makefile +++ /dev/null @@ -1,25 +0,0 @@ -clean: - @echo "Removing all data from pendulum for fresh start!" - @rm -rf db - @rm -rf mainnet_snapshots - @rm -rf testnet_snapshots - @rm -rf logs - @rm -rf snapshots - @rm -rf local_snapshots - @rm -rf modules - @rm -rf snapshot - @rm -rf mainnet - @rm -rf testnet - @rm -rf spent-addresses-db - @rm -rf spent-addresses-log - @rm -rf mainnetdb - @rm -rf mainnet.log - @rm -rf testnetdb - @rm -rf testnet.log - @rm -rf snapshot-mainnet - @rm -rf snapshot-testnet - @rm -rf logs -start-node: - @echo "Starting pendulum" - @java -jar target/pen*.jar --config config.ini - diff --git a/README.md b/README.md index 170fdcbf..af551eff 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Pendulum is a quorum based [Tangle](https://github.com/iotaledger/iri/) implementation designed towards reliable timekeeping and high-throughput messaging. -- **Latest release:** 1.0.5 pre-release +- **Latest release:** 1.0.6 pre-release - **License:** GPLv3 Special thanks to all of the [IOTA Contributors](https://github.com/iotaledger/iri/graphs/contributors)! @@ -196,6 +196,7 @@ Currently the following topics are covered: | `nav` | newly added validators | | `nrv` | newly removed validators | | `cvs` | current validators | +| `sldf` | transactions recently solidified by the node | | `
` | Watching all traffic on a specified address | diff --git a/docker/docker-compose-dev.yml b/docker/docker-compose-dev.yml new file mode 100644 index 00000000..60dd4933 --- /dev/null +++ b/docker/docker-compose-dev.yml @@ -0,0 +1,66 @@ +version: "3.1" + +networks: + helix_network: + +secrets: + seed: + file: .seed # + +services: + relayer: + container_name: relayer + # build first + image: ${DOCKER_IMAGE} + hostname: relayer + restart: on-failure + volumes: + - ../target/pendulum-1.0.3.jar:/pendulum/target/pendulum-dev.jar + # uncomment if you need persistent data + # - /docker-volumes/relayer/data:/pendulum/data + environment: + - DOCKER_PLM_JAR_FILE=pendulum-dev.jar + - DOCKER_PLM_REMOTE_LIMIT_API="interruptAttachToTangle" + - JAVA_MAX_MEMORY=500m + - JAVA_MIN_MEMORY=256m + - DOCKER_PLM_LOGGING_LEVEL=debug + - JAVA_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5044 + ports: + - "6550:6550" + - "8085:8085" + - "5044:5044" + expose: + - "4100" + command: ["-p","8085","-n", "udp://backend:4100", "--testnet"] + networks: + helix_network: + + backend: + container_name: backend + image: ${DOCKER_IMAGE} + hostname: backend + restart: on-failure + volumes: + - ../target/pendulum-1.0.3.jar:/pendulum/target/pendulum-dev.jar + #volumes: + # - /docker-volumes/backend/data:/pendulum/data + # - ./backend/seed.txt:/pendulum/conf/seed.txt:ro + environment: + - DOCKER_PLM_REMOTE_LIMIT_API="interruptAttachToTangle" + - DOCKER_PLM_JAR_FILE=pendulum-dev.jar + - JAVA_MAX_MEMORY=500m + - JAVA_MIN_MEMORY=256m + - DOCKER_PLM_LOGGING_LEVEL=debug + - JAVA_OPTIONS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=*:5045 + ports: + - "6551:6550" + - "8086:8085" + - "5045:5045" + expose: + - "4100" + secrets: + - seed + command: ["-p","8085","-n", "udp://relayer:4100", "--testnet", "--validator", "/run/secrets/seed"] + networks: + helix_network: + diff --git a/docker/run-dev.sh b/docker/run-dev.sh new file mode 100644 index 00000000..7a25aeaa --- /dev/null +++ b/docker/run-dev.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +usage() { + echo "Usage (local build + run): run-dev --build --image " + echo "Usage (git build + run): run-dev --git --image " + echo "Usage (pull + run): run-dev --image " + echo "Usage (local jar): run-dev --jar " + echo " (optional) with validator seed: --seed " +} + +run() { + echo "killing compose" + docker-compose -f docker-compose-dev.yml down + echo "spinning up compose" + docker-compose -f docker-compose-dev.yml up -d +} + +restart() { + echo "restarting" + docker-compose -f docker-compose-dev.yml restart relayer + docker-compose -f docker-compose-dev.yml restart backend +} + +# if no arguments supplied, display usage +if [ $# -le 0 ] +then + usage + exit 1 +fi + +build= +DOCKER_IMAGE= + +while [ "$1" != "" ]; do + case $1 in + -i | --image ) shift + DOCKER_IMAGE=$1 + ;; + -b | --build ) build=1 + ;; + --restart ) restart + exit + ;; + --local ) jar=1 + ;; + --git ) git=$1 + ;; + --seed ) echo "$1" > .seed + ;; + -h | --help ) usage + exit + ;; + * ) usage + exit 1 + esac + shift +done + +if [ "$jar" = "1" ]; then + echo "Building jar only, skipping tests" + mvn jar:jar -Dmaven.test.skip=true + run +fi + +if [ $DOCKER_IMAGE = "" ] +then + usage + exit 1 +fi + +export DOCKER_IMAGE=$DOCKER_IMAGE + +if [ "$build" = "1" ]; then + echo "building $DOCKER_IMAGE" + docker build -t $DOCKER_IMAGE ../ +fi + +if [ "$git" != "" ]; then + echo "building from github: $git" + docker build -t $DOCKER_IMAGE $git +fi + +run \ No newline at end of file diff --git a/pom.xml b/pom.xml index 2190b68d..c96603b4 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ pendulum - 1.0.5 + 1.0.6 Pendulum diff --git a/src/main/java/net/helix/pendulum/BundleValidator.java b/src/main/java/net/helix/pendulum/BundleValidator.java index f5e26430..03e9e76d 100644 --- a/src/main/java/net/helix/pendulum/BundleValidator.java +++ b/src/main/java/net/helix/pendulum/BundleValidator.java @@ -6,15 +6,14 @@ import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.crypto.Winternitz; import net.helix.pendulum.model.Hash; +import net.helix.pendulum.service.ValidationException; import net.helix.pendulum.service.snapshot.Snapshot; import net.helix.pendulum.storage.Tangle; +import org.bouncycastle.util.encoders.Hex; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; +import java.util.*; /** * Validates bundles. @@ -26,6 +25,8 @@ */ public class BundleValidator { + private final static Logger log = LoggerFactory.getLogger(BundleValidator.class); + /** * Fetches a bundle of transactions identified by the {@code tailHash} and validates the transactions. * Bundle is a group of transactions with the same bundle hash chained by their trunks. @@ -65,129 +66,152 @@ public class BundleValidator { public static List> validate(Tangle tangle, Snapshot initialSnapshot, Hash tailHash) throws Exception { TransactionViewModel tail = TransactionViewModel.fromHash(tangle, tailHash); - if (tail.getCurrentIndex() != 0 || tail.getValidity() == -1) { - return Collections.EMPTY_LIST; + if (tail.getCurrentIndex() != 0) { + log.trace("{} is not a tail", tail); + return Collections.emptyList(); } +// if (tail.getValidity() == -1) { +// log.trace("{} is not valid", tail); +// return Collections.emptyList(); +// } + List> transactions = new LinkedList<>(); final Map bundleTransactions = loadTransactionsFromTangle(tangle, tail); - //we don't really iterate, we just pick the tail tx. See the if on the next line - for (TransactionViewModel transactionViewModel : bundleTransactions.values()) { - - if (transactionViewModel.getCurrentIndex() == 0 && transactionViewModel.getValidity() >= 0) { - - final List instanceTransactionViewModels = new LinkedList<>(); - - final long lastIndex = transactionViewModel.lastIndex(); - long bundleValue = 0; - int i = 0; - final Sponge sha3Instance = SpongeFactory.create(SpongeFactory.Mode.S256); - final Sponge addressInstance = SpongeFactory.create(SpongeFactory.Mode.S256); - - final byte[] addressBytes = new byte[TransactionViewModel.ADDRESS_SIZE]; - final byte[] bundleHashBytes = new byte[TransactionViewModel.BUNDLE_SIZE]; - byte[] normalizedBundle = new byte[TransactionViewModel.BUNDLE_SIZE]; - byte[] digestBytes = new byte[Sha3.HASH_LENGTH]; - - //here we iterate over the txs by checking the trunk of the current transaction - MAIN_LOOP: - while (true) { - - instanceTransactionViewModels.add(transactionViewModel); - - //semantic checks - if ( - transactionViewModel.getCurrentIndex() != i - || transactionViewModel.lastIndex() != lastIndex - || ((bundleValue = Math.addExact(bundleValue, transactionViewModel.value())) < -TransactionViewModel.SUPPLY - || bundleValue > TransactionViewModel.SUPPLY) - ) { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - break; - } - - // It's supposed to become -3812798742493 after 3812798742493 and to go "down" to -1 but - // we hope that no one will create such long bundles - if (i++ == lastIndex) { - - if (bundleValue == 0) { - - if (instanceTransactionViewModels.get(0).getValidity() == 0) { - sha3Instance.reset(); - for (final TransactionViewModel transactionViewModel2 : instanceTransactionViewModels) { - sha3Instance.absorb(transactionViewModel2.getBytes(), TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_SIZE); - } - sha3Instance.squeeze(bundleHashBytes, 0, bundleHashBytes.length); - //verify bundle hash is correct - if (Arrays.equals(instanceTransactionViewModels.get(0).getBundleHash().bytes(), bundleHashBytes)) { - //normalizing the bundle in preparation for signature verification - normalizedBundle = Winternitz.normalizedBundle(bundleHashBytes); - - int offset = 0; - for (int j = 0; j < instanceTransactionViewModels.size(); ) { - - transactionViewModel = instanceTransactionViewModels.get(j); - //if it is a spent transaction that should be signed - if (transactionViewModel.value() < 0) { - // let's verify the signature by recalculating the public address - addressInstance.reset(); - do { - digestBytes = Winternitz.digest(SpongeFactory.Mode.S256, Arrays.copyOfRange(normalizedBundle, offset*Winternitz.NORMALIZED_FRAGMENT_LENGTH, (offset+1)*Winternitz.NORMALIZED_FRAGMENT_LENGTH), - Arrays.copyOfRange(instanceTransactionViewModels.get(j).getBytes(), 0, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE)); - addressInstance.absorb(digestBytes,0, Sha3.HASH_LENGTH); - offset++; - } //loop to traverse signature fragments divided between transactions - while (++j < instanceTransactionViewModels.size() - && instanceTransactionViewModels.get(j).getAddressHash().equals(transactionViewModel.getAddressHash()) - && instanceTransactionViewModels.get(j).value() == 0); - - addressInstance.squeeze(addressBytes, 0, addressBytes.length); - //signature verification - if (! Arrays.equals(transactionViewModel.getAddressHash().bytes(), addressBytes)) { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - break MAIN_LOOP; - } - } else { - j++; - } - } - //should only be reached after the above for loop is done - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, 1); - transactions.add(instanceTransactionViewModels); - } - //bundle hash verification failed - else { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - } - } - //bundle validity status is known - else { - transactions.add(instanceTransactionViewModels); - } - } - //total bundle value does not sum to 0 - else { - instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); - } - //break from main loop - break; - - } - //traverse to the next tx in the bundle - else { - transactionViewModel = bundleTransactions.get(transactionViewModel.getTrunkTransactionHash()); - if (transactionViewModel == null) { - //we found all the transactions and we can now return - break; - } - } - } - } + LinkedList sortedTxs; + try { + sortedTxs = validateOrder(bundleTransactions.values()); + validateValue(sortedTxs); + validateBundleHash(sortedTxs); + validateSignatures(sortedTxs); + } catch (ValidationException ve) { + tail.setValidity(tangle, initialSnapshot, -1); + log.warn("Bundle validation exception", ve); + return Collections.emptyList(); } + + tail.setValidity(tangle, initialSnapshot, 1); + transactions.add(sortedTxs); + + //we don't really iterate, we just pick the tail tx. See the if on the next line +// for (TransactionViewModel transactionViewModel : bundleTransactions.values()) { +// +// if (transactionViewModel.getCurrentIndex() == 0 && transactionViewModel.getValidity() >= 0) { +// +// final List instanceTransactionViewModels = new LinkedList<>(); +// +// final long lastIndex = tail.lastIndex(); +// long bundleValue = 0; +// int i = 0; +// final Sponge sha3Instance = SpongeFactory.create(SpongeFactory.Mode.S256); +// final Sponge addressInstance = SpongeFactory.create(SpongeFactory.Mode.S256); +// +// final byte[] addressBytes = new byte[TransactionViewModel.ADDRESS_SIZE]; +// final byte[] bundleHashBytes = new byte[TransactionViewModel.BUNDLE_SIZE]; +// byte[] normalizedBundle = new byte[TransactionViewModel.BUNDLE_SIZE]; +// byte[] digestBytes = new byte[Sha3.HASH_LENGTH]; +// +// //here we iterate over the txs by checking the trunk of the current transaction +// MAIN_LOOP: +// while (true) { +// +// instanceTransactionViewModels.add(transactionViewModel); +// +// //semantic checks +// if ( +// transactionViewModel.getCurrentIndex() != i +// || transactionViewModel.lastIndex() != lastIndex +// || ((bundleValue = Math.addExact(bundleValue, transactionViewModel.value())) < -TransactionViewModel.SUPPLY +// || bundleValue > TransactionViewModel.SUPPLY) +// ) { +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); +// break; +// } +// +// // It's supposed to become -3812798742493 after 3812798742493 and to go "down" to -1 but +// // we hope that no one will create such long bundles +// if (i++ == lastIndex) { +// +// if (bundleValue == 0) { +// +// if (instanceTransactionViewModels.get(0).getValidity() == 0) { +// sha3Instance.reset(); +// for (final TransactionViewModel transactionViewModel2 : instanceTransactionViewModels) { +// sha3Instance.absorb(transactionViewModel2.getBytes(), TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_SIZE); +// } +// sha3Instance.squeeze(bundleHashBytes, 0, bundleHashBytes.length); +// //verify bundle hash is correct +// if (Arrays.equals(instanceTransactionViewModels.get(0).getBundleHash().bytes(), bundleHashBytes)) { +// //normalizing the bundle in preparation for signature verification +// normalizedBundle = Winternitz.normalizedBundle(bundleHashBytes); +// +// int offset = 0; +// for (int j = 0; j < instanceTransactionViewModels.size(); ) { +// +// transactionViewModel = instanceTransactionViewModels.get(j); +// //if it is a spent transaction that should be signed +// if (transactionViewModel.value() < 0) { +// // let's verify the signature by recalculating the public address +// addressInstance.reset(); +// do { +// digestBytes = Winternitz.digest(SpongeFactory.Mode.S256, Arrays.copyOfRange(normalizedBundle, offset*Winternitz.NORMALIZED_FRAGMENT_LENGTH, (offset+1)*Winternitz.NORMALIZED_FRAGMENT_LENGTH), +// Arrays.copyOfRange(instanceTransactionViewModels.get(j).getBytes(), 0, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE)); +// addressInstance.absorb(digestBytes,0, Sha3.HASH_LENGTH); +// offset++; +// } //loop to traverse signature fragments divided between transactions +// while (++j < instanceTransactionViewModels.size() +// && instanceTransactionViewModels.get(j).getAddressHash().equals(transactionViewModel.getAddressHash()) +// && instanceTransactionViewModels.get(j).value() == 0); +// +// addressInstance.squeeze(addressBytes, 0, addressBytes.length); +// //signature verification +// if (! Arrays.equals(transactionViewModel.getAddressHash().bytes(), addressBytes)) { +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); +// break MAIN_LOOP; +// } +// } else { +// j++; +// } +// } +// //should only be reached after the above for loop is done +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, 1); +// transactions.add(instanceTransactionViewModels); +// } +// //bundle hash verification failed +// else { +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); +// } +// } +// //bundle validity status is known +// else { +// transactions.add(instanceTransactionViewModels); +// } +// } +// //total bundle value does not sum to 0 +// else { +// instanceTransactionViewModels.get(0).setValidity(tangle, initialSnapshot, -1); +// } +// //break from main loop +// break; +// +// } +// //traverse to the next tx in the bundle +// else { +// transactionViewModel = bundleTransactions.get(transactionViewModel.getTrunkTransactionHash()); +// if (transactionViewModel == null) { +// //we found all the transactions and we can now return +// break; +// } +// } +// } +// } +// } return transactions; } + + /** * Checks that the bundle's inputs and outputs are balanced. * @@ -233,4 +257,146 @@ private static Map loadTransactionsFromTangle(Tangle } return bundleTransactions; } + + /** + * Sorts the provided list according to the index + * + * @param bundleTxs unordered list of transactions + * @return List of TransactionViewModel in the bundle, from starting from currentIndex == 0 to + * currentIndex == tail.lastIndex() + * + * @throws ValidationException if there is a missing index or the size does not match the last index + */ + private static LinkedList validateOrder(Collection bundleTxs) { + SortedSet set = new TreeSet<>(Comparator.comparing(TransactionViewModel::getCurrentIndex)); + set.addAll(bundleTxs); + LinkedList sorted = new LinkedList<>(set); + int index = 0; + TransactionViewModel lastTx = sorted.getFirst(); + for (TransactionViewModel txvm : sorted) { + if (txvm.getCurrentIndex() != index) { + throw new ValidationException(String.format("%s does not match index %d", txvm.toString(), index)); + } + if (index > 0 && lastTx.getTrunkTransactionHash() != txvm.getHash()) { + throw new ValidationException(String.format("Trunk of %s should be equal to %s", + lastTx.toString(), txvm.getHash())); + } + lastTx = txvm; + index ++; + } + long expectedSize = sorted.getFirst().lastIndex() + 1; + if (sorted.size() != expectedSize) { + throw new ValidationException(String.format("The bundle is incomplete. Expected bundle size: %d", expectedSize)); + } + return sorted; + } + + /** + * + * @param bundleTxs Ordered list of the transaction bundles + * + * @throws ValidationException if the value transfers are out of bounds or the bundle value is inconsistent + * (i.e. cummulative value transfers don't add up to zero) + */ + private static void validateValue(LinkedList bundleTxs) { + long bundleValue = 0; + for (TransactionViewModel txvm : bundleTxs) { + bundleValue = Math.addExact(bundleValue, txvm.value()); + if (bundleValue < -TransactionViewModel.SUPPLY) { + throw new ValidationException("Bundle value is below the negative max supply"); + } + if (bundleValue > TransactionViewModel.SUPPLY) { + throw new ValidationException("Bundle value is above the max supply"); + } + } + + if (bundleValue != 0) { + throw new ValidationException("Bundle transaction values are inconsistent"); + } + } + + /** + * + * @param bundleTxs Ordered list of the bundle transactions + * + * @throws ValidationException If the bundle hash as stored in the tail does not match the actual bundle hash + * as stored in the essense part of the bundle transactions + */ + private static void validateBundleHash(LinkedList bundleTxs) { + if (bundleTxs == null || bundleTxs.size() == 0) { + throw new ValidationException("Bundle txs list is empty"); + } + + Sponge sha3Instance = SpongeFactory.create(SpongeFactory.Mode.S256); + byte[] bundleHashBytes = new byte[TransactionViewModel.BUNDLE_SIZE]; + sha3Instance.reset(); + for (final TransactionViewModel txvm : bundleTxs) { + sha3Instance.absorb(txvm.getBytes(), TransactionViewModel.ESSENCE_OFFSET, TransactionViewModel.ESSENCE_SIZE); + } + sha3Instance.squeeze(bundleHashBytes, 0, bundleHashBytes.length); + TransactionViewModel tail = bundleTxs.getFirst(); + Hash bundleHash = tail.getBundleHash(); + if (bundleHash == null) { + throw new ValidationException("Bundle hash of the tail tx is null: " + tail.toString()); + } + + if (!Arrays.equals(bundleHash.bytes(), bundleHashBytes)) { + throw new ValidationException(String.format("Bundle hash %s does not match calculated hash %s", + Hex.toHexString(bundleHash.bytes()), Hex.toHexString(bundleHashBytes))); + } + } + + /** + * After a spendig txs, a few zero-value txs bearing the signature should follow. + * @param bundleTxs Ordered list of the bundle transactions + * + * @throws ValidationException If the address hashes of the spending transactions + * do not match the public address (Winternitz OTS) + */ + private static void validateSignatures(LinkedList bundleTxs) { + TransactionViewModel tail = bundleTxs.getFirst(); + Hash bundleHash = tail.getBundleHash(); + if (bundleHash == null) { + throw new ValidationException("Bundle hash of the tail tx is null: " + tail.toString()); + } + byte[] bundleHashBytes = bundleHash.bytes(); + final Sponge addressInstance = SpongeFactory.create(SpongeFactory.Mode.S256); + final byte[] addressBytes = new byte[TransactionViewModel.ADDRESS_SIZE]; + byte[] normalizedBundle = Winternitz.normalizedBundle(bundleHashBytes); + int offset = 0; + + ArrayList bundleArray = new ArrayList<>(bundleTxs); + + for (int j = 0; j < bundleArray.size(); ) { + + TransactionViewModel transactionViewModel = bundleArray.get(j); + if (transactionViewModel.value() >= 0) { + j++; + log.trace("{} has non-negative value {}", transactionViewModel.getHash(), transactionViewModel.value()); + continue; + } + //if it is a spent transaction that should be signed + // let's verify the signature by recalculating the public address + addressInstance.reset(); + do { + byte[] digestBytes = Winternitz.digest(SpongeFactory.Mode.S256, + Arrays.copyOfRange(normalizedBundle, offset*Winternitz.NORMALIZED_FRAGMENT_LENGTH, (offset+1)*Winternitz.NORMALIZED_FRAGMENT_LENGTH), + Arrays.copyOfRange(bundleArray.get(j).getBytes(), 0, TransactionViewModel.SIGNATURE_MESSAGE_FRAGMENT_SIZE)); + addressInstance.absorb(digestBytes,0, Sha3.HASH_LENGTH); + offset++; + } //loop to traverse signature fragments divided between transactions + while (++j < bundleArray.size() + && bundleArray.get(j).getAddressHash().equals(transactionViewModel.getAddressHash()) + && bundleArray.get(j).value() == 0); + + addressInstance.squeeze(addressBytes, 0, addressBytes.length); + //signature verification + if (! Arrays.equals(transactionViewModel.getAddressHash().bytes(), addressBytes)) { + throw new ValidationException( + String.format("Signature verification failed: %s does not match %s", + Hex.toHexString(transactionViewModel.getAddressHash().bytes()), Hex.toHexString(addressBytes))); + } + + } + } } diff --git a/src/main/java/net/helix/pendulum/Main.java b/src/main/java/net/helix/pendulum/Main.java index e78443af..3a0770bc 100644 --- a/src/main/java/net/helix/pendulum/Main.java +++ b/src/main/java/net/helix/pendulum/Main.java @@ -46,7 +46,7 @@ public class Main { public static final String MAINNET_NAME = "Pendulum"; public static final String TESTNET_NAME = "Pendulum Testnet"; - public static final String VERSION = "1.0.5"; + public static final String VERSION = "1.0.6"; /** diff --git a/src/main/java/net/helix/pendulum/Pendulum.java b/src/main/java/net/helix/pendulum/Pendulum.java index bd1cb26b..f6498e06 100644 --- a/src/main/java/net/helix/pendulum/Pendulum.java +++ b/src/main/java/net/helix/pendulum/Pendulum.java @@ -100,7 +100,7 @@ public class Pendulum { public final CandidateTrackerImpl candidateTracker; public final LatestSolidMilestoneTrackerImpl latestSolidMilestoneTracker; public final SeenMilestonesRetrieverImpl seenMilestonesRetriever; - public final LedgerServiceImpl ledgerService = new LedgerServiceImpl(); + public final LedgerServiceImpl ledgerService; public final AsyncTransactionPruner transactionPruner; public final MilestoneSolidifierImpl milestoneSolidifier; public final CandidateSolidifierImpl candidateSolidifier; @@ -150,6 +150,8 @@ public Pendulum(PendulumConfig configuration) throws TransactionPruningException : null; transactionRequesterWorker = new TransactionRequesterWorkerImpl(); + ledgerService = new LedgerServiceImpl(); + // legacy code bundleValidator = new BundleValidator(); tangle = new Tangle(); @@ -294,6 +296,7 @@ public void shutdown() throws Exception { transactionValidator.shutdown(); tangle.shutdown(); + spentAddressesProvider.shutdown(); // free the resources of the snapshot provider last because all other instances need it snapshotProvider.shutdown(); } diff --git a/src/main/java/net/helix/pendulum/TransactionValidator.java b/src/main/java/net/helix/pendulum/TransactionValidator.java index 66890dc2..adb93aa8 100644 --- a/src/main/java/net/helix/pendulum/TransactionValidator.java +++ b/src/main/java/net/helix/pendulum/TransactionValidator.java @@ -479,6 +479,7 @@ private boolean quickSetSolid(final TransactionViewModel transactionViewModel) t if(solid) { transactionViewModel.updateSolid(true); transactionViewModel.updateHeights(tangle, snapshotProvider.getInitialSnapshot()); + tangle.publish("sldf %s", transactionViewModel.getHash()); return true; } } diff --git a/src/main/java/net/helix/pendulum/conf/APIConfig.java b/src/main/java/net/helix/pendulum/conf/APIConfig.java index 597987f4..3fb89ff2 100644 --- a/src/main/java/net/helix/pendulum/conf/APIConfig.java +++ b/src/main/java/net/helix/pendulum/conf/APIConfig.java @@ -18,14 +18,14 @@ public interface APIConfig extends Config { String getApiHost(); /** - * @return {@value Descriptions#REMOTE_LIMIT_API} + * @return {@value Descriptions#IGNORED_API_ENDPOINTS} */ - List getRemoteLimitApi(); + List getIgnoredApiEndpoints(); /** - * @return {@value Descriptions#REMOTE_TRUSTED_API_HOSTS} + * @return {@value Descriptions#ALLOWED_API_HOSTS} */ - List getRemoteTrustedApiHosts(); + List getAllowedApiHosts(); /** * @return {@value Descriptions#MAX_FIND_TRANSACTIONS} @@ -60,14 +60,14 @@ public interface APIConfig extends Config { interface Descriptions { String API_PORT = "The port that will be used by the API."; String API_HOST = "The host on which the API will listen to. Set to 0.0.0.0 to accept any host."; - String REMOTE_LIMIT_API = "Commands that should be ignored by API."; - String REMOTE_TRUSTED_API_HOSTS = "Open the API interface to defined hosts. You can specify multiple hosts in a comma separated list \"--remote-trusted-api-hosts 192.168.0.55,10.0.0.10\". You must also provide the \"--remote\" parameter. Warning: \"--remote-limit-api\" will have no effect for these hosts."; + String REMOTE = "Open the API interface to any host. Equivalent to \"--api-host 0.0.0.0\""; + String IGNORED_API_ENDPOINTS = "Commands that should be ignored by API. Note that by default localhost and any allowed api host have access to all endpoints."; + String ALLOWED_API_HOSTS = "Open the API interface to defined hosts. You can specify multiple hosts in a comma separated list \"--allowed_api_hosts 192.168.0.55,10.0.0.10\". You must also provide the \"--remote\" parameter. Warning: \"--ignored_api_endpoints\" will have no effect for these hosts."; String REMOTE_AUTH = "A string in the form of :. Used to access the API"; String MAX_FIND_TRANSACTIONS = "The maximal number of transactions that may be returned by the \"findTransactions\" API call. If the number of transactions found exceeds this number an error will be returned."; String MAX_REQUESTS_LIST = "The maximal number of parameters one can place in an API call. If the number parameters exceeds this number an error will be returned"; String MAX_GET_TRANSACTION_STRINGS = "The maximal number of transaction strings that may be returned by the \"getTransactionStrings\" API call. If the number of transactions found exceeds this number an error will be returned."; String MAX_BODY_LENGTH = "The maximal number of characters the body of an API call may hold. If a request body length exceeds this number an error will be returned."; - String REMOTE = "Open the API interface to any host. Equivalent to \"--api-host 0.0.0.0\""; String RESOURCE_PATH = "Resource path"; } } \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java index b6763176..0dc064ed 100644 --- a/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java +++ b/src/main/java/net/helix/pendulum/conf/BasePendulumConfig.java @@ -18,31 +18,183 @@ import java.nio.file.Paths; import java.util.*; import java.util.stream.Collectors; -/* - Note: the fields in this class are being deserialized from Jackson so they must follow Java Bean convention. - Meaning that every field must have a getter that is prefixed with `get` unless it is a boolean and then it should be - prefixed with `is`. -*/ + public abstract class BasePendulumConfig implements PendulumConfig { + /* + Pendulum default configuration values + Some of these values are not really "configurable" in the sense that the user can't change them using command + line flags, or params in the pendulum.ini file. Instead, if the user wanted to change them, the user would + need to alter the defaults found in this interface and recompile the source code. This is not recommended + since Pendulum has been tested under different conditions to ensure critical network properties with these + default values with different host machines operating together in cooperation to achieve a common goal. + + On the other hand, if your goal is to break the system, then go a head and change the default values. + However, note that the network will quickly discover your deviation and kick you out of the p2p system. + + Happy hacking! ;) + */ + public interface Defaults { + //API + int API_PORT = 8085; + String API_HOST = "localhost"; + List IGNORED_API_ENDPOINTS = PendulumUtils.createImmutableList(); + InetAddress ALLOWED_API_DEFAULT_HOST = InetAddress.getLoopbackAddress(); + List ALLOWED_API_HOSTS = PendulumUtils.createImmutableList(ALLOWED_API_DEFAULT_HOST); + int MAX_FIND_TRANSACTIONS = 100_000; + int MAX_REQUESTS_LIST = 1_000; + int MAX_GET_TRANSACTION_STRINGS = 10_000; + int MAX_BODY_LENGTH = 1_000_000; + String REMOTE_AUTH = ""; + + //Network + int UDP_RECEIVER_PORT = 4100; + int TCP_RECEIVER_PORT = 5100; + double P_REMOVE_REQUEST = 0.01d; + int SEND_LIMIT = -1; + int MAX_PEERS = 0; + boolean DNS_REFRESHER_ENABLED = true; + boolean DNS_RESOLUTION_ENABLED = true; + + //XI + String XI_DIR = "modules"; + + //DB + String DB_PATH = "mainnet-db"; + String DB_LOG_PATH = "mainnet-db-log"; + int DB_CACHE_SIZE = 100_000; + String ROCKS_DB = "rocksdb"; + boolean REVALIDATE = false; + boolean RESCAN_DB = false; + + //Protocol + double P_REPLY_RANDOM_TIP = 0.66d; + double P_DROP_TRANSACTION = 0d; + double P_SELECT_MILESTONE_CHILD = 0.7d; + double P_SEND_MILESTONE = 0.02d; + double P_PROPAGATE_REQUEST = 0.01d; + int MWM = 1; + int PACKET_SIZE = 800; + int REQ_HASH_SIZE = 32; + int QUEUE_SIZE = 1_000; + double P_DROP_CACHE_ENTRY = 0.02d; + int CACHE_SIZE_BYTES = 150_000; + + //Zmq + int ZMQ_THREADS = 1; + boolean ZMQ_ENABLE_IPC = false; + String ZMQ_IPC = "ipc://hlx"; + boolean ZMQ_ENABLE_TCP = false; + int ZMQ_PORT = 5556; + + //TipSel + int MAX_DEPTH = 15; + double ALPHA = 0.001d; + + //Tip solidification + boolean TIP_SOLIDIFIER_ENABLED = true; + + //PoW + int POW_THREADS = 8; + + //Resource directory: + String RESOUCER_PATH = "./src/main/resources"; + String DEFAULT_RESOUCE_PATH = "./resources"; + + //Validator Manager + boolean VALIDATOR_MANAGER_ENABLED = false; + Hash VALIDATOR_MANAGER_ADDRESS = HashFactory.ADDRESS.create( + "9474289ae28f0ea6e3b8bedf8fc52f14d2fa9528a4eb29d7879d8709fd2f6d37" + ); + int UPDATE_VALIDATOR_DELAY = 30000; + int START_ROUND_DELAY = 2; + String VALIDATOR_MANAGER_KEYFILE = "/ValidatorManager.key"; + int VALIDATOR_MANAGER_KEY_DEPTH = 15; + int VALIDATOR_MANAGER_SECURITY = 2; + + //Milestone + boolean VALIDATOR = false; + String VALIDATOR_PATH = null; + Set INITIAL_VALIDATORS = new HashSet<>(Arrays.asList( + HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832"), + HashFactory.ADDRESS.create("a5afe01e64ae959f266b382bb5927fd07b49e7e3180239535126844aaae9bf93"), + HashFactory.ADDRESS.create("e2debe246b5d1a6e05b57b0fc14edb51d136966a91a803b523586ad032f72f3d"), + HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), + HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), + HashFactory.ADDRESS.create("1c6b0ee311a7ddccf255c1097995714b285cb06628be1cef2080b0bef7700e12"), + HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832") + )); + + long GENESIS_TIME = 1569024001000L; + int ROUND_DURATION = 15000; + int ROUND_PAUSE = 5000; + String VALIDATOR_KEYFILE = "/Validator.key"; + String VALIDATOR_SEED_PATH = "/Validator.txt"; + int MILESTONE_KEY_DEPTH = 10; + int VALIDATOR_SECURITY = 2; + int NUMBER_OF_ACTIVE_VALIDATORS = 1; + double CONFIRMATION_THRESHOLD = 0.66; + + // bootstrap hash for empty round + Hash EMPTY_ROUND_HASH = HashFactory.ADDRESS.create( + "00000000000000000000000000000000000000000000656d707479726f756e64" + ); + + //Snapshot + boolean LOCAL_SNAPSHOTS_ENABLED = true; + boolean LOCAL_SNAPSHOTS_PRUNING_ENABLED = false; + int LOCAL_SNAPSHOTS_PRUNING_DELAY = 50000; + int LOCAL_SNAPSHOTS_INTERVAL_SYNCED = 10; + int LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = 1000; + String LOCAL_SNAPSHOTS_BASE_PATH = "./mainnet-snapshot"; + int LOCAL_SNAPSHOTS_DEPTH = 100; + String SNAPSHOT_FILE = "/snapshotMainnet.txt"; + String SNAPSHOT_SIG_FILE = "/snapshotMainnet.sig"; + long GLOBAL_SNAPSHOT_TIME = 1522235533L; + int MILESTONE_START_INDEX = 0; + int NUM_KEYS_IN_MILESTONE = 10; + int MAX_ANALYZED_TXS = 20_000; + + //Spent-addresses + String SPENT_ADDRESSES_DB_PATH = "mainnet-spent-addresses-db"; + String SPENT_ADDRESSES_DB_LOG_PATH = "mainnet-spent-addresses-db-log"; + String PREVIOUS_EPOCHS_SPENT_ADDRESSES_TXT = "/previousEpochsSpentAddresses.txt"; + String PREVIOUS_EPOCHS_SPENT_ADDRESSES_SIG = "/previousEpochsSpentAddresses.sig"; + + + //Logging + boolean SAVELOG_ENABLED = false; + String SAVELOG_BASE_PATH = "./logs/"; + String SAVELOG_XML_FILE = "/logback-save.xml"; + } + + // ******************************************* + // OUR IMPLEMENTATION OF THE CONFIG INTERFACES + // ******************************************* + protected static final String SPLIT_STRING_TO_LIST_REGEX = ",| "; private boolean help; + // PUBLIC SERVICE RELATED CONFIGURATION //API protected int apiPort = Defaults.API_PORT; protected String apiHost = Defaults.API_HOST; - protected List remoteLimitApi = Defaults.REMOTE_LIMIT_API; - protected List remoteTrustedApiHosts = Defaults.REMOTE_LIMIT_API_HOSTS; + protected List ignoredApiEndpoints = Defaults.IGNORED_API_ENDPOINTS; + protected List allowedApiHosts = Defaults.ALLOWED_API_HOSTS; protected int maxFindTransactions = Defaults.MAX_FIND_TRANSACTIONS; protected int maxRequestsList = Defaults.MAX_REQUESTS_LIST; protected int maxGetTransactionStrings = Defaults.MAX_GET_TRANSACTION_STRINGS; protected int maxBodyLength = Defaults.MAX_BODY_LENGTH; protected String remoteAuth = Defaults.REMOTE_AUTH; + //XI + protected String xiDir = Defaults.XI_DIR; + //We don't have a REMOTE config but we have a remote flag. We must add a field for JCommander private boolean remote; + // COMMUNICATION RELATED CONFIGURATION //Network protected int udpReceiverPort = Defaults.UDP_RECEIVER_PORT; protected int tcpReceiverPort = Defaults.TCP_RECEIVER_PORT; @@ -54,17 +206,6 @@ public abstract class BasePendulumConfig implements PendulumConfig { protected boolean dnsResolutionEnabled = Defaults.DNS_RESOLUTION_ENABLED; protected List neighbors = new ArrayList<>(); - //XI - protected String xiDir = Defaults.XI_DIR; - - //DB - protected String dbPath = Defaults.DB_PATH; - protected String dbLogPath = Defaults.DB_LOG_PATH; - protected int dbCacheSize = Defaults.DB_CACHE_SIZE; //KB - protected String mainDb = Defaults.ROCKS_DB; - protected boolean revalidate = Defaults.REVALIDATE; - protected boolean rescanDb = Defaults.RESCAN_DB; - //Protocol protected double pReplyRandomTip = Defaults.P_REPLY_RANDOM_TIP; protected double pDropTransaction = Defaults.P_DROP_TRANSACTION; @@ -72,20 +213,11 @@ public abstract class BasePendulumConfig implements PendulumConfig { protected double pSendMilestone = Defaults.P_SEND_MILESTONE; protected double pPropagateRequest = Defaults.P_PROPAGATE_REQUEST; - //ZMQ - protected boolean zmqEnableTcp = Defaults.ZMQ_ENABLE_TCP; - protected boolean zmqEnableIpc = Defaults.ZMQ_ENABLE_IPC; - protected int zmqPort = Defaults.ZMQ_PORT; - protected int zmqThreads = Defaults.ZMQ_THREADS; - protected String zmqIpc = Defaults.ZMQ_IPC; - protected int qSizeNode = Defaults.QUEUE_SIZE; - protected int cacheSizeBytes = Defaults.CACHE_SIZE_BYTES; - /** - * @deprecated This field was replaced by {@link #zmqEnableTcp} and {@link #zmqEnableIpc}. It is only needed - * for backward compatibility to --zmq-enabled parameter with JCommander. - */ - @Deprecated - private boolean zmqEnabled; + // CONSENSUS RELATED CONFIGURATION + //Milestone + protected String resourcePath = Defaults.RESOUCER_PATH; + protected String defaultResourcePath = Defaults.DEFAULT_RESOUCE_PATH; + protected int milestoneKeyDepth = Defaults.MILESTONE_KEY_DEPTH; //Tip Selection protected int maxDepth = Defaults.MAX_DEPTH; @@ -107,32 +239,56 @@ public abstract class BasePendulumConfig implements PendulumConfig { protected int localSnapshotsDepth = Defaults.LOCAL_SNAPSHOTS_DEPTH; protected String localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; - //Logging - protected boolean saveLogEnabled = Defaults.SAVELOG_ENABLED; - protected String saveLogBasePath = Defaults.SAVELOG_BASE_PATH; - protected String saveLogXMLFile = Defaults.SAVELOG_XML_FILE; + //Spent-addresses + protected String spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH; + protected String spentAddressesDbLogPath = Defaults.SPENT_ADDRESSES_DB_LOG_PATH; //Validator Manager protected boolean validatorManagerEnabled = Defaults.VALIDATOR_MANAGER_ENABLED; protected Hash validatorManagerAddress = Defaults.VALIDATOR_MANAGER_ADDRESS; protected int updateValidatorDelay = Defaults.UPDATE_VALIDATOR_DELAY; - protected int startRoundDelay = Defaults.START_ROUND_DELAY; protected String validatorManagerKeyfile = Defaults.VALIDATOR_MANAGER_KEYFILE; protected int validatorManagerKeyDepth = Defaults.VALIDATOR_MANAGER_KEY_DEPTH; protected int validatorManagerSecurity = Defaults.VALIDATOR_MANAGER_SECURITY; - //Milestone - protected String validatorPath = Defaults.VALIDATOR_PATH; - protected boolean validator = Defaults.VALIDATOR; - protected Set initialValidators = Defaults.INITIAL_VALIDATORS; + //Round and Timing protected long genesisTime = Defaults.GENESIS_TIME; protected int roundDuration = Defaults.ROUND_DURATION; protected int roundPause = Defaults.ROUND_PAUSE; - protected String resourcePath = Defaults.RESOUCER_PATH; - protected String defaultResourcePath = Defaults.DEFAULT_RESOUCE_PATH; - protected int milestoneKeyDepth = Defaults.MILESTONE_KEY_DEPTH; + protected int startRoundDelay = Defaults.START_ROUND_DELAY; + + //Validator + protected String validatorPath = Defaults.VALIDATOR_PATH; + protected boolean validator = Defaults.VALIDATOR; + protected Set initialValidators = Defaults.INITIAL_VALIDATORS; protected int validatorSecurity = Defaults.VALIDATOR_SECURITY; + + // ACCOUNTABILITY CONFIGURATION + //DB + protected String dbPath = Defaults.DB_PATH; + protected String dbLogPath = Defaults.DB_LOG_PATH; + protected int dbCacheSize = Defaults.DB_CACHE_SIZE; //KB + protected String mainDb = Defaults.ROCKS_DB; + protected boolean revalidate = Defaults.REVALIDATE; + protected boolean rescanDb = Defaults.RESCAN_DB; + + //Logging + protected boolean saveLogEnabled = Defaults.SAVELOG_ENABLED; + protected String saveLogBasePath = Defaults.SAVELOG_BASE_PATH; + protected String saveLogXMLFile = Defaults.SAVELOG_XML_FILE; + + //ZMQ + protected boolean zmqEnableTcp = Defaults.ZMQ_ENABLE_TCP; + protected boolean zmqEnableIpc = Defaults.ZMQ_ENABLE_IPC; + protected int zmqPort = Defaults.ZMQ_PORT; + protected int zmqThreads = Defaults.ZMQ_THREADS; + protected String zmqIpc = Defaults.ZMQ_IPC; + protected int qSizeNode = Defaults.QUEUE_SIZE; + protected int cacheSizeBytes = Defaults.CACHE_SIZE_BYTES; + + + @Override public JCommander parseConfigFromArgs(String[] args) throws ParameterException { //One can invoke help via INI file (feature/bug) so we always create JCommander even if args is empty @@ -142,7 +298,7 @@ public JCommander parseConfigFromArgs(String[] args) throws ParameterException { .acceptUnknownOptions(true) .allowParameterOverwriting(true) //This is the first line of JCommander Usage - .programName("java -jar hlx-" + Main.VERSION + ".jar") + .programName("java -jar pen-" + Main.VERSION + ".jar") .build(); if (ArrayUtils.isNotEmpty(args)) { jCommander.parse(args); @@ -167,7 +323,7 @@ public int getApiPort() { } @JsonProperty - @Parameter(names = {"--api-port", "-p"}, description = APIConfig.Descriptions.API_PORT) + @Parameter(names = {"--api_port", "-p"}, description = APIConfig.Descriptions.API_PORT) public void setPort(int apiPort) { this.apiPort = apiPort; } @@ -178,7 +334,7 @@ public String getApiHost() { } @JsonProperty - @Parameter(names = {"--api-host"}, description = APIConfig.Descriptions.API_HOST) + @Parameter(names = {"--api_host"}, description = APIConfig.Descriptions.API_HOST) protected void setApiHost(String apiHost) { this.apiHost = apiHost; } @@ -190,38 +346,37 @@ protected void setRemote(boolean remote) { } @Override - public List getRemoteLimitApi() { - return remoteLimitApi; + public List getIgnoredApiEndpoints() { + return ignoredApiEndpoints; } @JsonProperty - @Parameter(names = {"--remote-limit-api"}, description = APIConfig.Descriptions.REMOTE_LIMIT_API) - protected void setRemoteLimitApi(String remoteLimitApi) { - this.remoteLimitApi = PendulumUtils.splitStringToImmutableList(remoteLimitApi, SPLIT_STRING_TO_LIST_REGEX); + @Parameter(names = {"--ignored_api_endpoints"}, description = APIConfig.Descriptions.IGNORED_API_ENDPOINTS) + protected void setIgnoredApiEndpoints(String ignoredApiEndpoints) { + this.ignoredApiEndpoints = PendulumUtils.splitStringToImmutableList(ignoredApiEndpoints, SPLIT_STRING_TO_LIST_REGEX); } @Override - public List getRemoteTrustedApiHosts() { - return remoteTrustedApiHosts; + public List getAllowedApiHosts() { + return allowedApiHosts; } @JsonProperty - @Parameter(names = {"--remote-trusted-api-hosts"}, description = APIConfig.Descriptions.REMOTE_TRUSTED_API_HOSTS) - public void setRemoteTrustedApiHosts(String remoteTrustedApiHosts) { - List addresses = PendulumUtils.splitStringToImmutableList(remoteTrustedApiHosts, SPLIT_STRING_TO_LIST_REGEX); + @Parameter(names = {"--allowed_api_hosts"}, description = APIConfig.Descriptions.ALLOWED_API_HOSTS) + public void setAllowedApiHosts(String allowedApiHosts) { + List addresses = PendulumUtils.splitStringToImmutableList(allowedApiHosts, SPLIT_STRING_TO_LIST_REGEX); List inetAddresses = addresses.stream().map(host -> { try { return InetAddress.getByName(host.trim()); } catch (UnknownHostException e) { - throw new ParameterException("Invalid value for --remote-trusted-api-hosts address: ", e); + throw new ParameterException("Invalid value for --allowed_api_hosts address: ", e); } }).collect(Collectors.toList()); - // always make sure that localhost exists as trusted host - if (!inetAddresses.contains(Defaults.REMOTE_LIMIT_API_DEFAULT_HOST)) { - inetAddresses.add(Defaults.REMOTE_LIMIT_API_DEFAULT_HOST); + if (!inetAddresses.contains(Defaults.ALLOWED_API_DEFAULT_HOST)) { + inetAddresses.add(Defaults.ALLOWED_API_DEFAULT_HOST); } - this.remoteTrustedApiHosts = Collections.unmodifiableList(inetAddresses); + this.allowedApiHosts = Collections.unmodifiableList(inetAddresses); } @Override @@ -230,7 +385,7 @@ public int getMaxFindTransactions() { } @JsonProperty - @Parameter(names = {"--max-find-transactions"}, description = APIConfig.Descriptions.MAX_FIND_TRANSACTIONS) + @Parameter(names = {"--max_find_transactions"}, description = APIConfig.Descriptions.MAX_FIND_TRANSACTIONS) protected void setMaxFindTransactions(int maxFindTransactions) { this.maxFindTransactions = maxFindTransactions; } @@ -241,7 +396,7 @@ public int getMaxRequestsList() { } @JsonProperty - @Parameter(names = {"--max-requests-list"}, description = APIConfig.Descriptions.MAX_REQUESTS_LIST) + @Parameter(names = {"--max_requests_list"}, description = APIConfig.Descriptions.MAX_REQUESTS_LIST) protected void setMaxRequestsList(int maxRequestsList) { this.maxRequestsList = maxRequestsList; } @@ -252,7 +407,7 @@ public int getMaxTransactionStrings() { } @JsonProperty - @Parameter(names = {"--max-get-transaction-strings"}, description = APIConfig.Descriptions.MAX_GET_TRANSACTION_STRINGS) + @Parameter(names = {"--max_get_transaction_strings"}, description = APIConfig.Descriptions.MAX_GET_TRANSACTION_STRINGS) protected void setMaxGetTransactionStrings(int maxGetTransactionStrings) { this.maxGetTransactionStrings = maxGetTransactionStrings; } @@ -263,7 +418,7 @@ public int getMaxBodyLength() { } @JsonProperty - @Parameter(names = {"--max-body-length"}, description = APIConfig.Descriptions.MAX_BODY_LENGTH) + @Parameter(names = {"--max_body_length"}, description = APIConfig.Descriptions.MAX_BODY_LENGTH) protected void setMaxBodyLength(int maxBodyLength) { this.maxBodyLength = maxBodyLength; } @@ -274,7 +429,7 @@ public String getRemoteAuth() { } @JsonProperty - @Parameter(names = {"--remote-auth"}, description = APIConfig.Descriptions.REMOTE_AUTH) + @Parameter(names = {"--remote_auth"}, description = APIConfig.Descriptions.REMOTE_AUTH) protected void setRemoteAuth(String remoteAuth) { this.remoteAuth = remoteAuth; } @@ -285,7 +440,7 @@ public int getUdpReceiverPort() { } @JsonProperty - @Parameter(names = {"-u", "--udp-receiver-port"}, description = NetworkConfig.Descriptions.UDP_RECEIVER_PORT) + @Parameter(names = {"-u", "--udp_receiver_port"}, description = NetworkConfig.Descriptions.UDP_RECEIVER_PORT) public void setUdpReceiverPort(int udpReceiverPort) { this.udpReceiverPort = udpReceiverPort; } @@ -296,7 +451,7 @@ public int getTcpReceiverPort() { } @JsonProperty - @Parameter(names = {"-t", "--tcp-receiver-port"}, description = NetworkConfig.Descriptions.TCP_RECEIVER_PORT) + @Parameter(names = {"-t", "--tcp_receiver_port"}, description = NetworkConfig.Descriptions.TCP_RECEIVER_PORT) protected void setTcpReceiverPort(int tcpReceiverPort) { this.tcpReceiverPort = tcpReceiverPort; } @@ -307,7 +462,7 @@ public double getpRemoveRequest() { } @JsonProperty - @Parameter(names = {"--p-remove-request"}, description = NetworkConfig.Descriptions.P_REMOVE_REQUEST) + @Parameter(names = {"--p_remove_request"}, description = NetworkConfig.Descriptions.P_REMOVE_REQUEST) protected void setpRemoveRequest(double pRemoveRequest) { this.pRemoveRequest = pRemoveRequest; } @@ -318,7 +473,7 @@ public int getSendLimit() { } @JsonProperty - @Parameter(names = {"--send-limit"}, description = NetworkConfig.Descriptions.SEND_LIMIT) + @Parameter(names = {"--send_limit"}, description = NetworkConfig.Descriptions.SEND_LIMIT) protected void setSendLimit(int sendLimit) { this.sendLimit = sendLimit; } @@ -329,7 +484,7 @@ public int getMaxPeers() { } @JsonProperty - @Parameter(names = {"--max-peers"}, description = NetworkConfig.Descriptions.MAX_PEERS) + @Parameter(names = {"--max_peers"}, description = NetworkConfig.Descriptions.MAX_PEERS) protected void setMaxPeers(int maxPeers) { this.maxPeers = maxPeers; } @@ -340,18 +495,19 @@ public boolean isDnsRefresherEnabled() { } @JsonProperty - @Parameter(names = {"--dns-refresher"}, description = NetworkConfig.Descriptions.DNS_REFRESHER_ENABLED, arity = 1) + @Parameter(names = {"--dns_refresher"}, description = NetworkConfig.Descriptions.DNS_REFRESHER_ENABLED, arity = 1) protected void setDnsRefresherEnabled(boolean dnsRefresherEnabled) { this.dnsRefresherEnabled = dnsRefresherEnabled; } + @Override public boolean isDnsResolutionEnabled() { return dnsResolutionEnabled; } @JsonProperty - @Parameter(names = {"--dns-resolution"}, description = NetworkConfig.Descriptions.DNS_RESOLUTION_ENABLED, arity = 1) + @Parameter(names = {"--dns_resolution"}, description = NetworkConfig.Descriptions.DNS_RESOLUTION_ENABLED, arity = 1) protected void setDnsResolutionEnabled(boolean dnsResolutionEnabled) { this.dnsResolutionEnabled = dnsResolutionEnabled; } @@ -373,7 +529,7 @@ public String getXiDir() { } @JsonProperty - @Parameter(names = {"--XI-dir"}, description = XIConfig.Descriptions.XI_DIR) + @Parameter(names = {"--xi_dir"}, description = XIConfig.Descriptions.XI_DIR) protected void setXiDir(String xiDir) { this.xiDir = xiDir; } @@ -384,7 +540,7 @@ public String getDbPath() { } @JsonProperty - @Parameter(names = {"--db-path"}, description = DbConfig.Descriptions.DB_PATH) + @Parameter(names = {"--db_path"}, description = DbConfig.Descriptions.DB_PATH) protected void setDbPath(String dbPath) { this.dbPath = dbPath; } @@ -395,7 +551,7 @@ public String getDbLogPath() { } @JsonProperty - @Parameter(names = {"--db-log-path"}, description = DbConfig.Descriptions.DB_LOG_PATH) + @Parameter(names = {"--db_log_path"}, description = DbConfig.Descriptions.DB_LOG_PATH) protected void setDbLogPath(String dbLogPath) { this.dbLogPath = dbLogPath; } @@ -406,7 +562,7 @@ public int getDbCacheSize() { } @JsonProperty - @Parameter(names = {"--db-cache-size"}, description = DbConfig.Descriptions.DB_CACHE_SIZE) + @Parameter(names = {"--db_cache_size"}, description = DbConfig.Descriptions.DB_CACHE_SIZE) protected void setDbCacheSize(int dbCacheSize) { this.dbCacheSize = dbCacheSize; } @@ -422,6 +578,25 @@ protected void setMainDb(String mainDb) { this.mainDb = mainDb; } + + @JsonProperty + @Parameter(names = {"--spnt_add_db_path"}, description = SpentAddressesConfig.Descriptions.SPENT_ADDRESSES_DB_PATH) + protected void setSpentAddressesDbPath(String spentAddressesDbPath) { + this.spentAddressesDbPath = spentAddressesDbPath; + } + public String getSpentAddressesDbPath() { + return spentAddressesDbPath; + } + + @JsonProperty + @Parameter(names = {"--spnt_add_db_log_path"}, description = SpentAddressesConfig.Descriptions.SPENT_ADDRESSES_DB_LOG_PATH) + protected void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { + this.spentAddressesDbLogPath = spentAddressesDbLogPath; + } + public String getSpentAddressesDbLogPath() { + return spentAddressesDbLogPath; + } + @Override public boolean isRevalidate() { return revalidate; @@ -465,7 +640,7 @@ public double getpReplyRandomTip() { } @JsonProperty - @Parameter(names = {"--p-reply-random"}, description = ProtocolConfig.Descriptions.P_REPLY_RANDOM_TIP) + @Parameter(names = {"--p_reply_random"}, description = ProtocolConfig.Descriptions.P_REPLY_RANDOM_TIP) protected void setpReplyRandomTip(double pReplyRandomTip) { this.pReplyRandomTip = pReplyRandomTip; } @@ -476,7 +651,7 @@ public double getpDropTransaction() { } @JsonProperty - @Parameter(names = {"--p-drop-transaction"}, description = ProtocolConfig.Descriptions.P_DROP_TRANSACTION) + @Parameter(names = {"--p_drop_transaction"}, description = ProtocolConfig.Descriptions.P_DROP_TRANSACTION) protected void setpDropTransaction(double pDropTransaction) { this.pDropTransaction = pDropTransaction; } @@ -487,7 +662,7 @@ public double getpSelectMilestoneChild() { } @JsonProperty - @Parameter(names = {"--p-select-milestone"}, description = ProtocolConfig.Descriptions.P_SELECT_MILESTONE) + @Parameter(names = {"--p_select_milestone"}, description = ProtocolConfig.Descriptions.P_SELECT_MILESTONE) protected void setpSelectMilestoneChild(double pSelectMilestoneChild) { this.pSelectMilestoneChild = pSelectMilestoneChild; } @@ -498,7 +673,7 @@ public double getpSendMilestone() { } @JsonProperty - @Parameter(names = {"--p-send-milestone"}, description = ProtocolConfig.Descriptions.P_SEND_MILESTONE) + @Parameter(names = {"--p_send_milestone"}, description = ProtocolConfig.Descriptions.P_SEND_MILESTONE) protected void setpSendMilestone(double pSendMilestone) { this.pSendMilestone = pSendMilestone; } @@ -509,7 +684,7 @@ public double getpPropagateRequest() { } @JsonProperty - @Parameter(names = {"--p-propagate-request"}, description = ProtocolConfig.Descriptions.P_PROPAGATE_REQUEST) + @Parameter(names = {"--p_propagate_request"}, description = ProtocolConfig.Descriptions.P_PROPAGATE_REQUEST) protected void setpPropagateRequest(double pPropagateRequest) { this.pPropagateRequest = pPropagateRequest; } @@ -520,7 +695,7 @@ public boolean getLocalSnapshotsEnabled() { } @JsonProperty - @Parameter(names = {"--local-snapshots-enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_ENABLED) + @Parameter(names = {"--local_snapshots_enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_ENABLED) protected void setLocalSnapshotsEnabled(boolean localSnapshotsEnabled) { this.localSnapshotsEnabled = localSnapshotsEnabled; } @@ -531,7 +706,7 @@ public boolean getLocalSnapshotsPruningEnabled() { } @JsonProperty - @Parameter(names = {"--local-snapshots-pruning-enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_ENABLED) + @Parameter(names = {"--local_snapshots_pruning_enabled"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_ENABLED) protected void setLocalSnapshotsPruningEnabled(boolean localSnapshotsPruningEnabled) { this.localSnapshotsPruningEnabled = localSnapshotsPruningEnabled; } @@ -542,7 +717,7 @@ public int getLocalSnapshotsPruningDelay() { } @JsonProperty - @Parameter(names = {"--local-snapshots-pruning-delay"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_DELAY) + @Parameter(names = {"--local_snapshots_pruning_delay"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_PRUNING_DELAY) protected void setLocalSnapshotsPruningDelay(int localSnapshotsPruningDelay) { this.localSnapshotsPruningDelay = localSnapshotsPruningDelay; } @@ -553,7 +728,7 @@ public int getLocalSnapshotsIntervalSynced() { } @JsonProperty - @Parameter(names = {"--local-snapshots-interval-synced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_SYNCED) + @Parameter(names = {"--local_snapshots_interval_synced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_SYNCED) protected void setLocalSnapshotsIntervalSynced(int localSnapshotsIntervalSynced) { this.localSnapshotsIntervalSynced = localSnapshotsIntervalSynced; } @@ -564,7 +739,7 @@ public int getLocalSnapshotsIntervalUnsynced() { } @JsonProperty - @Parameter(names = {"--local-snapshots-interval-unsynced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED) + @Parameter(names = {"--local_snapshots_interval_unsynced"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED) protected void setLocalSnapshotsIntervalUnsynced(int localSnapshotsIntervalUnsynced) { this.localSnapshotsIntervalUnsynced = localSnapshotsIntervalUnsynced; } @@ -575,7 +750,7 @@ public int getLocalSnapshotsDepth() { } @JsonProperty - @Parameter(names = {"--local-snapshots-depth"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_DEPTH) + @Parameter(names = {"--local_snapshots_depth"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_DEPTH) protected void setLocalSnapshotsDepth(int localSnapshotsDepth) { this.localSnapshotsDepth = localSnapshotsDepth; } @@ -586,7 +761,7 @@ public String getLocalSnapshotsBasePath() { } @JsonProperty - @Parameter(names = {"--local-snapshots-base-path"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_BASE_PATH) + @Parameter(names = {"--local_snapshots_base_path"}, description = SnapshotConfig.Descriptions.LOCAL_SNAPSHOTS_BASE_PATH) protected void setLocalSnapshotsBasePath(String localSnapshotsBasePath) { this.localSnapshotsBasePath = localSnapshotsBasePath; } @@ -626,37 +801,18 @@ public int getNumberOfKeysInMilestone() { return Defaults.NUM_KEYS_IN_MILESTONE; } - /** - * Checks if ZMQ is enabled. - * - * @return true if zmqEnableTcp or zmqEnableIpc is set. - */ @Override public boolean isZmqEnabled() { return zmqEnableTcp || zmqEnableIpc; } - /** - * Activates ZMQ to listen on TCP and IPC. - * - * @param zmqEnabled true if ZMQ should listen in TCP and IPC. - * @deprecated Use {@link #setZmqEnableTcp(boolean) and/or {@link #setZmqEnableIpc(boolean)}} instead. - */ - @Deprecated - @JsonProperty - @Parameter(names = "--zmq-enabled", description = ZMQConfig.Descriptions.ZMQ_ENABLED, arity = 1) - protected void setZmqEnabled(boolean zmqEnabled) { - this.zmqEnableTcp = zmqEnabled; - this.zmqEnableIpc = zmqEnabled; - } - @Override public boolean isZmqEnableTcp() { return zmqEnableTcp; } @JsonProperty - @Parameter(names = "--zmq-enable-tcp", description = ZMQConfig.Descriptions.ZMQ_ENABLE_TCP, arity = 1) + @Parameter(names = "--zmq_enable_tcp", description = ZMQConfig.Descriptions.ZMQ_ENABLE_TCP, arity = 1) public void setZmqEnableTcp(boolean zmqEnableTcp) { this.zmqEnableTcp = zmqEnableTcp; } @@ -667,7 +823,7 @@ public boolean isZmqEnableIpc() { } @JsonProperty - @Parameter(names = "--zmq-enable-ipc", description = ZMQConfig.Descriptions.ZMQ_ENABLE_IPC, arity = 1) + @Parameter(names = "--zmq_enable_ipc", description = ZMQConfig.Descriptions.ZMQ_ENABLE_IPC, arity = 1) public void setZmqEnableIpc(boolean zmqEnableIpc) { this.zmqEnableIpc = zmqEnableIpc; } @@ -678,7 +834,7 @@ public int getZmqPort() { } @JsonProperty - @Parameter(names = "--zmq-port", description = ZMQConfig.Descriptions.ZMQ_PORT) + @Parameter(names = "--zmq_port", description = ZMQConfig.Descriptions.ZMQ_PORT) protected void setZmqPort(int zmqPort) { this.zmqPort = zmqPort; this.zmqEnableTcp = true; @@ -690,7 +846,7 @@ public int getZmqThreads() { } @JsonProperty - @Parameter(names = "--zmq-threads", description = ZMQConfig.Descriptions.ZMQ_THREADS) + @Parameter(names = "--zmq_threads", description = ZMQConfig.Descriptions.ZMQ_THREADS) protected void setZmqThreads(int zmqThreads) { this.zmqThreads = zmqThreads; } @@ -701,7 +857,7 @@ public String getZmqIpc() { } @JsonProperty - @Parameter(names = "--zmq-ipc", description = ZMQConfig.Descriptions.ZMQ_IPC) + @Parameter(names = "--zmq_ipc", description = ZMQConfig.Descriptions.ZMQ_IPC) protected void setZmqIpc(String zmqIpc) { this.zmqIpc = zmqIpc; this.zmqEnableIpc = true; @@ -713,7 +869,7 @@ public int getqSizeNode() { } @JsonProperty - @Parameter(names = "--queue-size", description = NetworkConfig.Descriptions.Q_SIZE_NODE) + @Parameter(names = "--queue_size", description = NetworkConfig.Descriptions.Q_SIZE_NODE) protected void setqSizeNode(int qSizeNode) { this.qSizeNode = qSizeNode; } @@ -724,7 +880,7 @@ public double getpDropCacheEntry() { } @JsonProperty - @Parameter(names = "--p-drop-cache", description = NetworkConfig.Descriptions.P_DROP_CACHE_ENTRY) + @Parameter(names = "--p_drop_cache", description = NetworkConfig.Descriptions.P_DROP_CACHE_ENTRY) protected void setpDropCacheEntry(double pDropCacheEntry) { this.pDropCacheEntry = pDropCacheEntry; } @@ -735,7 +891,7 @@ public int getCacheSizeBytes() { } @JsonProperty - @Parameter(names = "--cache-size", description = NetworkConfig.Descriptions.CACHE_SIZE_BYTES) + @Parameter(names = "--cache_size", description = NetworkConfig.Descriptions.CACHE_SIZE_BYTES) protected void setCacheSizeBytes(int cacheSizeBytes) { this.cacheSizeBytes = cacheSizeBytes; } @@ -746,7 +902,7 @@ public int getMaxDepth() { } @JsonProperty - @Parameter(names = "--max-depth", description = TipSelConfig.Descriptions.MAX_DEPTH) + @Parameter(names = "--max_depth", description = TipSelConfig.Descriptions.MAX_DEPTH) protected void setMaxDepth(int maxDepth) { this.maxDepth = maxDepth; } @@ -756,7 +912,7 @@ public double getAlpha() { return alpha; } - @JsonProperty("TIPSELECTION_ALPHA") + @JsonProperty @Parameter(names = "--alpha", description = TipSelConfig.Descriptions.ALPHA) protected void setAlpha(double alpha) { this.alpha = alpha; @@ -773,7 +929,7 @@ public int getBelowMaxDepthTransactionLimit() { } @JsonProperty - @Parameter(names = "--max-analyzed-transactions", description = TipSelConfig.Descriptions.BELOW_MAX_DEPTH_TRANSACTION_LIMIT) + @Parameter(names = "--max_analyzed_transactions", description = TipSelConfig.Descriptions.BELOW_MAX_DEPTH_TRANSACTION_LIMIT) protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { this.maxAnalyzedTransactions = maxAnalyzedTransactions; } @@ -781,25 +937,22 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { // Validator Manager public boolean getValidatorManagerEnabled() {return validatorManagerEnabled; } @JsonProperty - @Parameter(names = {"--validator-manager"}, description = ValidatorManagerConfig.Descriptions.VALIDATOR_MANAGER_ENABLED, arity = 1) + @Parameter(names = {"--validator_manager"}, description = ValidatorManagerConfig.Descriptions.VALIDATOR_MANAGER_ENABLED, arity = 1) protected void setValidatorManagerEnabled(boolean validatorManagerEnabled) { this.validatorManagerEnabled = validatorManagerEnabled; } @Override public Hash getValidatorManagerAddress() { return validatorManagerAddress; } - @Override - public boolean isDontValidateTestnetValidatorManagerSig() { return false; } - @Override public int getUpdateValidatorDelay() {return updateValidatorDelay; } @JsonProperty - @Parameter(names = {"--update-validator"}, description = ValidatorManagerConfig.Descriptions.UPDATE_VALIDATOR_DELAY) + @Parameter(names = {"--update_validator_delay"}, description = ValidatorManagerConfig.Descriptions.UPDATE_VALIDATOR_DELAY) protected void setUpdateValidatorDelay(int updateValidatorDelay) { this.updateValidatorDelay = updateValidatorDelay; } @Override public int getStartRoundDelay() {return startRoundDelay; } @JsonProperty - @Parameter(names = {"--start-validator"}, description = ValidatorManagerConfig.Descriptions.START_ROUND_DELAY) + @Parameter(names = {"--start_validator"}, description = ValidatorManagerConfig.Descriptions.START_ROUND_DELAY) protected void setStartRoundDelay(int startRoundDelay) { this.startRoundDelay = startRoundDelay; } @Override @@ -816,7 +969,7 @@ protected void setBelowMaxDepthTransactionLimit(int maxAnalyzedTransactions) { public String getValidatorPath() {return validatorPath; } @JsonProperty - @Parameter(names = {"--validator-path"}, description = MilestoneConfig.Descriptions.VALIDATOR_PATH) + @Parameter(names = {"--validator_path"}, description = ValidatorConfig.Descriptions.VALIDATOR_PATH) protected void setValidatorPath(String validatorPath) { this.validatorPath = validatorPath; } @@ -825,7 +978,7 @@ protected void setValidatorPath(String validatorPath) { public boolean isValidator() {return validator; } @JsonProperty - @Parameter(names = {"--validator"}, description = MilestoneConfig.Descriptions.VALIDATOR) + @Parameter(names = {"--validator"}, description = ValidatorConfig.Descriptions.VALIDATOR) protected void setValidator(boolean validator) { this.validator = validator; } @@ -834,8 +987,8 @@ protected void setValidator(boolean validator) { public Set getInitialValidators() {return initialValidators; } @Override - public boolean isDontValidateTestnetMilestoneSig() { - return false; + public boolean isValidateTestnetMilestoneSig() { + return true; } @Override @@ -844,8 +997,8 @@ public long getGenesisTime() { } @JsonProperty - @Parameter(names = {"--genesis"}, description = MilestoneConfig.Descriptions.GENESIS_TIME) - protected void setGenesisTime(int genesisTime) { this.genesisTime = genesisTime; } + @Parameter(names = {"--genesis"}, description = RoundConfig.Descriptions.GENESIS_TIME) + protected void setGenesisTime(long genesisTime) { this.genesisTime = genesisTime; } @Override public int getRoundDuration() { @@ -853,13 +1006,13 @@ public int getRoundDuration() { } @JsonProperty - @Parameter(names = {"--round"}, description = MilestoneConfig.Descriptions.ROUND_DURATION) + @Parameter(names = {"--round"}, description = RoundConfig.Descriptions.ROUND_DURATION) protected void setRoundDuration(int roundDuration) { this.roundDuration = roundDuration; } @Override public int getRoundPause() { return roundPause; } @JsonProperty - @Parameter(names = {"--round-pause"}, description = MilestoneConfig.Descriptions.ROUND_PAUSE) + @Parameter(names = {"--round_pause"}, description = RoundConfig.Descriptions.ROUND_PAUSE) protected void setRoundPause(int roundPause) { this.roundPause = roundPause; } @Override @@ -899,7 +1052,7 @@ public int getPowThreads() { return powThreads; } @JsonProperty - @Parameter(names = "--pow-threads", description = PoWConfig.Descriptions.POW_THREADS) + @Parameter(names = "--pow_threads", description = PoWConfig.Descriptions.POW_THREADS) protected void setPowThreads(int powThreads) { this.powThreads = powThreads; } @@ -910,7 +1063,7 @@ public boolean isSaveLogEnabled() { } @JsonProperty - @Parameter(names = {"--savelog-enabled"}, description = LoggingConfig.Descriptions.SAVELOG_ENABLED) + @Parameter(names = {"--savelog_enabled"}, description = LoggingConfig.Descriptions.SAVELOG_ENABLED) protected void setSaveLogEnabled(boolean saveLogEnabled) { this.saveLogEnabled = saveLogEnabled; } @@ -921,7 +1074,7 @@ public String getSaveLogBasePath() { } @JsonProperty - @Parameter(names = {"--savelog-path"}, description = LoggingConfig.Descriptions.SAVELOG_BASE_PATH) + @Parameter(names = {"--savelog_path"}, description = LoggingConfig.Descriptions.SAVELOG_BASE_PATH) protected void setSaveLogBasePath(String saveLogBasePath) { this.saveLogBasePath = saveLogBasePath; } @@ -932,132 +1085,9 @@ public String getSaveLogXMLFile() { } @JsonProperty - @Parameter(names = {"--savelog-xml"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE) + @Parameter(names = {"--savelog_xml"}, description = LoggingConfig.Descriptions.SAVELOG_XML_FILE) protected void setSaveLogXMLFile(String saveLogXMLFile) { this.saveLogXMLFile = saveLogXMLFile; } - public interface Defaults { - //API - int API_PORT = 8085; - String API_HOST = "localhost"; - List REMOTE_LIMIT_API = PendulumUtils.createImmutableList(); // "addNeighbors", "getNeighbors", "removeNeighbors", "attachToTangle", "interruptAttachingToTangle" <- TODO: limit these in production! - InetAddress REMOTE_LIMIT_API_DEFAULT_HOST = InetAddress.getLoopbackAddress(); - List REMOTE_LIMIT_API_HOSTS = PendulumUtils.createImmutableList(REMOTE_LIMIT_API_DEFAULT_HOST); - int MAX_FIND_TRANSACTIONS = 100_000; - int MAX_REQUESTS_LIST = 1_000; - int MAX_GET_TRANSACTION_STRINGS = 10_000; - int MAX_BODY_LENGTH = 1_000_000; - String REMOTE_AUTH = ""; - - //Network - int UDP_RECEIVER_PORT = 4100; - int TCP_RECEIVER_PORT = 5100; - double P_REMOVE_REQUEST = 0.01d; - int SEND_LIMIT = -1; - int MAX_PEERS = 0; - boolean DNS_REFRESHER_ENABLED = true; - boolean DNS_RESOLUTION_ENABLED = true; - - //XI - String XI_DIR = "modules"; - - //DB - String DB_PATH = "mainnetdb"; - String DB_LOG_PATH = "mainnet.log"; - int DB_CACHE_SIZE = 100_000; - String ROCKS_DB = "rocksdb"; - boolean REVALIDATE = false; - boolean RESCAN_DB = false; - - //Protocol - double P_REPLY_RANDOM_TIP = 0.66d; - double P_DROP_TRANSACTION = 0d; - double P_SELECT_MILESTONE_CHILD = 0.7d; - double P_SEND_MILESTONE = 0.02d; - double P_PROPAGATE_REQUEST = 0.01d; - int MWM = 1; - int PACKET_SIZE = 800; - int REQ_HASH_SIZE = 32; - int QUEUE_SIZE = 1_000; - double P_DROP_CACHE_ENTRY = 0.02d; - int CACHE_SIZE_BYTES = 150_000; - - //Zmq - int ZMQ_THREADS = 1; - boolean ZMQ_ENABLE_IPC = false; - String ZMQ_IPC = "ipc://hlx"; - boolean ZMQ_ENABLE_TCP = false; - int ZMQ_PORT = 5556; - - //TipSel - int MAX_DEPTH = 15; - double ALPHA = 0.001d; - - //Tip solidification - boolean TIP_SOLIDIFIER_ENABLED = true; - - //PoW - int POW_THREADS = 8; - - //Resource directory: - String RESOUCER_PATH = "./src/main/resources"; - String DEFAULT_RESOUCE_PATH = "./resources"; - - //Validator Manager - boolean VALIDATOR_MANAGER_ENABLED = false; - Hash VALIDATOR_MANAGER_ADDRESS = HashFactory.ADDRESS.create("9474289ae28f0ea6e3b8bedf8fc52f14d2fa9528a4eb29d7879d8709fd2f6d37"); - int UPDATE_VALIDATOR_DELAY = 30000; - int START_ROUND_DELAY = 2; - String VALIDATOR_MANAGER_KEYFILE = "/ValidatorManager.key"; - int VALIDATOR_MANAGER_KEY_DEPTH = 15; - int VALIDATOR_MANAGER_SECURITY = 2; - - //Milestone - boolean VALIDATOR = false; - String VALIDATOR_PATH = null; - Set INITIAL_VALIDATORS = new HashSet<>(Arrays.asList( - HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832"), - HashFactory.ADDRESS.create("a5afe01e64ae959f266b382bb5927fd07b49e7e3180239535126844aaae9bf93"), - HashFactory.ADDRESS.create("e2debe246b5d1a6e05b57b0fc14edb51d136966a91a803b523586ad032f72f3d"), - HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), - HashFactory.ADDRESS.create("1895a039c85b9a5c4e822c8fc51884aedecddfa09daccef642fff697157657b4"), - HashFactory.ADDRESS.create("1c6b0ee311a7ddccf255c1097995714b285cb06628be1cef2080b0bef7700e12"), - HashFactory.ADDRESS.create("eb0d925c1cfa4067db65e4b93fa17d451120cc5a719d637d44a39a983407d832") - )); - - long GENESIS_TIME = 1569024001000L; - int ROUND_DURATION = 15000; - int ROUND_PAUSE = 5000; - String VALIDATOR_KEYFILE = "/Validator.key"; - String VALIDATOR_SEED_PATH = "/Validator.txt"; - int MILESTONE_KEY_DEPTH = 10; - int VALIDATOR_SECURITY = 2; - int NUMBER_OF_ACTIVE_VALIDATORS = 1; - double CONFIRMATION_THRESHOLD = 0.66; - - Hash EMPTY_ROUND_HASH = HashFactory.ADDRESS.create("00000000000000000000000000000000000000000000656d707479726f756e64"); // bootstrap hash for empty round - - //Snapshot - boolean LOCAL_SNAPSHOTS_ENABLED = true; - boolean LOCAL_SNAPSHOTS_PRUNING_ENABLED = false; - int LOCAL_SNAPSHOTS_PRUNING_DELAY = 50000; - int LOCAL_SNAPSHOTS_INTERVAL_SYNCED = 10; - int LOCAL_SNAPSHOTS_INTERVAL_UNSYNCED = 1000; - String LOCAL_SNAPSHOTS_BASE_PATH = "./snapshot-mainnet"; - int LOCAL_SNAPSHOTS_DEPTH = 100; - String SNAPSHOT_FILE = "/snapshotMainnet.txt"; - String SNAPSHOT_SIG_FILE = "/snapshotMainnet.sig"; - String PREVIOUS_EPOCHS_SPENT_ADDRESSES_TXT = "/previousEpochsSpentAddresses.txt"; - String PREVIOUS_EPOCHS_SPENT_ADDRESSES_SIG = "/previousEpochsSpentAddresses.sig"; - long GLOBAL_SNAPSHOT_TIME = 1522235533L; - int MILESTONE_START_INDEX = 0; - int NUM_KEYS_IN_MILESTONE = 10; - int MAX_ANALYZED_TXS = 20_000; - - //Logging - boolean SAVELOG_ENABLED = false; - String SAVELOG_BASE_PATH = "./logs/"; - String SAVELOG_XML_FILE = "/logback-save.xml"; - } } diff --git a/src/main/java/net/helix/pendulum/conf/Config.java b/src/main/java/net/helix/pendulum/conf/Config.java index 5be42362..9548f8b7 100644 --- a/src/main/java/net/helix/pendulum/conf/Config.java +++ b/src/main/java/net/helix/pendulum/conf/Config.java @@ -1,14 +1,18 @@ package net.helix.pendulum.conf; public interface Config { + String TESTNET_FLAG = "--testnet"; + /** * @return {@value Descriptions#TESTNET} */ boolean isTestnet(); + interface Descriptions { String TESTNET = "Start in testnet mode."; } + class DescriptionHelper { protected static final String PROB_OF = "A number between 0 and 1 that represents the probability of "; } diff --git a/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java b/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java index 1626e3d2..0f79911f 100644 --- a/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ConsensusConfig.java @@ -6,5 +6,12 @@ * @implNote It currently extends two other interfaces. This has been done due to lack of separation of concerns in * the current code base and will be changed in the future */ -public interface ConsensusConfig extends SnapshotConfig, MilestoneConfig, ValidatorManagerConfig { + +public interface ConsensusConfig extends + MilestoneConfig, + RoundConfig, + SnapshotConfig, + SpentAddressesConfig, + ValidatorConfig, + ValidatorManagerConfig { } diff --git a/src/main/java/net/helix/pendulum/conf/GraphConfig.java b/src/main/java/net/helix/pendulum/conf/GraphConfig.java deleted file mode 100644 index c52b06ff..00000000 --- a/src/main/java/net/helix/pendulum/conf/GraphConfig.java +++ /dev/null @@ -1,9 +0,0 @@ -package net.helix.pendulum.conf; - -public interface GraphConfig extends Config{ - boolean isGraphEnabled(); - - interface Descriptions { - String GRAPH_ENABLED = "Enabling Graphstream"; - } -} \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/MainnetConfig.java b/src/main/java/net/helix/pendulum/conf/MainnetConfig.java index 206d4511..137875e2 100644 --- a/src/main/java/net/helix/pendulum/conf/MainnetConfig.java +++ b/src/main/java/net/helix/pendulum/conf/MainnetConfig.java @@ -1,6 +1,7 @@ package net.helix.pendulum.conf; public class MainnetConfig extends BasePendulumConfig { + @Override public boolean isTestnet() { return false; diff --git a/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java b/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java index d4426fa7..ec52e411 100644 --- a/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java +++ b/src/main/java/net/helix/pendulum/conf/MilestoneConfig.java @@ -1,75 +1,28 @@ package net.helix.pendulum.conf; -import net.helix.pendulum.model.Hash; - -import java.util.Set; /** * Configs that should be used for tracking milestones */ public interface MilestoneConfig extends Config { - /** - * @return {@value Descriptions#VALIDATOR} - */ - boolean isValidator(); - /** - * @return {@value Descriptions#VALIDATOR_PATH} - */ - String getValidatorPath(); - /** - * @return {@value Descriptions#VALIDATOR_SEED_PATH} - */ - String getValidatorSeedfile(); - /** - * Returns is validator is enabled, is dependent on isValidator and ValidatorPath - * @return {@value Descriptions#VALIDATOR} - */ - boolean isValidatorEnabled(); - /** - * @return Descriptions#INITIAL_VALIDATORS - */ - Set getInitialValidators(); - /** - * @return {@value Descriptions#DONT_VALIDATE_TESTNET_MILESTONE_SIG} - */ - boolean isDontValidateTestnetMilestoneSig(); - /** - * @return {@value Descriptions#GENESIS_TIME} - */ - long getGenesisTime(); - /** - * @return {@value Descriptions#ROUND_DURATION} - */ - int getRoundDuration(); - /** - * @return {@value Descriptions#ROUND_PAUSE} - */ - int getRoundPause(); - /** - * @return {@value Descriptions#VALIDATOR_KEYFILE} - */ - String getValidatorKeyfile(); /** * @return {@value Descriptions#MILESTONE_KEY_DEPTH} */ int getMilestoneKeyDepth(); + /** - * @return {@value Descriptions#VALIDATOR_SECURITY} + * @return {@value Descriptions#MILESTONE_START_INDEX} */ - int getValidatorSecurity(); + int getMilestoneStartIndex(); + interface Descriptions { - String VALIDATOR = "Flag that enables applying as a validator in the network."; - String VALIDATOR_PATH = "A path to a file containing the seed / keyfile has to be passed."; - String VALIDATOR_SEED_PATH = "A path to a file containing the seed has to be passed."; - String INITIAL_VALIDATORS = "The addresses of validators the network starts with"; - String DONT_VALIDATE_TESTNET_MILESTONE_SIG = "Disable validator validation on testnet"; - String GENESIS_TIME = "Time when the ledger started."; - String ROUND_DURATION = "Duration of a round in milli secounds."; - String ROUND_PAUSE = "Duration of time to finalize the round in milli secounds."; - String VALIDATOR_KEYFILE = "Filepath to validator keyfile"; String MILESTONE_KEY_DEPTH = "Depth of the merkle tree the milestones are signed with."; - String VALIDATOR_SECURITY = "Security level of transactions sent from a validator (milestones, registrations)"; + String MILESTONE_START_INDEX = "The start index of the milestones. This index is encoded in each milestone " + + "transaction by the coordinator."; + String NUMBER_OF_KEYS_IN_A_MILESTONE = "The depth of the Merkle tree which in turn determines the number of" + + "leaves (private keys) that the coordinator can use to sign a message."; + } } diff --git a/src/main/java/net/helix/pendulum/conf/PendulumConfig.java b/src/main/java/net/helix/pendulum/conf/PendulumConfig.java index 7a58c710..77c530d8 100644 --- a/src/main/java/net/helix/pendulum/conf/PendulumConfig.java +++ b/src/main/java/net/helix/pendulum/conf/PendulumConfig.java @@ -6,11 +6,25 @@ import java.io.File; /** - * A container for all possible configuration parameters of SBX. + * A container for all possible configuration parameters of Pendulum. * In charge of how we parse the configuration from given inputs. */ -public interface PendulumConfig extends APIConfig, NodeConfig, XIConfig, DbConfig, ConsensusConfig, ZMQConfig, TipSelConfig, PoWConfig, SolidificationConfig, LoggingConfig { - File CONFIG_FILE = new File("hlx.ini"); +public interface PendulumConfig extends + APIConfig, + ConsensusConfig, +// extends MilestoneConfig, RoundConfig, SnapshotConfig, SpentAddressesConfig, ValidatorConfig, ValidatorManagerConfig + DbConfig, + LoggingConfig, + NodeConfig,//extends NetworkConfig, ProtocolConfig + PoWConfig, + SolidificationConfig, + TipSelConfig, + XIConfig, + ZMQConfig + { + + File CONFIG_FILE = new File("pendulum.ini"); + /** * Parses the args to populate the configuration object * @@ -19,5 +33,6 @@ public interface PendulumConfig extends APIConfig, NodeConfig, XIConfig, DbConfi * @throws ParameterException if the parsing failed */ JCommander parseConfigFromArgs(String[] args) throws ParameterException; + boolean isHelp(); } diff --git a/src/main/java/net/helix/pendulum/conf/ProtocolConfig.java b/src/main/java/net/helix/pendulum/conf/ProtocolConfig.java index 4e1091bd..7c6a5bd0 100644 --- a/src/main/java/net/helix/pendulum/conf/ProtocolConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ProtocolConfig.java @@ -37,5 +37,6 @@ interface Descriptions { String P_SEND_MILESTONE = DescriptionHelper.PROB_OF + "sending a milestoneTracker transaction when the node looks for a random transaction to send to a neighbor."; String P_REPLY_RANDOM_TIP = DescriptionHelper.PROB_OF + "replying to a random transaction request, even though your node doesn't have anything to request."; String P_PROPAGATE_REQUEST = DescriptionHelper.PROB_OF + "propagating the request of a transaction to a neighbor node if it can't be found. This should be low since we don't want to propagate non-existing transactions that spam the network."; + } } \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/RoundConfig.java b/src/main/java/net/helix/pendulum/conf/RoundConfig.java new file mode 100644 index 00000000..f9fc9456 --- /dev/null +++ b/src/main/java/net/helix/pendulum/conf/RoundConfig.java @@ -0,0 +1,23 @@ +package net.helix.pendulum.conf; + +public interface RoundConfig extends Config { + + /** + * @return {@value Descriptions#GENESIS_TIME} + */ + long getGenesisTime(); + /** + * @return {@value Descriptions#ROUND_DURATION} + */ + int getRoundDuration(); + /** + * @return {@value Descriptions#ROUND_PAUSE} + */ + int getRoundPause(); + + interface Descriptions { + String GENESIS_TIME = "Time when the ledger started."; + String ROUND_DURATION = "Duration of a round in milli secounds."; + String ROUND_PAUSE = "Duration of time to finalize the round in milli secounds."; + } +} diff --git a/src/main/java/net/helix/pendulum/conf/SnapshotConfig.java b/src/main/java/net/helix/pendulum/conf/SnapshotConfig.java index df45c54d..07278a8a 100644 --- a/src/main/java/net/helix/pendulum/conf/SnapshotConfig.java +++ b/src/main/java/net/helix/pendulum/conf/SnapshotConfig.java @@ -1,7 +1,7 @@ package net.helix.pendulum.conf; /** - * Configurations for handling global snapshot data + * Configurations for handling snapshot data */ public interface SnapshotConfig extends Config { @@ -50,10 +50,7 @@ public interface SnapshotConfig extends Config { */ String getSnapshotSignatureFile(); - /** - * @return {@value Descriptions#MILESTONE_START_INDEX} - */ - int getMilestoneStartIndex(); + /** * @return {@value Descriptions#LOCAL_SNAPSHOTS_BASE_PATH} @@ -87,12 +84,7 @@ interface Descriptions { String SNAPSHOT_TIME = "Epoch time of the last snapshot."; String SNAPSHOT_FILE = "Path of the file that contains the state of the ledger at the last snapshot."; String SNAPSHOT_SIGNATURE_FILE = "Path to the file that contains a signature for the snapshot file."; - String MILESTONE_START_INDEX = "The start index of the milestones. This index is encoded in each milestone " + - "transaction by the coordinator."; - String NUMBER_OF_KEYS_IN_A_MILESTONE = "The depth of the Merkle tree which in turn determines the number of" + - "leaves (private keys) that the coordinator can use to sign a message."; - String PREVIOUS_EPOCH_SPENT_ADDRESSES_FILE = "The file that contains the list of all used addresses " + - "from previous epochs"; - String PREVIOUS_EPOCH_SPENT_ADDRESSES_SIG_FILE = "The file that contains signature of the previous epochs spent addresses file"; + + } } \ No newline at end of file diff --git a/src/main/java/net/helix/pendulum/conf/SpentAddressesConfig.java b/src/main/java/net/helix/pendulum/conf/SpentAddressesConfig.java new file mode 100644 index 00000000..0464bd33 --- /dev/null +++ b/src/main/java/net/helix/pendulum/conf/SpentAddressesConfig.java @@ -0,0 +1,35 @@ +package net.helix.pendulum.conf; + +/** + * Configurations for handling global snapshot data + */ +public interface SpentAddressesConfig extends Config { + + /** + * @return {@value Descriptions#SPENT_ADDRESSES_DB_PATH} + */ + String getSpentAddressesDbPath(); + + /** + * @return {@value Descriptions#SPENT_ADDRESSES_DB_LOG_PATH} + */ + String getSpentAddressesDbLogPath(); + + /** + * @return {@value Descriptions#PREVIOUS_EPOCH_SPENT_ADDRESSES_FILE} + */ + String getPreviousEpochSpentAddressesFiles(); + + /** + * @return {@value Descriptions#PREVIOUS_EPOCH_SPENT_ADDRESSES_SIG_FILE} + */ + String getPreviousEpochSpentAddressesSigFile(); + + interface Descriptions { + + String SPENT_ADDRESSES_DB_PATH = "The path where the spent addresses db is stored"; + String SPENT_ADDRESSES_DB_LOG_PATH = "The path where the spent addresses db log is stored"; + String PREVIOUS_EPOCH_SPENT_ADDRESSES_FILE = "The file that contains the list of all used addresses from previous epochs"; + String PREVIOUS_EPOCH_SPENT_ADDRESSES_SIG_FILE = "The file that contains signature of the previous epochs spent addresses file"; + } +} diff --git a/src/main/java/net/helix/pendulum/conf/TestnetConfig.java b/src/main/java/net/helix/pendulum/conf/TestnetConfig.java index 82361cb6..55367d21 100644 --- a/src/main/java/net/helix/pendulum/conf/TestnetConfig.java +++ b/src/main/java/net/helix/pendulum/conf/TestnetConfig.java @@ -8,7 +8,7 @@ public class TestnetConfig extends BasePendulumConfig { - protected boolean dontValidateTestnetMilestoneSig = Defaults.DONT_VALIDATE_MILESTONE_SIG; + protected boolean validateTestnetMilestoneSig = Defaults.VALIDATE_MILESTONE_SIG; protected String snapshotFile = Defaults.SNAPSHOT_FILE; protected String snapshotSignatureFile = Defaults.SNAPSHOT_SIG; protected long snapshotTime = Defaults.SNAPSHOT_TIME; @@ -23,6 +23,8 @@ public TestnetConfig() { super(); dbPath = Defaults.DB_PATH; dbLogPath = Defaults.DB_LOG_PATH; + spentAddressesDbPath = Defaults.SPENT_ADDRESSES_DB_PATH; + spentAddressesDbLogPath= Defaults.SPENT_ADDRESSES_DB_LOG_PATH; localSnapshotsBasePath = Defaults.LOCAL_SNAPSHOTS_BASE_PATH; } @@ -32,14 +34,14 @@ public boolean isTestnet() { } @Override - public boolean isDontValidateTestnetMilestoneSig() { - return dontValidateTestnetMilestoneSig; + public boolean isValidateTestnetMilestoneSig() { + return validateTestnetMilestoneSig; } @JsonProperty - @Parameter(names = "--testnet-no-milestone-sign-validation", description = MilestoneConfig.Descriptions.DONT_VALIDATE_TESTNET_MILESTONE_SIG) - protected void setDontValidateTestnetMilestoneSig(boolean dontValidateTestnetMilestoneSig) { - this.dontValidateTestnetMilestoneSig = dontValidateTestnetMilestoneSig; + @Parameter(names = "--validate_testnet_milestone_sig", description = ValidatorConfig.Descriptions.VALIDATE_TESTNET_MILESTONE_SIG) + protected void setValidateTestnetMilestoneSig(boolean validateTestnetMilestoneSig) { + this.validateTestnetMilestoneSig = validateTestnetMilestoneSig; } @Override @@ -92,7 +94,7 @@ public int getMilestoneStartIndex() { } @JsonProperty - @Parameter(names = "--milestone-start", description = SnapshotConfig.Descriptions.MILESTONE_START_INDEX) + @Parameter(names = "--milestone-start", description = MilestoneConfig.Descriptions.MILESTONE_START_INDEX) protected void setMilestoneStartIndex(int milestoneStartIndex) { this.milestoneStartIndex = milestoneStartIndex; } @@ -103,7 +105,7 @@ public int getNumberOfKeysInMilestone() { } @JsonProperty("NUMBER_OF_KEYS_IN_A_MILESTONE") - @Parameter(names = "--milestone-keys", description = SnapshotConfig.Descriptions.NUMBER_OF_KEYS_IN_A_MILESTONE) + @Parameter(names = "--milestone-keys", description = MilestoneConfig.Descriptions.NUMBER_OF_KEYS_IN_A_MILESTONE) protected void setNumberOfKeysInMilestone(int numberOfKeysInMilestone) { this.numberOfKeysInMilestone = numberOfKeysInMilestone; } @@ -148,19 +150,37 @@ public void setDbLogPath(String dbLogPath) { super.setDbLogPath(dbLogPath); } + @JsonProperty + @Override + public void setSpentAddressesDbPath(String spentAddressesDbPath) { + if (Objects.equals(MainnetConfig.Defaults.SPENT_ADDRESSES_DB_PATH, spentAddressesDbPath)) { + throw new ParameterException("Testnet spent-addresses db folder cannot be configured to mainnet's spent-addresses-db folder"); + } + super.setSpentAddressesDbPath(spentAddressesDbPath); + } + + @JsonProperty + @Override + public void setSpentAddressesDbLogPath(String spentAddressesDbLogPath) { + if (Objects.equals(MainnetConfig.Defaults.SPENT_ADDRESSES_DB_LOG_PATH, spentAddressesDbLogPath)) { + throw new ParameterException("Testnet spent-addresses db log folder cannot be configured to mainnet's spent-addresses-db log folder"); + } + super.setSpentAddressesDbLogPath(spentAddressesDbLogPath); + } + @Override public long getGenesisTime() { return genesisTime; } @JsonProperty - @Parameter(names = {"--genesis-testnet"}, description = MilestoneConfig.Descriptions.GENESIS_TIME) + @Parameter(names = {"--genesis-testnet"}, description = RoundConfig.Descriptions.GENESIS_TIME) protected void setGenesisTime(int genesisTime) { this.genesisTime = genesisTime; } public interface Defaults { long GENESIS_TIME = 1571279107785L; - boolean DONT_VALIDATE_MILESTONE_SIG = false; - String LOCAL_SNAPSHOTS_BASE_PATH = "snapshot-testnet"; + boolean VALIDATE_MILESTONE_SIG = true; + String LOCAL_SNAPSHOTS_BASE_PATH = "testnet-snapshot"; String SNAPSHOT_FILE = "/snapshotTestnet.txt"; int REQUEST_HASH_SIZE = 32; String SNAPSHOT_SIG = "/snapshotTestnet.sig"; @@ -169,8 +189,9 @@ public interface Defaults { int MILESTONE_START_INDEX = 0; int KEYS_IN_MILESTONE = 10; int PACKET_SIZE = 800; - String DB_PATH = "testnetdb"; - String DB_LOG_PATH = "testnetdb.log"; + String DB_PATH = "testnet-db"; + String DB_LOG_PATH = "testnet-db-log"; + String SPENT_ADDRESSES_DB_PATH = "testnet-spent-addresses-db"; + String SPENT_ADDRESSES_DB_LOG_PATH = "testnet-spent-addresses-db-log"; } } - diff --git a/src/main/java/net/helix/pendulum/conf/ValidatorConfig.java b/src/main/java/net/helix/pendulum/conf/ValidatorConfig.java new file mode 100644 index 00000000..1e0f9fe5 --- /dev/null +++ b/src/main/java/net/helix/pendulum/conf/ValidatorConfig.java @@ -0,0 +1,55 @@ +package net.helix.pendulum.conf; + +import net.helix.pendulum.model.Hash; + +import java.util.Set; + +public interface ValidatorConfig extends Config { + + /** + * @return {@value Descriptions#VALIDATOR_SECURITY} + */ + int getValidatorSecurity(); + /** + * @return {@value Descriptions#VALIDATOR} + */ + boolean isValidator(); + /** + * @return {@value Descriptions#VALIDATOR_PATH} + */ + String getValidatorPath(); + /** + * @return {@value Descriptions#VALIDATOR_SEED_PATH} + */ + String getValidatorSeedfile(); + /** + * Returns is validator is enabled, is dependent on isValidator and ValidatorPath + * @return {@value Descriptions#VALIDATOR} + */ + boolean isValidatorEnabled(); + /** + * @return Descriptions#INITIAL_VALIDATORS + */ + Set getInitialValidators(); + /** + * @return {@value Descriptions#VALIDATE_TESTNET_MILESTONE_SIG} + */ + boolean isValidateTestnetMilestoneSig(); + + /** + * @return {@value Descriptions#VALIDATOR_KEYFILE} + */ + String getValidatorKeyfile(); + + interface Descriptions { + String INITIAL_VALIDATORS = "The addresses of validators the network starts with"; + String VALIDATOR = "Flag that enables applying as a validator in the network."; + String VALIDATOR_PATH = "A path to a file containing the seed / keyfile has to be passed."; + String VALIDATOR_SEED_PATH = "A path to a file containing the seed has to be passed."; + String VALIDATE_TESTNET_MILESTONE_SIG = "Disable validator validation on testnet"; + String VALIDATOR_KEYFILE = "Filepath to validator keyfile"; + String VALIDATOR_SECURITY = "Security level of transactions sent from a validator (milestones, registrations)"; + + } + +} diff --git a/src/main/java/net/helix/pendulum/conf/ValidatorManagerConfig.java b/src/main/java/net/helix/pendulum/conf/ValidatorManagerConfig.java index 20ca5656..fd48d95f 100644 --- a/src/main/java/net/helix/pendulum/conf/ValidatorManagerConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ValidatorManagerConfig.java @@ -14,10 +14,6 @@ public interface ValidatorManagerConfig extends Config { * @return Descriptions#VALIDATOR_MANAGER_ADDRESS */ Hash getValidatorManagerAddress(); - /** - * @return {@value Descriptions#DONT_VALIDATE_TESTNET_VALIDATOR_MANAGER_SIG} - */ - boolean isDontValidateTestnetValidatorManagerSig(); /** * @return {@value Descriptions#UPDATE_VALIDATOR_DELAY} */ @@ -43,7 +39,6 @@ public interface ValidatorManagerConfig extends Config { interface Descriptions { String VALIDATOR_MANAGER_ENABLED = "Flag that determines if the node is a validator manager."; String VALIDATOR_MANAGER_ADDRESS = "The address of the node that publishes validators"; - String DONT_VALIDATE_TESTNET_VALIDATOR_MANAGER_SIG = "Disable validatomanager validation on testnet"; String UPDATE_VALIDATOR_DELAY = "The desired delay for updating validators in seconds."; String START_ROUND_DELAY = "The number of rounds between validators are published and the round they start to operate."; String VALIDATOR_MANAGER_KEYFILE = "Filepath to validatomanager keyfile"; diff --git a/src/main/java/net/helix/pendulum/conf/XIConfig.java b/src/main/java/net/helix/pendulum/conf/XIConfig.java index aebbc73a..f5615506 100644 --- a/src/main/java/net/helix/pendulum/conf/XIConfig.java +++ b/src/main/java/net/helix/pendulum/conf/XIConfig.java @@ -5,11 +5,14 @@ */ public interface XIConfig extends Config{ + String XI_DIR = "modules"; + /** * @return Descriptions#XI_DIR */ String getXiDir(); + interface Descriptions { String XI_DIR = "The folder where XI modules should be added for automatic discovery."; } diff --git a/src/main/java/net/helix/pendulum/conf/ZMQConfig.java b/src/main/java/net/helix/pendulum/conf/ZMQConfig.java index 94e62644..e283da98 100644 --- a/src/main/java/net/helix/pendulum/conf/ZMQConfig.java +++ b/src/main/java/net/helix/pendulum/conf/ZMQConfig.java @@ -34,9 +34,8 @@ public interface ZMQConfig extends Config { interface Descriptions { String ZMQ_PORT = "The port used to connect to the ZMQ feed"; String ZMQ_IPC = "The path that is used to communicate with ZMQ in IPC"; - String ZMQ_ENABLED = "Enable zmq channels (deprecated). Use --zmq-enable-tcp or --zmq-enable-ipc instead"; String ZMQ_ENABLE_TCP = "Enable zmq channels on tcp port 5556. Use --zmq-port=[PORT] to override."; - String ZMQ_ENABLE_IPC = "Enable zmq channels on ipc://iri. Use --zmq-ipc=[SOCKET] to override."; + String ZMQ_ENABLE_IPC = "Enable zmq channels on ipc://pendulum. Use --zmq-ipc=[SOCKET] to override."; String ZMQ_THREADS = "The threads used by ZMQ publisher"; } } diff --git a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java index 8d72113d..85d524f0 100644 --- a/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/RoundViewModel.java @@ -12,7 +12,6 @@ import net.helix.pendulum.storage.Persistable; import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Pair; -import net.helix.pendulum.utils.PendulumUtils; import net.helix.pendulum.utils.Serializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -250,18 +249,29 @@ public static Set getMilestoneTrunk(Tangle tangle, TransactionViewModel tr // add previous milestones to non analyzed transactions RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); if (prevMilestone == null) { - if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { + log.trace("prev milestone is null"); + if (transaction.getTrunkTransactionHash().equals(Hash.NULL_HASH)) { + log.trace("prev milestone = trunk = null"); trunk.add(Hash.NULL_HASH); + } else { + log.trace("trunk is not null: {}", transaction.getTrunkTransactionHash()); } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - if (transaction.getTrunkTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + List prevMilestonesList = new ArrayList<>(prevMilestones); + Collections.sort(prevMilestonesList); + List> merkleTree = Merkle.buildMerkleTree(prevMilestonesList); + Hash merkleRoot = merkleTree.get(merkleTree.size() - 1).get(0); + if (transaction.getTrunkTransactionHash().equals(merkleRoot)) { + log.trace("merkle root = trunk"); if (prevMilestones.isEmpty()) { + log.trace("prev round was empty"); trunk.add(Hash.NULL_HASH); } else { trunk.addAll(prevMilestones); } + } else { + log.trace("merkle root ({}) != trunk ({})", merkleRoot, transaction.getTrunkTransactionHash()); } } } @@ -279,31 +289,49 @@ public static Set getMilestoneBranch(Tangle tangle, TransactionViewModel t if (transaction.getCurrentIndex() == transaction.lastIndex()) { // tips merkle root Set confirmedTips = getTipSet(tangle, milestoneTx.getHash(), security); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(confirmedTips)); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size()-1).get(0))) { + List confirmedTipsList = new ArrayList<>(confirmedTips); + Collections.sort(confirmedTipsList); + List> merkleTree = Merkle.buildMerkleTree(confirmedTipsList); + Hash merkleRoot = merkleTree.get(merkleTree.size()-1).get(0); + if (transaction.getBranchTransactionHash().equals(merkleRoot)) { + log.trace("merkle root = branch"); if (confirmedTips.isEmpty()){ + log.trace("confirmed tips = null"); branch.add(Hash.NULL_HASH); } else { branch.addAll(confirmedTips); } + } else { + log.trace("merkle root ({}) != branch ({})", merkleRoot, transaction.getBranchTransactionHash()); } } else { // add previous milestones to non analyzed transactions RoundViewModel prevMilestone = RoundViewModel.get(tangle, round-1); if (prevMilestone == null) { + log.trace("prev milestone is null"); if (transaction.getBranchTransactionHash().equals(Hash.NULL_HASH)) { + log.trace("prev milestone = branch = null"); branch.add(Hash.NULL_HASH); + } else { + log.trace("branch is not null: {}", transaction.getBranchTransactionHash()); } } else { Set prevMilestones = prevMilestone.getHashes(); - List> merkleTree = Merkle.buildMerkleTree(new ArrayList<>(prevMilestones)); - if (transaction.getBranchTransactionHash().equals(merkleTree.get(merkleTree.size() - 1).get(0))) { + List prevMilestonesList = new ArrayList<>(prevMilestones); + Collections.sort(prevMilestonesList); + List> merkleTree = Merkle.buildMerkleTree(prevMilestonesList); + Hash merkleRoot = merkleTree.get(merkleTree.size() - 1).get(0); + if (transaction.getBranchTransactionHash().equals(merkleRoot)) { + log.trace("merkle root = branch"); if (prevMilestones.isEmpty()) { + log.trace("prev round was empty"); branch.add(Hash.NULL_HASH); } else { branch.addAll(prevMilestones); } + } else { + log.trace("merkle root ({}) != branch ({})", merkleRoot, transaction.getBranchTransactionHash()); } } } @@ -475,7 +503,9 @@ public Integer index() { } public Hash getMerkleRoot() { - List> merkleTree = Merkle.buildMerkleTree(new LinkedList<>(getHashes())); + List milestoneHashes = new ArrayList<>(getHashes()); + Collections.sort(milestoneHashes); + List> merkleTree = Merkle.buildMerkleTree(milestoneHashes); Hash root = merkleTree.get(merkleTree.size()-1).get(0); return root; } diff --git a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java index 5079801c..95fc6c38 100644 --- a/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java +++ b/src/main/java/net/helix/pendulum/controllers/TransactionViewModel.java @@ -9,8 +9,10 @@ import net.helix.pendulum.storage.Tangle; import net.helix.pendulum.utils.Converter; import net.helix.pendulum.utils.Pair; +import net.helix.pendulum.utils.log.interval.IntervalLogger; import java.util.*; +import java.util.stream.Collectors; /** @@ -22,6 +24,11 @@ public class TransactionViewModel { private final Transaction transaction; + /** + * Holds the logger of this class (a rate limited logger that doesn't spam the CLI output).
+ */ + private static final IntervalLogger log = new IntervalLogger(TransactionViewModel.class); + public static final int SIZE = 768; public static final long SUPPLY = 4292493394837504L; @@ -111,6 +118,24 @@ public static TransactionViewModel fromHash(Tangle tangle, final Hash hash) thro return transactionViewModel; } + /** + * Get TransactionViewModel of a given transaction hashes. Uses @see #Tangle.load(Class, Indexable) + * @param tangle + * @param hashes transaction hash + * @return TransactionViewModel of the transaction + */ + public static List fromHashes(Set hashes, Tangle tangle) { + return hashes.stream().map( + h -> { + try { + return TransactionViewModel.fromHash(tangle, h); + } catch (Exception e) { + log.error("Could not get transaction for hash " + h, e); + } + return null; + }).filter(t -> t != null).collect(Collectors.toList()); + } + /** * Get TransactionViewModel of a given transaction hash. Uses @see #Tangle.maybeHas(Class, Indexable), * which checks the possible existence of an entry in the database. @@ -611,6 +636,7 @@ public static void updateSolidTransactions(Tangle tangle, Snapshot initialSnapsh if(!transactionViewModel.isSolid()) { transactionViewModel.updateSolid(true); transactionViewModel.update(tangle, initialSnapshot, "solid|height"); + tangle.publish("sldf %s", transactionViewModel.getHash()); } } } diff --git a/src/main/java/net/helix/pendulum/service/API.java b/src/main/java/net/helix/pendulum/service/API.java index 449cd9e7..c63beb00 100644 --- a/src/main/java/net/helix/pendulum/service/API.java +++ b/src/main/java/net/helix/pendulum/service/API.java @@ -178,7 +178,6 @@ public API(ApiArgs args) { commandRoute.put(ApiCommand.GET_MISSING_TRANSACTIONS, getMissingTransactions()); commandRoute.put(ApiCommand.CHECK_CONSISTENCY, checkConsistency()); commandRoute.put(ApiCommand.WERE_ADDRESSES_SPENT_FROM, wereAddressesSpentFrom()); - // commandRoute.put(ApiCommand.GET_MILESTONES, wereAddressesSpentFrom()); } /** @@ -247,7 +246,7 @@ private AbstractResponse process(final String requestString, InetAddress netAddr // Is this command allowed to be run from this request address? // We check the remote limit API configuration. - if (configuration.getRemoteLimitApi().contains(command) && !configuration.getRemoteTrustedApiHosts().contains(netAddress)) { + if (configuration.getIgnoredApiEndpoints().contains(command) && !configuration.getAllowedApiHosts().contains(netAddress)) { return AccessLimitedResponse.create("COMMAND " + command + " is not available on this node"); } @@ -702,7 +701,7 @@ private AbstractResponse getConfirmationStatesStatement(final List trans log.trace("tx_confirmations {}:[{}:{}]", transaction.getHash().toString(), transaction.getConfirmations(), (double) transaction.getConfirmations() / n); // is transaction finalized - if(((double)transaction.getConfirmations() / n) > threshold) { + if(transaction.getRoundIndex() > 0 && ((double)transaction.getConfirmations() / n) > threshold) { confirmationStates[count] = 1; } // not finalized yet @@ -1678,6 +1677,7 @@ private List addMilestoneReferences(List confirmedTips, int roundInd txToApprove.add(previousRound.getMerkleRoot()); // merkle root of latest milestones } //branch + Collections.sort(confirmedTips); // sort tips before building merkle root List> merkleTreeTips = Merkle.buildMerkleTree(confirmedTips); txToApprove.add(merkleTreeTips.get(merkleTreeTips.size() - 1).get(0)); // merkle root of confirmed tips } diff --git a/src/main/java/net/helix/pendulum/service/Feature.java b/src/main/java/net/helix/pendulum/service/Feature.java index b954f3aa..80616010 100644 --- a/src/main/java/net/helix/pendulum/service/Feature.java +++ b/src/main/java/net/helix/pendulum/service/Feature.java @@ -75,7 +75,7 @@ public static Feature[] calculateFeatures(PendulumConfig configuration) { List apiFeatures = new ArrayList(Arrays.asList(PROOF_OF_WORK)); - for (String disabled : configuration.getRemoteLimitApi()) { + for (String disabled : configuration.getIgnoredApiEndpoints()) { switch (disabled) { case "attachToTangle": apiFeatures.remove(PROOF_OF_WORK); diff --git a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java index abe258bb..80b7c058 100644 --- a/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/ledger/impl/LedgerServiceImpl.java @@ -283,20 +283,27 @@ private boolean generateStateDiff(RoundViewModel round) throws LedgerException { Map balanceChanges = generateBalanceDiff(new HashSet<>(), confirmedTips == null? new HashSet<>() : confirmedTips, snapshotProvider.getLatestSnapshot().getIndex() + 1); successfullyProcessed = balanceChanges != null; - if (successfullyProcessed) { - successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( - new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); - if (successfullyProcessed) { - milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); - - if (!balanceChanges.isEmpty()) { - new StateDiffViewModel(balanceChanges, round.index()).store(tangle); - } + if (successfullyProcessed) { + successfullyProcessed = snapshotProvider.getLatestSnapshot().patchedState( + new SnapshotStateDiffImpl(balanceChanges)).isConsistent(); + TransactionViewModel.fromHashes(confirmedTips, tangle).forEach(tvm -> { + try { + tvm.setRoundIndex(tvm.getRoundIndex() == 0 ? round.index() : tvm.getRoundIndex()); + tvm.update(tangle, snapshotProvider.getInitialSnapshot(), "roundIndex"); + } catch (Exception e) { + log.error("Error during transaction round index update: " + tvm.getHash(), e); } + }); + + milestoneService.updateRoundIndexOfMilestoneTransactions(round.index()); + + if (!balanceChanges.isEmpty()) { + new StateDiffViewModel(balanceChanges, round.index()).store(tangle); } - } finally { - snapshotProvider.getLatestSnapshot().unlockRead(); } + } finally { + snapshotProvider.getLatestSnapshot().unlockRead(); + } return successfullyProcessed; } catch (Exception e) { diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java index 05675696..53ac6b6d 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/LatestSolidMilestoneTrackerImpl.java @@ -143,40 +143,70 @@ public void shutdown() { @Override public void trackLatestSolidMilestones() throws MilestoneException { try { + int currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + int latestMilestoneIndex = milestoneTracker.getCurrentRoundIndex(); + log.delegate().trace("Current Solid Round = " + currentSolidRoundIndex + ", Current Milestone index = " + latestMilestoneIndex + + ", Still in Round = " + (currentSolidRoundIndex == latestMilestoneIndex - 1) + + ", Round active = " + milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime())); + + RoundViewModel nextRound; - if (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) { - log.delegate().trace("Current Solid Round = " + currentSolidRoundIndex + ", Current Round = " + milestoneTracker.getCurrentRoundIndex() + ", Still in Round = " + (currentSolidRoundIndex == milestoneTracker.getCurrentRoundIndex() - 1) + ", Round active = " + milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime())); - } - while (!Thread.currentThread().isInterrupted() && (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) - && (currentSolidRoundIndex != milestoneTracker.getCurrentRoundIndex() - 1 || !milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime()))) { + //if (currentSolidRoundIndex < milestoneTracker.getCurrentRoundIndex()) { + // log.delegate().trace("Current Solid Round = " + currentSolidRoundIndex + ", Current Round = " + milestoneTracker.getCurrentRoundIndex() + ", Still in Round = " + (currentSolidRoundIndex == milestoneTracker.getCurrentRoundIndex() - 1) + ", Round active = " + milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime())); + //} + while (!Thread.currentThread().isInterrupted()) { + currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); + latestMilestoneIndex = milestoneTracker.getCurrentRoundIndex(); + + if (currentSolidRoundIndex >= latestMilestoneIndex) { + tracer.trace("Latest milestone index is behind current solid round"); + break; + } + + if ((currentSolidRoundIndex == latestMilestoneIndex - 1) && milestoneTracker.isRoundActive(RoundIndexUtil.getCurrentTime())) { + tracer.trace("The round is still active, nothing to update"); + break; + } + nextRound = RoundViewModel.get(tangle, currentSolidRoundIndex + 1); if (nextRound == null) { // round has finished without milestones RoundViewModel latest = RoundViewModel.latest(tangle); - if (latest != null && latest.index() > currentSolidRoundIndex + 1 && isRoundSolid(latest)) { - log.delegate().trace("Round #" + (currentSolidRoundIndex + 1) + " has finished without milestones"); - nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>()); - nextRound.store(tangle); - tracer.trace("created and stored empty round: {}", nextRound.index()); + tracer.trace("Latest round: {}", latest); + if (latest == null) { + tracer.trace("Latest round is null"); + break; + } + + if (latest.index() <= currentSolidRoundIndex + 1) { + tracer.trace("Latest round is not ahead current solid round "); + break; } - // round hasn't finished yet - else { - tracer.trace("round has not finished: {}", latest); + + if (!isRoundSolid(latest)) { + tracer.trace("Latest round is not solid"); break; } + + log.delegate().trace("Round #" + (currentSolidRoundIndex + 1) + " has finished without milestones"); + nextRound = new RoundViewModel(currentSolidRoundIndex + 1, new HashSet<>()); + nextRound.store(tangle); + tracer.trace("created and stored empty round: {}", nextRound.index()); } - log.delegate().trace("Round = " + nextRound.index() + ", solid = " + isRoundSolid(nextRound)); - if (isRoundSolid(nextRound)) { + + boolean nextRoundSolid = isRoundSolid(nextRound); + + log.delegate().trace("Round = " + nextRound.index() + ", solid = " + nextRoundSolid); + if (nextRoundSolid) { tracer.trace("solid round: {}", nextRound.toString()); // TODO: Ask Oliver about these classes? //syncValidatorTracker(); //syncLatestMilestoneTracker(nextRound.index()); applyRoundToLedger(nextRound); logChange(currentSolidRoundIndex); - currentSolidRoundIndex = snapshotProvider.getLatestSnapshot().getIndex(); tangle.publish("ctx %s %d", nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)), nextRound.index()); log.delegate().trace("ctx= " + nextRound.getReferencedTransactions(tangle, nextRound.getConfirmedTips(tangle, BasePendulumConfig.Defaults.VALIDATOR_SECURITY)) + ", round=" + nextRound.index()); } diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java index 6f518758..d67f123c 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestonePublisher.java @@ -47,7 +47,7 @@ public MilestonePublisher(PendulumConfig configuration, API api, CandidateTracke delay = config.getRoundDuration(); mwm = config.getMwm(); - sign = !config.isDontValidateTestnetMilestoneSig(); + sign = config.isValidateTestnetMilestoneSig(); pubkeyDepth = config.getMilestoneKeyDepth(); keyfileIndex = 0; maxKeyIndex = (int) Math.pow(2, pubkeyDepth); diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java index cae1f9a4..aac82451 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneServiceImpl.java @@ -185,7 +185,7 @@ public MilestoneValidity validateMilestone(TransactionViewModel transactionViewM Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); log.trace("valid signature: {}", validSignature); - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || + if ((config.isTestnet() && !config.isValidateTestnetMilestoneSig()) || (validatorAddresses.contains(senderAddress)) && validSignature) { transactionViewModel.isMilestone(tangle, snapshotProvider.getInitialSnapshot(), true); diff --git a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java index 07bc196b..443b0294 100644 --- a/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/milestone/impl/MilestoneTrackerImpl.java @@ -201,8 +201,6 @@ public void setRoundIndexAndConfirmations(RoundViewModel currentRVM, Transaction // The confirmation counter should be incremented with each milestone reference for (Hash tx : referencedTipSet) { TransactionViewModel txvm = TransactionViewModel.fromHash(tangle, tx); - txvm.setRoundIndex(txvm.getRoundIndex() == 0 ? roundIndex : txvm.getRoundIndex()); - txvm.update(tangle, snapshotProvider.getInitialSnapshot(), "roundIndex"); txvm.setConfirmations(txvm.getConfirmations() + 1); txvm.update(tangle, snapshotProvider.getInitialSnapshot(), "confirmation"); } @@ -232,8 +230,8 @@ public int getCurrentRoundIndex() { @Override public int getRound(long time) { return config.isTestnet() ? - RoundIndexUtil.getRound(time, TestnetConfig.Defaults.GENESIS_TIME, BasePendulumConfig.Defaults.ROUND_DURATION) : - RoundIndexUtil.getRound(time, BasePendulumConfig.Defaults.GENESIS_TIME, BasePendulumConfig.Defaults.ROUND_DURATION); + RoundIndexUtil.getRound(time, TestnetConfig.Defaults.GENESIS_TIME, roundDuration) : + RoundIndexUtil.getRound(time, BasePendulumConfig.Defaults.GENESIS_TIME, roundDuration); } @Override @@ -305,38 +303,32 @@ public boolean processMilestoneCandidate(TransactionViewModel transaction) throw // - senderAddress must be part of validator addresses // - signature belongs to senderAddress // - index is bigger than snapshot index - // - attachment timestamp is in correct time window for the index + // - attachment timestamp is in correct time window for the index (removed) // - there doesn't already exist a milestone with the same address for that round - long calculatedRoundIndex = getRound(transaction.getAttachmentTimestamp()); - tracer.trace("ri, calculated ri= {}, {}", roundIndex, calculatedRoundIndex); - if (roundIndex == calculatedRoundIndex && isRoundActive(transaction.getAttachmentTimestamp())) { - tracer.trace("round is active"); - RoundViewModel currentRoundViewModel; - - // a milestone already arrived for that round, just update - if ((currentRoundViewModel = RoundViewModel.get(tangle, roundIndex)) != null) { - // check if there is already a milestone with the same address - if (RoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { - currentRoundViewModel.addMilestone(transaction.getHash()); - currentRoundViewModel.update(tangle); - tracer.trace("updated the round with mstn: {}", transaction.getHash()); - } + RoundViewModel currentRoundViewModel; + // a milestone already arrived for that round, just update + if ((currentRoundViewModel = RoundViewModel.get(tangle, roundIndex)) != null) { + // check if there is already a milestone with the same address + if (RoundViewModel.getMilestone(tangle, roundIndex, transaction.getAddressHash()) == null) { + currentRoundViewModel.addMilestone(transaction.getHash()); + currentRoundViewModel.update(tangle); + tracer.trace("updated the round with mstn: {}", transaction.getHash()); + } else { + tracer.trace("already exists a milestone from this validator: {}", transaction.getAddressHash()); } - // this is the first milestone for that round, make new database entry - else { - tracer.trace("current rvm is null"); - Set milestones = new HashSet<>(); - milestones.add(transaction.getHash()); - currentRoundViewModel = new RoundViewModel(roundIndex, milestones); - currentRoundViewModel.store(tangle); - } - addMilestoneToRoundLog(transaction.getHash(), roundIndex, currentRoundViewModel.size(), validators.size()); - setRoundIndexAndConfirmations(currentRoundViewModel, transaction, roundIndex); - publishMilestoneRefs(transaction); - } else { - tracer.trace("round is not active"); } + // this is the first milestone for that round, make new database entry + else { + tracer.trace("current rvm is null"); + Set milestones = new HashSet<>(); + milestones.add(transaction.getHash()); + currentRoundViewModel = new RoundViewModel(roundIndex, milestones); + currentRoundViewModel.store(tangle); + } + addMilestoneToRoundLog(transaction.getHash(), roundIndex, currentRoundViewModel.size(), validators.size()); + setRoundIndexAndConfirmations(currentRoundViewModel, transaction, roundIndex); + publishMilestoneRefs(transaction); if (!transaction.isSolid()) { tracer.trace("non solid: {}", transaction.getHash()); diff --git a/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java b/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java index 9c6c905d..ed3ec99b 100644 --- a/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java +++ b/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotProviderImpl.java @@ -1,7 +1,7 @@ package net.helix.pendulum.service.snapshot.impl; import net.helix.pendulum.SignedFiles; -import net.helix.pendulum.conf.SnapshotConfig; +import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.service.snapshot.*; @@ -60,7 +60,7 @@ public class SnapshotProviderImpl implements SnapshotProvider { /** * Holds a cached version of the builtin snapshot. * - * Note: The builtin snapshot is embedded in the iri.jar and will not change. To speed up tests that need the + * Note: The builtin snapshot is embedded in the pendulum.jar and will not change. To speed up tests that need the * snapshot multiple times while creating their own version of the LocalSnapshotManager, we cache the instance * here so they don't have to rebuild it from the scratch every time (massively speeds up the unit tests). */ @@ -70,7 +70,8 @@ public class SnapshotProviderImpl implements SnapshotProvider { /** * Holds Snapshot related configuration parameters. */ - private SnapshotConfig config; + private ConsensusConfig config; + /** * Internal property for the value returned by {@link SnapshotProvider#getInitialSnapshot()}. @@ -99,7 +100,7 @@ public class SnapshotProviderImpl implements SnapshotProvider { * @return the initialized instance itself to allow chaining * */ - public SnapshotProviderImpl init(SnapshotConfig config) throws SnapshotException { + public SnapshotProviderImpl init(ConsensusConfig config) throws SnapshotException { this.config = config; File pathToLocalSnapshotDir = Paths.get(this.config.getLocalSnapshotsBasePath()).toFile(); if (!pathToLocalSnapshotDir.exists() || !pathToLocalSnapshotDir.isDirectory()) { diff --git a/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java b/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java index 96fa0f4a..bddd0641 100644 --- a/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/snapshot/impl/SnapshotServiceImpl.java @@ -189,8 +189,8 @@ public void replayMilestones(Snapshot snapshot, int targetRoundIndex) throws Sna // todo: unfortunately we need to have getRound in milestoneTracker and here, as RoundIndexUtils is static and we need to check isTestnet. public int getRound(long time) { return config.isTestnet() ? - RoundIndexUtil.getRound(time, TestnetConfig.Defaults.GENESIS_TIME, BasePendulumConfig.Defaults.ROUND_DURATION) : - RoundIndexUtil.getRound(time, BasePendulumConfig.Defaults.GENESIS_TIME, BasePendulumConfig.Defaults.ROUND_DURATION); + RoundIndexUtil.getRound(time, TestnetConfig.Defaults.GENESIS_TIME, config.getRoundDuration()) : + RoundIndexUtil.getRound(time, BasePendulumConfig.Defaults.GENESIS_TIME, config.getRoundDuration()); } /** diff --git a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java index 57ede2e6..7bd5ed92 100644 --- a/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java +++ b/src/main/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImpl.java @@ -1,7 +1,7 @@ package net.helix.pendulum.service.spentaddresses.impl; +import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.conf.SnapshotConfig; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; import net.helix.pendulum.model.persistables.SpentAddress; @@ -30,20 +30,14 @@ */ public class SpentAddressesProviderImpl implements SpentAddressesProvider { - //@VisibleForTesting - public RocksDBPersistenceProvider rocksDBPersistenceProvider; + private RocksDBPersistenceProvider rocksDBPersistenceProvider; - private SnapshotConfig config; + private ConsensusConfig config; /** * Creates a new instance of SpentAddressesProvider */ public SpentAddressesProviderImpl() { - Map> columnFamilies = new HashMap<>(); - columnFamilies.put("spent-addresses", SpentAddress.class); - this.rocksDBPersistenceProvider = new RocksDBPersistenceProvider(SPENT_ADDRESSES_DB, - SPENT_ADDRESSES_LOG, 1000, - columnFamilies, null); } /** @@ -53,11 +47,15 @@ public SpentAddressesProviderImpl() { * @return the current instance * @throws SpentAddressesException if we failed to create a file at the designated location */ - public SpentAddressesProviderImpl init(SnapshotConfig config) - throws SpentAddressesException { + public SpentAddressesProviderImpl init(ConsensusConfig config) throws SpentAddressesException { this.config = config; + Map> columnFamilies = new HashMap<>(); + columnFamilies.put("spent-addresses", SpentAddress.class); + rocksDBPersistenceProvider = new RocksDBPersistenceProvider( + config.getSpentAddressesDbPath(), config.getSpentAddressesDbLogPath(), + 1000, columnFamilies, null); try { - this.rocksDBPersistenceProvider.init(); + rocksDBPersistenceProvider.init(); readPreviousEpochsSpentAddresses(); } catch (Exception e) { @@ -65,6 +63,12 @@ public SpentAddressesProviderImpl init(SnapshotConfig config) } return this; } + + public void shutdown() { + if (rocksDBPersistenceProvider != null) { + rocksDBPersistenceProvider.shutdown(); + } + } private void readPreviousEpochsSpentAddresses() throws SpentAddressesException { if (config.isTestnet()) { diff --git a/src/main/java/net/helix/pendulum/service/transactionpruning/async/AsyncTransactionPruner.java b/src/main/java/net/helix/pendulum/service/transactionpruning/async/AsyncTransactionPruner.java index 2fa3976b..b3d254dd 100644 --- a/src/main/java/net/helix/pendulum/service/transactionpruning/async/AsyncTransactionPruner.java +++ b/src/main/java/net/helix/pendulum/service/transactionpruning/async/AsyncTransactionPruner.java @@ -1,7 +1,7 @@ package net.helix.pendulum.service.transactionpruning.async; import net.helix.pendulum.conf.BasePendulumConfig; -import net.helix.pendulum.conf.SnapshotConfig; +import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.controllers.TipsViewModel; import net.helix.pendulum.service.snapshot.SnapshotProvider; import net.helix.pendulum.service.spentaddresses.SpentAddressesService; @@ -75,7 +75,7 @@ public class AsyncTransactionPruner implements TransactionPruner { /** * Configuration with important snapshot related parameters. */ - private SnapshotConfig config; + private ConsensusConfig config; /** * Holds a reference to the {@link ThreadIdentifier} for the cleanup thread. @@ -129,7 +129,7 @@ public class AsyncTransactionPruner implements TransactionPruner { */ public AsyncTransactionPruner init(Tangle tangle, SnapshotProvider snapshotProvider, SpentAddressesService spentAddressesService, TipsViewModel tipsViewModel, - SnapshotConfig config) { + ConsensusConfig config) { this.tangle = tangle; this.snapshotProvider = snapshotProvider; diff --git a/src/main/java/net/helix/pendulum/service/transactionpruning/async/MilestonePrunerJobQueue.java b/src/main/java/net/helix/pendulum/service/transactionpruning/async/MilestonePrunerJobQueue.java index a0b6c3ba..1a4e2c0c 100644 --- a/src/main/java/net/helix/pendulum/service/transactionpruning/async/MilestonePrunerJobQueue.java +++ b/src/main/java/net/helix/pendulum/service/transactionpruning/async/MilestonePrunerJobQueue.java @@ -1,6 +1,6 @@ package net.helix.pendulum.service.transactionpruning.async; -import net.helix.pendulum.conf.SnapshotConfig; +import net.helix.pendulum.conf.ConsensusConfig; import net.helix.pendulum.service.transactionpruning.TransactionPruner; import net.helix.pendulum.service.transactionpruning.TransactionPrunerJobStatus; import net.helix.pendulum.service.transactionpruning.TransactionPruningException; @@ -40,12 +40,12 @@ public class MilestonePrunerJobQueue implements JobQueue { * different way than other jobs and consolidate the queue whenever we add a new job. * * @param transactionPruner reference to the container of this queue - * @param snapshotConfig reference to the config with snapshot related parameters + * @param consensusConfig reference to the consensus related configuration parameters */ - public MilestonePrunerJobQueue(TransactionPruner transactionPruner, SnapshotConfig snapshotConfig) { + public MilestonePrunerJobQueue(TransactionPruner transactionPruner, ConsensusConfig consensusConfig) { this.transactionPruner = transactionPruner; - youngestFullyCleanedMilestoneIndex = snapshotConfig.getMilestoneStartIndex(); + youngestFullyCleanedMilestoneIndex = consensusConfig.getMilestoneStartIndex(); } /** diff --git a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java index 36780497..0dcec02c 100644 --- a/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validator/impl/ValidatorServiceImpl.java @@ -60,7 +60,7 @@ public ValidatorValidity validateValidators(TransactionViewModel transactionView Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getValidatorManagerKeyDepth()); - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (config.getValidatorManagerAddress().equals(senderAddress) && validSignature)) { + if ((config.isTestnet() && !config.isValidateTestnetMilestoneSig()) || (config.getValidatorManagerAddress().equals(senderAddress) && validSignature)) { return VALID; } else { return INVALID; diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/CandidateTrackerImpl.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/CandidateTrackerImpl.java index abc11964..f628c3f9 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/CandidateTrackerImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/CandidateTrackerImpl.java @@ -1,10 +1,7 @@ package net.helix.pendulum.service.validatormanager.impl; import net.helix.pendulum.conf.PendulumConfig; -import net.helix.pendulum.controllers.AddressViewModel; -import net.helix.pendulum.controllers.BundleViewModel; -import net.helix.pendulum.controllers.TransactionViewModel; -import net.helix.pendulum.controllers.ValidatorViewModel; +import net.helix.pendulum.controllers.*; import net.helix.pendulum.crypto.SpongeFactory; import net.helix.pendulum.model.Hash; import net.helix.pendulum.model.HashFactory; @@ -143,6 +140,7 @@ public CandidateTrackerImpl init(Tangle tangle, SnapshotProvider snapshotProvide validators = config.getInitialValidators(); startRound = RoundIndexUtil.getRound(RoundIndexUtil.getCurrentTime(), config.getGenesisTime(), config.getRoundDuration(), 2); + recoverValidators(); return this; } @@ -303,6 +301,31 @@ public boolean processCandidate(TransactionViewModel transaction) throws Validat } } + /** + * Try to recover the latest persisted validator set. + * + */ + private void recoverValidators() { + try { + RoundViewModel latest = RoundViewModel.latest(tangle); + if (latest == null) { + log.debug("Latest round is null"); + return; + } + + ValidatorViewModel vvm = ValidatorViewModel.findClosestPrevValidators(tangle, latest.index()); + Set recovered = vvm.getHashes(); + if (recovered == null || recovered.isEmpty()) { + log.debug("The recovered set of validators is empty"); + return; + } + + validators = new HashSet<>(recovered); + } catch (Exception e) { + log.error("Could not recover the latest validator set, using the initial set", e); + } + } + /** * {@inheritDoc} * Adds a candidate to the {@code candidatesToNominate} diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java index ca25e285..32732807 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorManagerServiceImpl.java @@ -76,7 +76,7 @@ public CandidateValidity validateCandidate(TransactionViewModel transactionViewM Hash senderAddress = tail.getAddressHash(); boolean validSignature = Merkle.validateMerkleSignature(bundleTransactionViewModels, mode, senderAddress, securityLevel, config.getMilestoneKeyDepth()); - if ((config.isTestnet() && config.isDontValidateTestnetMilestoneSig()) || (validator.contains(senderAddress)) && validSignature) { + if ((config.isTestnet() && !config.isValidateTestnetMilestoneSig()) || (validator.contains(senderAddress)) && validSignature) { return VALID; } else { return INVALID; diff --git a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorPublisher.java b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorPublisher.java index d654eacd..b6ff562c 100644 --- a/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorPublisher.java +++ b/src/main/java/net/helix/pendulum/service/validatormanager/impl/ValidatorPublisher.java @@ -26,7 +26,7 @@ public ValidatorPublisher(PendulumConfig configuration, API api) { this.api = api; delay = config.getUpdateValidatorDelay(); mwm = config.getMwm(); - sign = !config.isDontValidateTestnetValidatorManagerSig(); + sign = config.isValidateTestnetMilestoneSig(); currentKeyIndex = 0; startRoundDelay = config.getStartRoundDelay(); } diff --git a/src/main/java/net/helix/pendulum/storage/Tangle.java b/src/main/java/net/helix/pendulum/storage/Tangle.java index d118e3e5..4462efb9 100644 --- a/src/main/java/net/helix/pendulum/storage/Tangle.java +++ b/src/main/java/net/helix/pendulum/storage/Tangle.java @@ -146,6 +146,7 @@ public void publish(String message, Object... objects) { for(MessageQProvider provider: this.messageQProviders) { provider.publish(message, objects); } + log.trace(String.format(message, objects)); } public Set keysWithMissingReferences(Class modelClass, Class referencedClass) throws Exception { diff --git a/src/main/java/net/helix/pendulum/utils/collections/impl/BoundedLinkedListImpl.java b/src/main/java/net/helix/pendulum/utils/collections/impl/BoundedLinkedListImpl.java new file mode 100644 index 00000000..cc154543 --- /dev/null +++ b/src/main/java/net/helix/pendulum/utils/collections/impl/BoundedLinkedListImpl.java @@ -0,0 +1,236 @@ +package net.helix.pendulum.utils.collections.impl; + +import net.helix.pendulum.utils.collections.interfaces.BoundedLinkedSet; +import org.apache.commons.collections4.set.ListOrderedSet; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; +import java.util.concurrent.locks.ReentrantLock; + +/** + * Date: 2019-11-15 + * Author: zhelezov + */ +public class BoundedLinkedListImpl implements BoundedLinkedSet { + + private static final Logger log = LoggerFactory.getLogger(BoundedLinkedListImpl.class); + + + private ListOrderedSet queue = ListOrderedSet.listOrderedSet(new LinkedList()); + private ReentrantLock lock = new ReentrantLock(true); + private int maxCapacity; + + public BoundedLinkedListImpl(int maxCapacity) { + this.maxCapacity = maxCapacity; + } + + @Override + public LinkedList drain() { + LinkedList drainedTo = new LinkedList<>(); + lock.lock(); + try { + drainedTo.addAll(queue); + queue.clear(); + } finally { + lock.unlock(); + } + return drainedTo; + } + + @Override + public E poll() { + lock.lock(); + E first = null; + try { + Iterator it = queue.iterator(); + if (it.hasNext()) { + first = it.next(); + it.remove(); + } + } finally { + lock.unlock(); + } + return first; + } + + @Override + public E peek() { + lock.lock(); + E first = null; + try { + Iterator it = queue.iterator(); + if (it.hasNext()) { + first = it.next(); + } + } finally { + lock.unlock(); + } + return first; + } + + @Override + public boolean push(E element) { + lock.lock(); + try { + if (queue.contains(element)) { + return false; + } + + if (queue.size() >= maxCapacity) { + // TODO: different eviction policies + log.warn("The queue reached it max capacity, dropping the last element"); + queue.remove(queue.size()-1); + } + + queue.add(0, element); + } finally { + lock.unlock(); + } + return false; + } + + @Override + public List batchPoll(int size) { + lock.lock(); + List popped = new LinkedList<>(); + try { + Iterator it = queue.iterator(); + int counter = 0; + while ((counter < size) && it.hasNext()) { + popped.add(it.next()); + it.remove(); + counter++; + } + return popped; + } finally { + lock.unlock(); + } + } + + @Override + public int getMaxSize() { + return maxCapacity; + } + + @Override + public int size() { + lock.lock(); + try { + return queue.size(); + } finally { + lock.unlock(); + } + } + + @Override + public boolean isEmpty() { + lock.lock(); + try { + return queue.isEmpty(); + } finally { + lock.unlock(); + } + } + + @Override + public boolean contains(Object o) { + lock.lock(); + try { + return queue.contains(o); + } finally { + lock.unlock(); + } + } + + @Override + public Iterator iterator() { + lock.lock(); + try { + List copy = new ArrayList<>(queue); + return copy.iterator(); + } finally { + lock.unlock(); + } + } + + @Override + public Object[] toArray() { + lock.lock(); + try { + return queue.toArray(); + } finally { + lock.unlock(); + } + } + + @Override + public T[] toArray(T[] a) { + lock.lock(); + try { + return queue.toArray(a); + } finally { + lock.unlock(); + } + } + + @Override + public boolean add(E e) { + lock.lock(); + try { + if (queue.size() >= maxCapacity) { + // TODO: different eviction policies + log.warn("The queue reached it max capacity, dropping the element"); + return false; + } + return queue.add(e); + } finally { + lock.unlock(); + } + } + + @Override + public boolean remove(Object o) { + lock.lock(); + try { + return queue.remove(o); + } finally { + lock.unlock(); + } + } + + @Override + public boolean containsAll(Collection c) { + lock.lock(); + try { + return queue.containsAll(c); + } finally { + lock.unlock(); + } + } + + @Override + public boolean addAll(Collection c) { + throw new UnsupportedOperationException("Add all not supported"); + } + + @Override + public boolean removeAll(Collection c) { + throw new UnsupportedOperationException("Remove all not supported"); + } + + @Override + public boolean retainAll(Collection c) { + throw new UnsupportedOperationException("Retain all not supported"); + } + + @Override + public void clear() { + lock.lock(); + try { + queue.clear(); + } finally { + lock.unlock(); + } + } +} diff --git a/src/main/java/net/helix/pendulum/utils/collections/interfaces/BoundedLinkedSet.java b/src/main/java/net/helix/pendulum/utils/collections/interfaces/BoundedLinkedSet.java new file mode 100644 index 00000000..fc57c9f1 --- /dev/null +++ b/src/main/java/net/helix/pendulum/utils/collections/interfaces/BoundedLinkedSet.java @@ -0,0 +1,52 @@ +package net.helix.pendulum.utils.collections.interfaces; + +import java.util.LinkedList; +import java.util.List; + +/** + * A double-sided queue (e.g. can work as a stack and as a queue) + * with bounded capacity and without duplicates. Similar to LinkedHashSet + * but with following differences: + * + * -- thread-safe + * -- A copy iterator never throws ConcurrentModificationException + * as a new copy is created + * + * Date: 2019-11-15 + * Author: zhelezov + */ +public interface BoundedLinkedSet extends BoundedCollection { + + /** + * Drains the queue into a copy + * @return List of the elements in the same order as they were in the queue + */ + LinkedList drain(); + + /** + * Retrieves the first element + * @return null if empty + */ + E poll(); + + /** + * Returns the first element without removal + * @return null if empty + */ + E peek(); + + + /** + * Adds the top of the queue + * @return true if added + */ + boolean push(E element); + + /** + * Retrieves up to size top elements from the queue + * @param size the maximal number of elements to retrieve + * @return the list containing the elements + */ + List batchPoll(int size); + +} diff --git a/src/main/resources/logback-save.xml b/src/main/resources/logback-save.xml index f74267b8..b2cfae52 100644 --- a/src/main/resources/logback-save.xml +++ b/src/main/resources/logback-save.xml @@ -58,6 +58,24 @@ + + ${user.dir}/logs/tangle-${instance.name}.log + + + + ${user.dir}/logs/tangle-${instance.name}.%d{yyyy-MM-dd}.log + + + 30 + 10GB + + + + + %d %msg%n + + + System.err @@ -80,4 +98,8 @@ + + + + diff --git a/src/test/java/net/helix/pendulum/conf/ConfigTest.java b/src/test/java/net/helix/pendulum/conf/ConfigTest.java index 8791aa0c..168b8b60 100644 --- a/src/test/java/net/helix/pendulum/conf/ConfigTest.java +++ b/src/test/java/net/helix/pendulum/conf/ConfigTest.java @@ -45,6 +45,177 @@ public static void tearDownAfterClass() throws IOException { FileUtils.forceDelete(configFile); } + /* + test that --remote returns true without specifically giving true on cmd line + */ + @Test + public void remoteFlagTest() { + String[] args = {"--remote"}; + PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); + pendulumConfig.parseConfigFromArgs(args); + Assert.assertEquals("The api interface should be open to the public", + "0.0.0.0", pendulumConfig.getApiHost()); + } + + /* + The order of flags provided on cmd line makes a difference. + Test that specifying --remote after any --api_host flag will set the value for api_host. + */ + @Test + public void testFlagPrecedentRule(){ + String[] args = { + "--api_host", "localhost", + "--remote" + }; + PendulumConfig penConfig = ConfigFactory.createPendulumConfig(false); + penConfig.parseConfigFromArgs(args); + Assert.assertNotEquals(penConfig.getApiHost(), "localhost"); + Assert.assertEquals(penConfig.getApiHost(), "0.0.0.0"); + } + @Test + public void testIgnoredApiHosts(){ + String[] args = { + "--ignored_api_endpoints", + "addNeighbor, getNeighbors, removeNeighbors, attachToTangle, interruptAttachingToTangle" + }; + PendulumConfig penConfig = ConfigFactory.createPendulumConfig(false); + penConfig.parseConfigFromArgs(args); + Assert.assertEquals(penConfig.getIgnoredApiEndpoints(), + Arrays.asList( + "addNeighbor", "getNeighbors", "removeNeighbors", "attachToTangle", "interruptAttachingToTangle") + ); + } + /* + Test all known command line arguments with non-default value to make sure they are being parsed and set correctly. + */ + @Test + public void testCommandLineArgs(){ + String[] args = { + "--xi_dir", "not_here", + "--remote", "true", + //"--api_host", "localhost", + "--api_port", "1234", + //"--allowed_api_hosts", "0.0.0.0", + "--pow_threads", "2", + //"--remote_auth", "pendulum:swings", + //"--ignored_api_endpoints", "addNeighbor, getNeighbors, removeNeighbors, attachToTangle, interruptAttachingToTangle", + "--max_body_length", "123456", + "--max_find_transactions", "123456", + "--max_get_transaction_strings", "123456", + "--max_requests_list", "123456", + "--db", "mongodb", //not available, but we're just making sure the cmd line flags work as expected + "--db_cache_size", "123456", + "--db_log_path", "somewhere_fancy.log", + "--db_path", "some_dir", + "--local_snapshots_base_path", "./snapshot-go-here", + "--local_snapshots_depth", "123456", + "--local_snapshots_enabled", "false", + "--local_snapshots_interval_synced", "123456", + "--local_snapshots_interval_unsynced", "123456", + "--local_snapshots_pruning_delay", "123456", + "--local_snapshots_pruning_enabled", "true", + "--neighbors", "udp://localhost:4101", + "--tcp_receiver_port", "1234", + "--udp_receiver_port", "1234", + "--max_peers", "1234", + "--p_drop_cache", "1.0", + "--p_drop_transaction", "1.0", + "--p_propagate_request", "1.0", + "--p_remove_request", "1.0", + "--p_reply_random", "1.0", + "--p_select_milestone", "1.0", + "--p_send_milestone", "1.0", + "--queue_size", "1234", + "--dns_refresher", "false", + "--dns_resolution", "false", + "--send_limit", "2", + "--cache_size", "1234", + "--update_validator_delay", "1234", + "--validator", "true", + "--validator_manager", "true", + "--validator_path", "./somewhere.txt", + "--start_validator", "10", + "--round", "60000", + "--round_pause", "60000", + "--max_analyzed_transactions", "60000", + "--max_depth", "10", + "--rescan", "true", + "--revalidate", "true", + "--remote_auth", "pendulum:swings", + "--alpha", "1.0", + "--genesis", "2345024001000", + "--zmq_enable_ipc", "true", + "--zmq_enable_tcp", "true", + "--zmq_enabled", "true", + "--zmq_ipc", "ipc://pendulum", + "--zmq_port", "1234", + "--zmq_threads", "10", + "--savelog_enabled", "true", + "--savelog_path", "./somewhere/", + "--savelog_xml", "/save.xml" + }; + + PendulumConfig penConfig = ConfigFactory.createPendulumConfig(false); + penConfig.parseConfigFromArgs(args); + Assert.assertEquals("xi_dir ", "not_here", penConfig.getXiDir()); + Assert.assertEquals("api_port ", 1234, penConfig.getApiPort()); + Assert.assertEquals("pow_threads ", 2, penConfig.getPowThreads()); + Assert.assertEquals("max_body_length ", 123456, penConfig.getMaxBodyLength()); + Assert.assertEquals("max_find_transactions ", 123456, penConfig.getMaxFindTransactions()); + Assert.assertEquals("max_get_transaction_strings ", 123456, penConfig.getMaxTransactionStrings()); + Assert.assertEquals("max_requests_list ", 123456, penConfig.getMaxRequestsList()); + Assert.assertEquals("db ", "mongodb" ,penConfig.getMainDb()); + Assert.assertEquals("db_cache_size ", 123456, penConfig.getDbCacheSize()); + Assert.assertEquals("db_log_path ", "somewhere_fancy.log" , penConfig.getDbLogPath()); + Assert.assertEquals("db_path ", "some_dir", penConfig.getDbPath()); + Assert.assertEquals("local_snapshots_base_path ", "./snapshot-go-here", penConfig.getLocalSnapshotsBasePath()); + Assert.assertEquals("local_snapshots_depth ", 123456, penConfig.getLocalSnapshotsDepth()); + Assert.assertEquals("local_snapshots_enabled ", false, penConfig.getLocalSnapshotsEnabled()); + Assert.assertEquals("local_snapshots_interval_synced ", 123456, penConfig.getLocalSnapshotsIntervalSynced()); + Assert.assertEquals("local_snapshots_interval_unsynced ", 123456, penConfig.getLocalSnapshotsIntervalUnsynced()); + Assert.assertEquals("local_snapshots_pruning_delay ", 123456, penConfig.getLocalSnapshotsPruningDelay()); + Assert.assertEquals("local_snapshots_pruning_enabled ", true, penConfig.getLocalSnapshotsPruningEnabled()); + Assert.assertEquals("neighbors ", Arrays.asList("udp://localhost:4101"), penConfig.getNeighbors()); + Assert.assertEquals("tcp_receiver_port ", 1234, penConfig.getTcpReceiverPort()); + Assert.assertEquals("udp_receiver_port ", 1234, penConfig.getUdpReceiverPort()); + Assert.assertEquals("max_peers ", 1234, penConfig.getMaxPeers()); + Assert.assertEquals("p_drop_cache ", 1.0, penConfig.getpDropCacheEntry(), 0d); + Assert.assertEquals("p_drop_transaction ", 1.0, penConfig.getpDropTransaction(), 0d); + Assert.assertEquals("p_propagate_request ", 1.0, penConfig.getpPropagateRequest(), 0d); + Assert.assertEquals("p_remove_request ", 1.0, penConfig.getpRemoveRequest(), 0d); + Assert.assertEquals("p_reply_random ", 1.0, penConfig.getpReplyRandomTip(), 0d); + Assert.assertEquals("p_select_milestone ", 1.0, penConfig.getpSelectMilestoneChild(), 0d); + Assert.assertEquals("p_send_milestone ",1.0, penConfig.getpSendMilestone(), 0d); + Assert.assertEquals("queue_size ", 1234, penConfig.getqSizeNode()); + Assert.assertEquals("dns_refresher ", false, penConfig.isDnsRefresherEnabled()); + Assert.assertEquals("dns_resolution ", false, penConfig.isDnsResolutionEnabled()); + Assert.assertEquals("send_limit ", 2, penConfig.getSendLimit()); + Assert.assertEquals("cache_size ", 1234, penConfig.getCacheSizeBytes()); + Assert.assertEquals("update_validator_delay ", 1234, penConfig.getUpdateValidatorDelay()); + Assert.assertEquals("validator ", true, penConfig.isValidator()); + Assert.assertEquals("validator_manager ", true, penConfig.getValidatorManagerEnabled()); + Assert.assertEquals("validator_path ", "./somewhere.txt", penConfig.getValidatorPath()); + Assert.assertEquals("start_validator ", 10, penConfig.getStartRoundDelay()); + Assert.assertEquals("round ", 60000, penConfig.getRoundDuration()); + Assert.assertEquals("round_pause ", 60000, penConfig.getRoundPause()); + Assert.assertEquals("max_analyzed_transactions ", 60000, penConfig.getBelowMaxDepthTransactionLimit()); + Assert.assertEquals("max_depth ", 10, penConfig.getMaxDepth()); + Assert.assertEquals("rescan ", true, penConfig.isRescanDb()); + Assert.assertEquals("revalidate ", true, penConfig.isRevalidate()); + Assert.assertEquals("remote_auth ", "pendulum:swings", penConfig.getRemoteAuth()); + Assert.assertEquals("alpha ", 1.0, penConfig.getAlpha(), 0d); + Assert.assertEquals("genesis ", 2345024001000L, penConfig.getGenesisTime()); + Assert.assertEquals("zmq_enable_ipc ", true, penConfig.isZmqEnableIpc()); + Assert.assertEquals("zmq_enable_tcp ", true, penConfig.isZmqEnableTcp()); + Assert.assertEquals("zmq_enabled ", true, penConfig.isZmqEnabled()); + Assert.assertEquals("zmq_ipc ", "ipc://pendulum", penConfig.getZmqIpc()); + Assert.assertEquals("zmq_port ", 1234, penConfig.getZmqPort()); + Assert.assertEquals("zmq_threads ", 10, penConfig.getZmqThreads()); + Assert.assertEquals("savelog_enabled ", true, penConfig.isSaveLogEnabled()); + Assert.assertEquals("savelog_path ", "./somewhere/", penConfig.getSaveLogBasePath()); + Assert.assertEquals("savelog_xml ", "/save.xml", penConfig.getSaveLogXMLFile()); + } + /* Test that iterates over common configs. It also attempts to check different types of types (double, boolean, string) */ @@ -55,32 +226,31 @@ public void argsParsingMainnetTest() { "-u", "4200", "-t", "5200", "-n", "udp://neighbor1 neighbor, tcp://neighbor2", - "--api-host", "1.1.1.1", - "--remote-limit-api", "call1 call2, call3", - "--max-find-transactions", "500", - "--max-requests-list", "1000", - "--max-get-transaction-strings", "4000", - "--max-body-length", "220", - "--remote-auth", "2.2.2.2", - "--p-remove-request", "0.23", - "--send-limit", "1000", - "--max-peers", "10", - "--dns-refresher", "false", - "--dns-resolution", "false", - "--XI-dir", "/XI", - "--db-path", "/db", - "--db-log-path", "/dblog", - "--zmq-enabled", "true", - //we ignore this on mainnet - "--mwm", "4", - "--testnet-coordinator", "TTTTTTTTT", - "--test-no-coo-validation", - //this should be ignored everywhere - "--fake-config" + "--api_host", "1.1.1.1", + "--ignored_api_endpoints", "call1 call2, call3", + "--max_find_transactions", "500", + "--max_requests_list", "1000", + "--max_get_transaction_strings", "4000", + "--max_body_length", "220", + "--remote_auth", "2.2.2.2", + "--p_remove_request", "0.23", + "--send_limit", "1000", + "--max_peers", "10", + "--dns_refresher", "false", + "--dns_resolution", "false", + "--xi_dir", "/XI", + "--db_path", "/db", + "--db_log_path", "/dblog", + "--zmq_enabled", "true", + "--mwm", "4", //we ignore this on mainnet + "--testnet_coordinator", "TTTTTTTTT", + "--test_no_coo_validation", + "--fake_config" //this should be ignored everywhere }; - PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); - Assert.assertThat("wrong config class created", pendulumConfig, CoreMatchers.instanceOf(MainnetConfig.class)); + PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); + Assert.assertThat("wrong config class created", + pendulumConfig, CoreMatchers.instanceOf(MainnetConfig.class)); pendulumConfig.parseConfigFromArgs(args); Assert.assertEquals("port value", 8089, pendulumConfig.getApiPort()); Assert.assertEquals("udp port", 4200, pendulumConfig.getUdpReceiverPort()); @@ -88,33 +258,26 @@ public void argsParsingMainnetTest() { Assert.assertEquals("neighbors", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); Assert.assertEquals("api host", "1.1.1.1", pendulumConfig.getApiHost()); - Assert.assertEquals("remote limit api", Arrays.asList("call1", "call2", "call3"), - pendulumConfig.getRemoteLimitApi()); + Assert.assertEquals("ignored api endpoints", Arrays.asList("call1", "call2", "call3"), + pendulumConfig.getIgnoredApiEndpoints()); Assert.assertEquals("max find transactions", 500, pendulumConfig.getMaxFindTransactions()); Assert.assertEquals("max requests list", 1000, pendulumConfig.getMaxRequestsList()); Assert.assertEquals("max get bytes", 4000, pendulumConfig.getMaxTransactionStrings()); Assert.assertEquals("max body length", 220, pendulumConfig.getMaxBodyLength()); - Assert.assertEquals("remote-auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); + Assert.assertEquals("remote_auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); Assert.assertEquals("p remove request", 0.23d, pendulumConfig.getpRemoveRequest(), 0d); Assert.assertEquals("send limit", 1000, pendulumConfig.getSendLimit()); Assert.assertEquals("max peers", 10, pendulumConfig.getMaxPeers()); Assert.assertEquals("dns refresher", false, pendulumConfig.isDnsRefresherEnabled()); Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); - Assert.assertEquals("XI-dir", "/XI", pendulumConfig.getXiDir()); + Assert.assertEquals("xi-dir", "/XI", pendulumConfig.getXiDir()); Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); - Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); + //Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); Assert.assertNotEquals("mwm", 4, pendulumConfig.getMwm()); Assert.assertNotEquals("coo", pendulumConfig.getValidatorManagerAddress(), "TTTTTTTTT"); - Assert.assertEquals("--testnet-no-milestone-sign-validation", false, pendulumConfig.isDontValidateTestnetMilestoneSig()); } - @Test - public void remoteFlagTest() { - String[] args = {"--remote"}; - PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(false); - pendulumConfig.parseConfigFromArgs(args); - Assert.assertEquals("The api interface should be open to the public", "0.0.0.0", pendulumConfig.getApiHost()); - } + @Test public void argsParsingTestnetTest() { @@ -123,28 +286,28 @@ public void argsParsingTestnetTest() { "-u", "4200", "-t", "5200", "-n", "udp://neighbor1 neighbor, tcp://neighbor2", - "--api-host", "1.1.1.1", - "--remote-limit-api", "call1 call2, call3", - "--max-find-transactions", "500", - "--max-requests-list", "1000", - "--max-get-transaction-strings", "4000", - "--max-body-length", "220", - "--remote-auth", "2.2.2.2", - "--p-remove-request", "0.23", - "--send-limit", "1000", - "--max-peers", "10", - "--dns-refresher", "false", - "--dns-resolution", "false", - "--XI-dir", "/XI", - "--db-path", "/db", - "--db-log-path", "/dblog", - "--zmq-enabled", "true", + "--api_host", "1.1.1.1", + "--ignored_api_endpoints", "call1 call2, call3", + "--max_find_transactions", "500", + "--max_requests_list", "1000", + "--max_get_transaction_strings", "4000", + "--max_body_length", "220", + "--remote_auth", "2.2.2.2", + "--p_remove_request", "0.23", + "--send_limit", "1000", + "--max_peers", "10", + "--dns_refresher", "false", + "--dns_resolution", "false", + "--xi_dir", "/XI", + "--db_path", "/db", + "--db_log_path", "/dblog", + "--zmq_enabled", "true", //we ignore this on mainnet "--mwm", "4", - "--testnet-coordinator", "TTTTTTTTT", - "--testnet-no-milestone-sign-validation", + "--testnet_coordinator", "TTTTTTTTT", + "--testnet_no_milestone_sign_validation", //this should be ignored everywhere - "--fake-config" + "--fake_config" }; PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(true); Assert.assertThat("wrong config class created", pendulumConfig, CoreMatchers.instanceOf(TestnetConfig.class)); @@ -156,34 +319,34 @@ public void argsParsingTestnetTest() { Assert.assertEquals("neighbors", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); Assert.assertEquals("api host", "1.1.1.1", pendulumConfig.getApiHost()); - Assert.assertEquals("remote limit api", Arrays.asList("call1", "call2", "call3"), - pendulumConfig.getRemoteLimitApi()); + Assert.assertEquals("ignored api endpoints", Arrays.asList("call1", "call2", "call3"), + pendulumConfig.getIgnoredApiEndpoints()); Assert.assertEquals("max find transactions", 500, pendulumConfig.getMaxFindTransactions()); Assert.assertEquals("max requests list", 1000, pendulumConfig.getMaxRequestsList()); Assert.assertEquals("max get tx strings", 4000, pendulumConfig.getMaxTransactionStrings()); Assert.assertEquals("max body length", 220, pendulumConfig.getMaxBodyLength()); - Assert.assertEquals("remote-auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); + Assert.assertEquals("remote_auth", "2.2.2.2", pendulumConfig.getRemoteAuth()); Assert.assertEquals("p remove request", 0.23d, pendulumConfig.getpRemoveRequest(), 0d); Assert.assertEquals("send limit", 1000, pendulumConfig.getSendLimit()); Assert.assertEquals("max peers", 10, pendulumConfig.getMaxPeers()); Assert.assertEquals("dns refresher", false, pendulumConfig.isDnsRefresherEnabled()); Assert.assertEquals("dns resolution", false, pendulumConfig.isDnsResolutionEnabled()); - Assert.assertEquals("XI-dir", "/XI", pendulumConfig.getXiDir()); + Assert.assertEquals("xi_dir", "/XI", pendulumConfig.getXiDir()); Assert.assertEquals("db path", "/db", pendulumConfig.getDbPath()); - Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); + //Assert.assertEquals("zmq enabled", true, pendulumConfig.isZmqEnabled()); Assert.assertEquals("mwm", 4, pendulumConfig.getMwm()); //Assert.assertEquals("coo", "TTTTTTTTT", pendulumConfig.getValidatorManagerAddress()); - Assert.assertEquals("--testnet-no-milestone-sign-validation", true, - pendulumConfig.isDontValidateTestnetMilestoneSig()); + Assert.assertEquals("validate testnet milestone signatures", true, + pendulumConfig.isValidateTestnetMilestoneSig()); } @Test public void iniParsingMainnetTest() throws Exception { String iniContent = new StringBuilder() - .append("[HLX]").append(System.lineSeparator()) + .append("[PENDULUM]").append(System.lineSeparator()) .append("PORT = 8088").append(System.lineSeparator()) .append("NEIGHBORS = udp://neighbor1 neighbor, tcp://neighbor2").append(System.lineSeparator()) - .append("ZMQ_ENABLED = true").append(System.lineSeparator()) + //.append("ZMQ_ENABLED = true").append(System.lineSeparator()) .append("P_REMOVE_REQUEST = 0.4").append(System.lineSeparator()) .append("MWM = 4").append(System.lineSeparator()) .append("FAKE").append(System.lineSeparator()) @@ -199,7 +362,7 @@ public void iniParsingMainnetTest() throws Exception { Assert.assertEquals("PORT", 8088, pendulumConfig.getApiPort()); Assert.assertEquals("NEIGHBORS", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); - Assert.assertEquals("ZMQ_ENABLED", true, pendulumConfig.isZmqEnabled()); + //Assert.assertEquals("ZMQ_ENABLED", true, pendulumConfig.isZmqEnabled()); Assert.assertEquals("P_REMOVE_REQUEST", 0.4d, pendulumConfig.getpRemoveRequest(), 0); Assert.assertNotEquals("MWM", 4, pendulumConfig.getMwm()); } @@ -207,16 +370,16 @@ public void iniParsingMainnetTest() throws Exception { @Test public void iniParsingTestnetTest() throws Exception { String iniContent = new StringBuilder() - .append("[HLX]").append(System.lineSeparator()) + .append("[PENDULUM]").append(System.lineSeparator()) .append("PORT = 8088").append(System.lineSeparator()) .append("NEIGHBORS = udp://neighbor1 neighbor, tcp://neighbor2").append(System.lineSeparator()) - .append("ZMQ_ENABLED = true").append(System.lineSeparator()) + //.append("ZMQ_ENABLED = true").append(System.lineSeparator()) .append("DNS_RESOLUTION_ENABLED = true").append(System.lineSeparator()) .append("P_REMOVE_REQUEST = 0.4").append(System.lineSeparator()) .append("MWM = 4").append(System.lineSeparator()) .append("NUMBER_OF_KEYS_IN_A_MILESTONE = 3").append(System.lineSeparator()) .append("DONT_VALIDATE_TESTNET_MILESTONE_SIG = true").append(System.lineSeparator()) - .append("TIPSELECTION_ALPHA = 1.1").append(System.lineSeparator()) + .append("ALPHA = 1.1").append(System.lineSeparator()) //doesn't do anything .append("REMOTE") .append("FAKE").append(System.lineSeparator()) @@ -232,7 +395,7 @@ public void iniParsingTestnetTest() throws Exception { Assert.assertEquals("PORT", 8088, pendulumConfig.getApiPort()); Assert.assertEquals("NEIGHBORS", Arrays.asList("udp://neighbor1", "neighbor", "tcp://neighbor2"), pendulumConfig.getNeighbors()); - Assert.assertEquals("ZMQ_ENABLED", true, pendulumConfig.isZmqEnabled()); + //Assert.assertEquals("ZMQ_ENABLED", true, pendulumConfig.isZmqEnabled()); Assert.assertEquals("DNS_RESOLUTION_ENABLED", true, pendulumConfig.isDnsResolutionEnabled()); //true by default Assert.assertEquals("DNS_REFRESHER_ENABLED", true, pendulumConfig.isDnsRefresherEnabled()); @@ -243,9 +406,9 @@ public void iniParsingTestnetTest() throws Exception { Assert.assertEquals("P_REMOVE_REQUEST", 0.4d, pendulumConfig.getpRemoveRequest(), 0); Assert.assertEquals("MWM", 4, pendulumConfig.getMwm()); Assert.assertEquals("NUMBER_OF_KEYS_IN_A_MILESTONE", 3, pendulumConfig.getNumberOfKeysInMilestone()); - Assert.assertEquals("TIPSELECTION_ALPHA", 1.1d, pendulumConfig.getAlpha(), 0); - Assert.assertEquals("DONT_VALIDATE_TESTNET_MILESTONE_SIG", - pendulumConfig.isDontValidateTestnetMilestoneSig(), true); + Assert.assertEquals("ALPHA", 1.1d, pendulumConfig.getAlpha(), 0); + Assert.assertEquals("VALIDATE_TESTNET_MILESTONE_SIG", + pendulumConfig.isValidateTestnetMilestoneSig(), true); //prove that REMOTE did nothing Assert.assertEquals("API_HOST", pendulumConfig.getApiHost(), "localhost"); } @@ -253,7 +416,7 @@ public void iniParsingTestnetTest() throws Exception { @Test(expected = IllegalArgumentException.class) public void testInvalidIni() throws IOException { String iniContent = new StringBuilder() - .append("[HLX]").append(System.lineSeparator()) + .append("[PENDULUM]").append(System.lineSeparator()) .append("REVALIDATE") .toString(); try (Writer writer = new FileWriter(configFile)) { @@ -262,26 +425,30 @@ public void testInvalidIni() throws IOException { ConfigFactory.createFromFile(configFile, false); } - //@Test + @Test public void backwardsIniCompatibilityTest() { Collection configNames = PendulumUtils.getAllSetters(TestnetConfig.class) .stream() .map(this::deriveNameFromSetter) .collect(Collectors.toList()); + Stream.of(LegacyDefaultConf.values()) .map(Enum::name) // make it explicit that we have removed some configs + // in some cases, we have renamed the config param (to e.g. fix double negative variable names) .filter(config -> !ArrayUtils.contains(new String[]{"CONFIG", "TESTNET", "DEBUG", - "MIN_RANDOM_WALKS", "MAX_RANDOM_WALKS"}, config)) + "MIN_RANDOM_WALKS", "MAX_RANDOM_WALKS", "DONT_VALIDATE_TESTNET_MILESTONE_SIG", "ZMQ_ENABLED", + "COORDINATOR", "REMOTE_LIMIT_API", "MWM", "TIPSELECTION_ALPHA"}, config)) .forEach(config -> - Assert.assertThat(configNames, IsCollectionContaining.hasItem(config))); + Assert.assertThat(configNames, IsCollectionContaining.hasItem(config)) + ); } @Test - public void dontValidateMilestoneSigDefaultValueTest() { + public void validateMilestoneSigDefaultValueTest() { PendulumConfig pendulumConfig = ConfigFactory.createPendulumConfig(true); - Assert.assertFalse("By default testnet should be validating milestones", - pendulumConfig.isDontValidateTestnetMilestoneSig()); + Assert.assertTrue("By default testnet should be validating milestones", + pendulumConfig.isValidateTestnetMilestoneSig()); } private String deriveNameFromSetter(Method setter) { diff --git a/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java b/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java index cc278d2b..61197b51 100644 --- a/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java +++ b/src/test/java/net/helix/pendulum/conf/ZMQConfigTest.java @@ -7,10 +7,11 @@ public class ZMQConfigTest { - @Test + // We have removed the zmq_enabled param completely. + //@Test public void isZmqEnabledLegacy() { String[] args = { - "--zmq-enabled", "true", + "--zmq_enabled", "true", }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -22,8 +23,8 @@ public void isZmqEnabledLegacy() { @Test public void isZmqEnabled() { String[] args = { - "--zmq-enable-tcp", "true", - "--zmq-enable-ipc", "true", + "--zmq_enable_tcp", "true", + "--zmq_enable_ipc", "true", }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -35,7 +36,7 @@ public void isZmqEnabled() { @Test public void isZmqEnableTcp() { String[] args = { - "--zmq-enable-tcp", "true" + "--zmq_enable_tcp", "true" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -46,7 +47,7 @@ public void isZmqEnableTcp() { @Test public void isZmqEnableIpc() { String[] args = { - "--zmq-enable-ipc", "true" + "--zmq_enable_ipc", "true" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -57,7 +58,7 @@ public void isZmqEnableIpc() { @Test public void getZmqPort() { String[] args = { - "--zmq-port", "8899" + "--zmq_port", "8899" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -68,7 +69,7 @@ public void getZmqPort() { @Test public void getZmqThreads() { String[] args = { - "--zmq-threads", "5" + "--zmq_threads", "5" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); @@ -78,7 +79,7 @@ public void getZmqThreads() { @Test public void getZmqIpc() { String[] args = { - "--zmq-ipc", "ipc://test" + "--zmq_ipc", "ipc://test" }; PendulumConfig config = ConfigFactory.createPendulumConfig(false); config.parseConfigFromArgs(args); diff --git a/src/test/java/net/helix/pendulum/service/APIIntegrationTest.java b/src/test/java/net/helix/pendulum/service/APIIntegrationTest.java index c88308e8..b0a7c210 100644 --- a/src/test/java/net/helix/pendulum/service/APIIntegrationTest.java +++ b/src/test/java/net/helix/pendulum/service/APIIntegrationTest.java @@ -117,11 +117,6 @@ public static void shutdown() { Xi.shutdown(); api.shutDown(); pendulum.shutdown(); - // shutdown spentAddressesProvider.rocksDBPersistenceProvider to avoid an exception in other unit-tests - if (pendulum.spentAddressesProvider != null - && pendulum.spentAddressesProvider.rocksDBPersistenceProvider != null) { - pendulum.spentAddressesProvider.rocksDBPersistenceProvider.shutdown(); - } } catch (final Exception e) { log.error("Exception occurred shutting down Pendulum node: ", e); fail("Exception occurred shutting down Pendulum node"); diff --git a/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java b/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java index ec358342..cc3a53e9 100644 --- a/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java +++ b/src/test/java/net/helix/pendulum/service/spentaddresses/impl/SpentAddressesProviderImplTest.java @@ -2,6 +2,7 @@ import net.helix.pendulum.TransactionTestUtils; import net.helix.pendulum.conf.ConfigFactory; +import net.helix.pendulum.conf.PendulumConfig; import net.helix.pendulum.model.Hash; import java.util.LinkedList; @@ -22,15 +23,14 @@ public class SpentAddressesProviderImplTest { @BeforeClass public static void setUp() throws Exception { + PendulumConfig config = ConfigFactory.createPendulumConfig(true); provider = new SpentAddressesProviderImpl(); - provider.init(ConfigFactory.createPendulumConfig(true)); + provider.init(config); } @AfterClass public static void shutdown() { - if (provider.rocksDBPersistenceProvider != null) { - provider.rocksDBPersistenceProvider.shutdown(); - } + provider.shutdown(); } @Test