Fix/apl 1836 cmdline precedence

apl-conf/pom.xml

@@ -16,6 +16,16 @@
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-api</artifactId>
     </dependency>
+    <dependency>
+          <groupId>org.projectlombok</groupId>
+          <artifactId>lombok</artifactId>
+          <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>com.apollocurrency</groupId>
+      <artifactId>apl-utils</artifactId>
+      <version>${project.version}</version>
+    </dependency>         
     <!-- unit tests -->
     <dependency>
       <groupId>org.junit.platform</groupId>

apl-conf/src/main/java/com/apollocurrency/aplwallet/apl/conf/ConfigRecord.java

@@ -0,0 +1,31 @@
+/*
+ * Copyright © 2018-2021 Apollo Foundation
+ */
+package com.apollocurrency.aplwallet.apl.conf;
+
+import com.apollocurrency.aplwallet.apl.util.Version;
+import lombok.Builder;
+import lombok.Builder.Default;
+
+/**
+ *
+ * @author Oleksiy Lukin alukin@gmail.com
+ */
+@Builder
+public class ConfigRecord {
+    public String name;
+    @Default
+    public String defaultValue="";
+    @Default
+    public String description="";
+    @Default
+    public Version sinceRelease = new Version("1.0.0");
+    @Default
+    public Version deprecatedSince = new Version("999.0.0");
+    @Default
+    public String cmdLineOpt="";
+    @Default
+    public String envVar="";
+    @Default
+    public boolean isRequired = false;
+}

apl-conf/src/main/java/com/apollocurrency/aplwallet/apl/conf/ConfigVerifier.java

@@ -0,0 +1,154 @@
+/*
+ * Copyright © 2018-2021 Apollo Foundation
+ */
+package com.apollocurrency.aplwallet.apl.conf;
+
+
+import com.apollocurrency.aplwallet.apl.util.Version;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Config Properies known by Apollo application
+ * @author Oleksiy Lukin alukin@gmail.com
+ */
+@Slf4j
+public class ConfigVerifier {
+    private final static String FIRST_RELEASE = "1.0.0";
+    private final static String DEX_RELEASE = "1.36.0";
+    private final static String SHARDING_RELEASE = "1.35.0";
+    private final static String MARIADB_RELEASE = "1.48.0";
+
+    private ConfigVerifier() {
+    }
+
+    /**
+     * property name mapped to parameters
+     */
+    @Getter
+    private Map<String,ConfigRecord> knownProps;
+
+    public boolean isSupported(String propName) {
+        return knownProps.containsKey(propName);
+    }
+
+    public ConfigRecord get(String propName){
+        ConfigRecord res =  knownProps.get(propName);
+        return res;
+    }
+
+    private ConfigRecord getOrAdd(String propName){
+        ConfigRecord res = knownProps.get(propName);
+        if (res == null) {
+            res = ConfigRecord.builder()
+                .name(propName)
+                .build();
+            knownProps.put(propName, res);
+        }
+        return res;
+    }
+
+    public List<ConfigRecord> listDeprecated(Version currVersion){
+        List<ConfigRecord> res = new ArrayList<>();
+        knownProps.values().stream().filter(pr -> (pr.deprecatedSince.lessThan(currVersion))).forEachOrdered(pr -> {
+            res.add(pr);
+        });
+        return res;
+    }
+
+    public List<ConfigRecord> listNewAfter(Version ver){
+        List<ConfigRecord> res = new ArrayList<>();
+        knownProps.values().stream().filter(pr -> (pr.sinceRelease.greaterThan(ver))).forEachOrdered(pr -> {
+            res.add(pr);
+        });
+        return res;
+    }
+
+ /**
+  * Dumps all known properties with comment lines
+  * containing all available information about config property
+  * @param pos output stream where to dump
+  */
+    public void dumpToProperties(OutputStream pos) throws IOException{
+        Writer w = new OutputStreamWriter(pos);
+        for(ConfigRecord cr: knownProps.values()){
+            w.write("# "+cr.description);
+            w.write("# Command line option: "+cr.cmdLineOpt+" Environment variable: "+cr.envVar);
+            w.write(cr.name+"="+cr.defaultValue);
+        }
+    }
+
+
+/**
+ * Parse properties file comparing to known properties and filling undefined with defaults
+ * @param config Properties file from resource or disk.
+ * Unknown properties will be logged with WARN level; missing required properties will
+ * be filled with default and warning will be logged
+ * @param currentVersion
+ * @return ready to use properties
+ */
+    public Properties parse(Properties config, Version currentVersion){
+        //go through supplied config and check it, warn on deprecated and on unknown
+        for(Object key: config.keySet()){
+            String name = (String)key;
+            String value = config.getProperty(name);
+            ConfigRecord cr = knownProps.get(name);
+            if (cr == null) {
+                log.warn("Unknown config property: '{}' with value: '{}'. It probably will be ignored", name, value);
+            } else if (currentVersion.greaterThan(cr.deprecatedSince)) {
+                log.warn("Config property: '{}' is deprecated since version '{}'", name, cr.deprecatedSince);
+            }
+
+        }
+        //define required properties, warn on undefined
+        for(ConfigRecord pr: knownProps.values()){
+            String val = config.getProperty(pr.name);
+            if (pr.isRequired && (val == null || val.isEmpty())) {
+                config.put(pr.name, pr.defaultValue);
+                log.warn("Required property: '{}' is not defined in config. Putting default value: '{}'", pr.name, pr.defaultValue);
+            }
+        }
+        return config;
+    }
+
+    /**
+     * All known properties must be initiated in this method; All known properties are defined
+     * in resource file conf/apl-blockchain.properties or in files in corresponding testnet config
+     * directories.TODO: keywords for deprecation, command line options and env variables
+     * @param resourcePath path to properties file in resources
+     * @return created properties fully initiated with default values
+     */
+    public static ConfigVerifier create( String resourcePath ) throws IOException {
+        ConfigVerifier kp = new ConfigVerifier();
+        DefaultConfig dc = null;
+        if (resourcePath == null || resourcePath.isEmpty()) {
+            resourcePath="conf/apl-blockchain.properties";
+        }
+        ClassLoader classloader = Thread.currentThread().getContextClassLoader();
+        log.debug("Trying to load resource by path = '{}' ...", resourcePath);
+        try (InputStream is = classloader.getResourceAsStream(resourcePath)) {
+             dc = DefaultConfig.fromStream(is);
+        }
+
+        kp.knownProps = dc.getKnownProperties();
+
+    //TODO: hardcode command line and env if so
+
+        kp.getOrAdd("apl.customDbDir").sinceRelease = new Version(MARIADB_RELEASE);
+
+        kp.getOrAdd("apl.customVaultKeystoreDir").sinceRelease = (new Version(DEX_RELEASE));
+
+        kp.getOrAdd("apl.customPidFile").sinceRelease = new Version(MARIADB_RELEASE);
+
+        return kp;
+    }
+}

apl-conf/src/main/java/com/apollocurrency/aplwallet/apl/conf/DefaultConfig.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright © 2018-2021 Apollo Foundation
+ */
+package com.apollocurrency.aplwallet.apl.conf;
+
+import com.apollocurrency.aplwallet.apl.util.Version;
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+/**
+ * Parser of default config from resources.
+ * Default config must contain all properties known to the program with default values,
+ * description and keywords signaling corresponding command line options and
+ * environment variables. Keywords are prefixed
+ * @author Oleksiy Lukin alukin@gmail.com
+ */
+@Slf4j
+public class DefaultConfig {
+
+    public static final String KW_DEPRECATED = "DEPRECATED";
+    public static final String KW_SINCE = "SINCE";
+    public static final String KW_COMMAND_LINE = "CMDLINE";
+    public static final String KW_ENV_VAR = "ENVVAR";
+
+    @Getter
+    final Map<String, ConfigRecord> knownProperties = new HashMap<>();
+
+    static ConfigRecord createCr(String key, String value, List<String> comments) {
+        Objects.requireNonNull(key, "key is NULL");
+        Objects.requireNonNull(value, "value is NULL");
+        Objects.requireNonNull(comments, "comments list is NULL");
+        String comment = comments.stream().collect(Collectors.joining(" "));
+        ConfigRecord cr = ConfigRecord.builder()
+                .name(key)
+                .defaultValue(value)
+                .description(comment)
+                .build();
+        for (String line : comments) {
+            if (line.startsWith("!")) {
+                int end = line.indexOf(" ")>3 ? line.indexOf(" "): line.length()-1;
+                String cmd = line.substring(1,end).trim();
+                String val = line.substring(end).trim();
+                int val_end = val.indexOf(" ")>0 ? val.indexOf(" "): val.length();
+                val = val.substring(0,val_end);
+                switch(cmd) {
+                    case KW_DEPRECATED:
+                        cr.deprecatedSince = new Version(val);
+                        break;
+                    case KW_SINCE:
+                        cr.sinceRelease = new Version(val);
+                        break;
+                    case KW_COMMAND_LINE:
+                        cr.cmdLineOpt=val;
+                        break;
+                    case KW_ENV_VAR:
+                        cr.envVar = val;
+                        break;
+                    default:
+                        log.warn("Unknown keyword in the property comments. Property: {} Keyword: {}", key, cmd);
+                }
+            }
+        }
+
+        return cr;
+    }
+
+    public static DefaultConfig fromStream(InputStream is) throws IOException {
+        Objects.requireNonNull(is, "Input Stream is NULL");
+        DefaultConfig conf = new DefaultConfig();
+        List<String> commentBuf = new ArrayList<>();
+        try ( BufferedReader br = new BufferedReader(new InputStreamReader(is))) {
+            String line;
+            while ((line = br.readLine()) != null) {
+                line = line.trim();
+                if (line.startsWith("#")) {
+                    commentBuf.add(line.substring(1));
+                } else if (line.isBlank()) {
+                    commentBuf.clear();
+                } else {
+                    String[] parts = line.split("=");
+                    String key = parts[0];
+                    String value = "";
+                    if (parts.length > 1) {
+                        value = parts[1];
+                    }
+                    int idx = value.indexOf("#");
+                    if (idx >= 0) {
+                        value = value.substring(idx);
+                    }
+
+                    ConfigRecord cr = createCr(key, value, commentBuf);
+                    conf.knownProperties.put(key, cr);
+                }
+            }
+        }
+        return conf;
+    }
+}

apl-conf/src/main/resources/conf/apl-blockchain.properties

@@ -23,10 +23,10 @@
 
 # Set the socks proxy host.
 #socksProxyHost=localhost
-
+socksProxyHost=
 # Set the socks proxy port.
 #socksProxyPort=9050
-
+socksProxyPort=
 
 
 #### PEER NETWORKING ####
@@ -66,6 +66,7 @@ apl.myAddress=
 apl.myPlatform=
 
 # My hallmark, if available.
+#!DEPRECATED 1.25.1
 apl.myHallmark=
 
 # Maximum number of inbound connections.
@@ -447,9 +448,13 @@ apl.batchCommitSize=100
 apl.sharding.backupDb=false
 
 # if true, app will not import last shard, instead will start from genesis block
+#!SINCE 1.35.0
+#!CMDLINE --no-shard-import
 apl.noshardimport=false
 
 # if true, app will not divide database for shards, will store large database files instead
+#!SINCE 1.35.0
+#!CMDLINE --no-shard-import
 apl.noshardcreate=false
 
 # Minimal delay (in seconds) after reached shard height to wait before starting sharding process.