feat: app archive scanner cli

gradle.properties

@@ -4,6 +4,7 @@ publishVersion = 1.1.0
 kotlinxCollectionsVersion = 0.3.5
 kotlinxCoroutinesVersion = 1.5.2-native-mt
 kotlinxSerializationVersion = 1.3.1
+kotlinxCliVersion = 0.3.6
 atomicfuVersion = 0.23.0
 grgitVersion = 4.1.1
 ktorVersion = 1.6.8
@@ -20,7 +21,7 @@ apacheHttpClientVersion = 5.2.3
 aesyDatasizeVersion = 1.0.0
 bytebuddyVersion = 1.14.11
 
-sharedLibsRef = feature/max-retries-EPMDJ-10975
+sharedLibsRef = main
 sharedLibsLocalPath = lib-jvm-shared
 nativeAgentLibName = drill-agent
 nativeAgentHookEnabled = false

java-agent/build.gradle.kts

@@ -18,6 +18,7 @@ plugins {
     kotlin("plugin.serialization")
     id("com.github.johnrengelman.shadow")
     id("com.github.hierynomus.license")
+    application
 }
 
 group = "com.epam.drill.agent"
@@ -35,6 +36,7 @@ val nativeAgentLibName: String by parent!!.extra
 val nativeAgentHookEnabled: String by parent!!.extra
 val macosLd64: String by parent!!.extra
 val bytebuddyVersion: String by parent!!.extra
+val kotlinxCliVersion: String by parent!!.extra
 
 repositories {
     mavenCentral()
@@ -95,6 +97,7 @@ kotlin {
                 implementation(project(":common"))
                 implementation(project(":agent-config"))
                 implementation(project(":agent-instrumentation"))
+                implementation(project(":konform"))
             }
         }
         val commonTest by getting {
@@ -109,9 +112,15 @@ kotlin {
                 implementation("com.alibaba:transmittable-thread-local:$transmittableThreadLocalVersion")
                 implementation("io.aesy:datasize:$aesyDatasizeVersion")
                 implementation("net.bytebuddy:byte-buddy:$bytebuddyVersion")
+                implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:${kotlinxCoroutinesVersion}")
+                implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:${kotlinxSerializationVersion}")
+                implementation("org.jetbrains.kotlinx:kotlinx-serialization-protobuf:${kotlinxSerializationVersion}")
+                implementation("org.jetbrains.kotlinx:kotlinx-cli:${kotlinxCliVersion}")
+                implementation(project(":common"))
                 implementation(project(":agent-transport"))
                 implementation(project(":agent-instrumentation"))
-                runtimeOnly(project(":test2code"))
+                implementation(project(":test2code"))
+                implementation(project(":test2code-common"))
             }
         }
         val configureNativeDependencies: KotlinSourceSet.() -> Unit = {
@@ -178,7 +187,13 @@ kotlin {
             "net.bytebuddy",
             "mu",
         )
+        project.setProperty("mainClassName", "com.epam.drill.agent.AppArchiveScannerCliKt")
         val runtimeJar by registering(ShadowJar::class) {
+            manifest {
+                attributes(mapOf(
+                    "Main-Class" to "com.epam.drill.agent.AppArchiveScannerCliKt"
+                ))
+            }
             mergeServiceFiles()
             isZip64 = true
             archiveFileName.set("drill-runtime.jar")
@@ -221,6 +236,10 @@ distributions {
     }
 }
 
+application {
+    mainClass.set("com.epam.drill.agent.AppArchiveScannerCliKt")
+}
+
 @Suppress("UNUSED_VARIABLE")
 license {
     headerURI = URI("https://raw.githubusercontent.com/Drill4J/drill4j/develop/COPYRIGHT")

java-agent/src/commonMain/kotlin/com/epam/drill/agent/configuration/ParameterDefinitions.kt

@@ -16,25 +16,36 @@
 package com.epam.drill.agent.configuration
 
 import com.epam.drill.agent.common.configuration.AgentParameterDefinition
+import com.epam.drill.agent.common.configuration.AgentParameterDefinitionCollection
+import com.epam.drill.agent.common.configuration.NullableAgentParameterDefinition
 
-object ParameterDefinitions {
+object ParameterDefinitions: AgentParameterDefinitionCollection() {
 
-    val API_URL = AgentParameterDefinition.forString(name = "apiUrl", parser = { if (!it.endsWith("/")) "$it/" else it } )
-    val API_KEY = AgentParameterDefinition.forString(name = "apiKey")
-    val MESSAGE_QUEUE_LIMIT = AgentParameterDefinition.forString(name = "messageQueueLimit", defaultValue = "512Mb")
-    val MESSAGE_MAX_RETRIES = AgentParameterDefinition.forInt(name = "messageMaxRetries", defaultValue = Int.MAX_VALUE)
-    val SSL_TRUSTSTORE = AgentParameterDefinition.forString(name = "sslTruststore")
-    val SSL_TRUSTSTORE_PASSWORD = AgentParameterDefinition.forString(name = "sslTruststorePassword")
-    val LOG_LEVEL = AgentParameterDefinition.forString(name = "logLevel", defaultValue = "INFO")
-    val LOG_FILE = AgentParameterDefinition.forString(name = "logFile")
-    val LOG_LIMIT = AgentParameterDefinition.forInt(name = "logLimit", defaultValue = 512)
-    val IS_WEB_APP = AgentParameterDefinition.forBoolean(name = "isWebApp")
-    val IS_KAFKA = AgentParameterDefinition.forBoolean(name = "isKafka")
-    val IS_CADENCE = AgentParameterDefinition.forBoolean(name = "isCadence")
-    val IS_TLS_APP = AgentParameterDefinition.forBoolean(name = "isTlsApp")
-    val IS_ASYNC_APP = AgentParameterDefinition.forBoolean(name = "isAsyncApp")
-    val IS_COMPATIBILITY_TESTS = AgentParameterDefinition.forBoolean(name = "isCompatibilityTests", defaultValue = false)
-    val USE_PROTOBUF_SERIALIZER = AgentParameterDefinition.forBoolean(name = "useProtobufSerializer", defaultValue = true)
-    val USE_GZIP_COMPRESSION = AgentParameterDefinition.forBoolean(name = "useGzipCompression", defaultValue = true)
-    val IS_WS_MESSAGES = AgentParameterDefinition.forBoolean(name = "isWsMsg", defaultValue = true)
+    val API_URL = AgentParameterDefinition.forString(
+        name = "apiUrl",
+        description = "URL to Drill4J Backend /api endpoint. Example: http://localhost:8090/api",
+        parser = { if (!it.endsWith("/")) "$it/" else it  },
+        validator = { validTransportUrl() }).register()
+    val API_KEY = NullableAgentParameterDefinition.forString(
+        name = "apiKey",
+        description = "Drill4J API key. It is recommended to set it with DRILL_API_KEY env variable, rather than using command line argument"
+    ).register()
+    val MESSAGE_QUEUE_LIMIT = AgentParameterDefinition.forString(name = "messageQueueLimit", defaultValue = "512Mb").register()
+    val MESSAGE_MAX_RETRIES = AgentParameterDefinition.forInt(name = "messageMaxRetries", defaultValue = Int.MAX_VALUE).register()
+    val SSL_TRUSTSTORE = NullableAgentParameterDefinition.forString(name = "sslTruststore").register()
+    val SSL_TRUSTSTORE_PASSWORD = NullableAgentParameterDefinition.forString(name = "sslTruststorePassword").register()
+    val LOG_LEVEL = AgentParameterDefinition.forString(name = "logLevel", defaultValue = "INFO").register()
+    val LOG_FILE = NullableAgentParameterDefinition.forString(name = "logFile").register()
+    val LOG_LIMIT = AgentParameterDefinition.forInt(name = "logLimit", defaultValue = 512).register()
+    val IS_WEB_APP = AgentParameterDefinition.forBoolean(name = "isWebApp", defaultValue = false).register()
+    val IS_KAFKA = AgentParameterDefinition.forBoolean(name = "isKafka", defaultValue = false).register()
+    val IS_CADENCE = AgentParameterDefinition.forBoolean(name = "isCadence", defaultValue = false).register()
+    val IS_TLS_APP = AgentParameterDefinition.forBoolean(name = "isTlsApp", defaultValue = false).register()
+    val IS_ASYNC_APP = AgentParameterDefinition.forBoolean(name = "isAsyncApp", defaultValue = false).register()
+    val IS_COMPATIBILITY_TESTS =
+        AgentParameterDefinition.forBoolean(name = "isCompatibilityTests", defaultValue = false).register()
+    val USE_PROTOBUF_SERIALIZER =
+        AgentParameterDefinition.forBoolean(name = "useProtobufSerializer", defaultValue = true).register()
+    val USE_GZIP_COMPRESSION = AgentParameterDefinition.forBoolean(name = "useGzipCompression", defaultValue = true).register()
+    val IS_WS_MESSAGES = AgentParameterDefinition.forBoolean(name = "isWsMsg", defaultValue = true).register()
 }

java-agent/src/jvmMain/kotlin/com/epam/drill/agent/AppArchiveScannerCli.kt

@@ -0,0 +1,118 @@
+/**
+ * Copyright 2020 - 2022 EPAM Systems
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.epam.drill.agent
+
+import com.epam.drill.agent.common.configuration.AgentConfiguration
+import com.epam.drill.agent.common.configuration.AgentParameterDefinitionCollection
+import com.epam.drill.agent.common.configuration.BaseAgentParameterDefinition
+import com.epam.drill.agent.common.transport.AgentMessage
+import com.epam.drill.agent.common.transport.AgentMessageDestination
+import com.epam.drill.agent.configuration.AgentParameterValidationError
+import com.epam.drill.agent.configuration.AgentParametersValidator
+import com.epam.drill.agent.configuration.DefaultAgentConfiguration
+import com.epam.drill.agent.configuration.DefaultParameterDefinitions
+import com.epam.drill.agent.configuration.ParameterDefinitions
+import com.epam.drill.agent.logging.LoggingConfiguration
+import com.epam.drill.agent.test2code.Test2Code
+import com.epam.drill.agent.test2code.configuration.Test2CodeParameterDefinitions
+import com.epam.drill.agent.transport.HttpAgentMessageDestinationMapper
+import com.epam.drill.agent.transport.JsonAgentMessageSerializer
+import com.epam.drill.agent.transport.SimpleAgentMessageSender
+import com.epam.drill.agent.transport.http.HttpAgentMessageTransport
+import java.io.File
+import kotlin.system.exitProcess
+import kotlin.takeIf
+
+fun main(args: Array<String>) {
+    LoggingConfiguration.readDefaultConfiguration()
+
+    val argsMap: Map<String, String> = args
+        .filter { it.startsWith("--") && it.contains("=") }
+        .associate {
+            val (key, value) = it.removePrefix("--").split("=", limit = 2)
+            key to value
+        }.filter { it.value.isNotEmpty() }
+    val envMap = System.getenv()
+        .filterKeys { it.startsWith("DRILL_") }
+        .filterValues { !it.isNullOrEmpty() }
+        .mapKeys { toParameterName(it) }
+    val configuration = DefaultAgentConfiguration(envMap + argsMap)
+    val definitions = collectAgentParameterDefinitions(
+        DefaultParameterDefinitions,
+        ParameterDefinitions,
+        Test2CodeParameterDefinitions
+    )
+    val validator = AgentParametersValidator(configuration.parameters)
+    try {
+        validator.validate(*definitions.toTypedArray()) }
+    catch (e: AgentParameterValidationError) {
+        println(e.message)
+        exitProcess(1)
+    }
+
+    val commitSha = configuration.parameters[DefaultParameterDefinitions.COMMIT_SHA]
+    val buildVersion = configuration.parameters[DefaultParameterDefinitions.BUILD_VERSION]
+    if (commitSha == null && buildVersion == null) {
+        throw IllegalArgumentException("Either commitSha or buildVersion must be provided")
+    }
+
+    configuration.parameters[ParameterDefinitions.LOG_LEVEL]
+        .let(LoggingConfiguration::setLoggingLevels)
+    configuration.parameters[ParameterDefinitions.LOG_FILE]
+        ?.let(LoggingConfiguration::setLoggingFilename)
+    configuration.parameters[ParameterDefinitions.LOG_LIMIT].let(LoggingConfiguration::setLogMessageLimit)
+
+    val transport = HttpAgentMessageTransport(
+        serverAddress = configuration.parameters[ParameterDefinitions.API_URL],
+        apiKey = configuration.parameters[ParameterDefinitions.API_KEY] ?: "",
+        sslTruststore = configuration.parameters[ParameterDefinitions.SSL_TRUSTSTORE]
+            ?.let { resolvePath(configuration, it) } ?: "",
+        sslTruststorePass = configuration.parameters[ParameterDefinitions.SSL_TRUSTSTORE_PASSWORD] ?: "",
+        gzipCompression = configuration.parameters[ParameterDefinitions.USE_GZIP_COMPRESSION],
+    )
+    val serializer = JsonAgentMessageSerializer<AgentMessage>()
+    val mapper = HttpAgentMessageDestinationMapper()
+    val sender = SimpleAgentMessageSender(transport, serializer, mapper)
+    val test2Code = Test2Code(
+        id = "test2Code",
+        agentContext = RequestAgentContext,
+        sender = sender,
+        configuration = configuration
+    )
+    sender.send(AgentMessageDestination("PUT", "builds"), configuration.agentMetadata)
+    test2Code.scanAndSendMetadataClasses()
+}
+
+//TODO: duplicate with JvmModuleMessageSender
+private fun resolvePath(configuration: AgentConfiguration, path: String) = File(path).run {
+    val installationDir = File(configuration.parameters[DefaultParameterDefinitions.INSTALLATION_DIR] ?: "")
+    val resolved = this.takeIf(File::exists)
+        ?: this.takeUnless(File::isAbsolute)?.let(installationDir::resolve)
+    resolved?.takeUnless(File::isDirectory)?.absolutePath ?: path
+}
+
+//TODO: duplicate from ValidatedParametersProvider
+internal fun toParameterName(entry: Map.Entry<String, String>) = entry.key
+    .removePrefix("DRILL_")
+    .lowercase()
+    .split("_")
+    .joinToString("") { it.replaceFirstChar(Char::uppercase) }
+    .replaceFirstChar(Char::lowercase)
+
+private fun collectAgentParameterDefinitions(vararg collections: AgentParameterDefinitionCollection): List<BaseAgentParameterDefinition<*>> {
+    return collections.flatMap { it.getAll() }
+
+}

java-agent/src/jvmMain/kotlin/com/epam/drill/agent/transport/JvmModuleMessageSender.kt

@@ -48,10 +48,10 @@ actual object JvmModuleMessageSender : AgentMessageSender<AgentMessage> {
     private fun messageSender(): QueuedAgentMessageSender<AgentMessage> {
         val transport = HttpAgentMessageTransport(
             serverAddress = Configuration.parameters[ParameterDefinitions.API_URL],
-            apiKey = Configuration.parameters[ParameterDefinitions.API_KEY],
-            sslTruststore = Configuration.parameters[ParameterDefinitions.SSL_TRUSTSTORE].takeIf(String::isNotEmpty)
+            apiKey = Configuration.parameters[ParameterDefinitions.API_KEY] ?: "",
+            sslTruststore = Configuration.parameters[ParameterDefinitions.SSL_TRUSTSTORE]
                 ?.let(::resolvePath) ?: "",
-            sslTruststorePass = Configuration.parameters[ParameterDefinitions.SSL_TRUSTSTORE_PASSWORD],
+            sslTruststorePass = Configuration.parameters[ParameterDefinitions.SSL_TRUSTSTORE_PASSWORD] ?: "",
             gzipCompression = Configuration.parameters[ParameterDefinitions.USE_GZIP_COMPRESSION],
         )
         val serializer = takeIf { Configuration.parameters[ParameterDefinitions.USE_PROTOBUF_SERIALIZER] }?.let {

java-agent/src/nativeMain/kotlin/com/epam/drill/agent/configuration/AgentLoggingConfiguration.kt

@@ -31,7 +31,7 @@ object AgentLoggingConfiguration {
 
     fun updateNativeLoggingConfiguration() {
         val logLevel = Configuration.parameters[ParameterDefinitions.LOG_LEVEL]
-        val logFile = Configuration.parameters[ParameterDefinitions.LOG_FILE].takeIf(String::isNotEmpty)
+        val logFile = Configuration.parameters[ParameterDefinitions.LOG_FILE]
         val logLimit = Configuration.parameters[ParameterDefinitions.LOG_LIMIT]
 
         LoggingConfiguration.setLoggingLevels(logLevel)
@@ -50,7 +50,7 @@ object AgentLoggingConfiguration {
     @OptIn(ExperimentalForeignApi::class)
     fun updateJvmLoggingConfiguration() {
         val logLevel = Configuration.parameters[ParameterDefinitions.LOG_LEVEL]
-        val logFile = Configuration.parameters[ParameterDefinitions.LOG_FILE].takeIf(String::isNotEmpty)
+        val logFile = Configuration.parameters[ParameterDefinitions.LOG_FILE]
         val logLimit = Configuration.parameters[ParameterDefinitions.LOG_LIMIT]
 
         callObjectVoidMethodWithString(LoggingConfiguration::class, "setLoggingLevels", logLevel)

test2code/build.gradle.kts

@@ -41,6 +41,7 @@ dependencies {
     implementation("io.github.microutils:kotlin-logging-jvm:$microutilsLoggingVersion")
 
     implementation(project(":common"))
+    implementation(project(":agent-config"))
     implementation(project(":test2code-common"))
     implementation(project(":test2code-jacoco"))
     implementation(project(":konform"))