Introduce Tool.USVM in ContestEstimator, implement runUsvmGeneration (#2666)

* Add `Tool.USVM` in `ContestEstimator`, implement `runUsvmGeneration`

* Remove redundant TODO

* Bump UtBot dependencies version to match USVM dependencies

* Use `UTestExecutionResult.trace` instead of `JcCoverage`

Author: IlyaMuravjov

gradle.properties

@@ -41,12 +41,15 @@ junit4PlatformVersion=1.9.0
 # NOTE: Mockito versions 5+ are not compatible with Java 8: https://www.davidvlijmincx.com/posts/upgrade-to-mockito-5
 mockitoVersion=4.11.0
 mockitoInlineVersion=4.11.0
-ksmtVersion=0.4.3
+# TODO usvm-sbft-merge: UtBot engine uses ksmt 0.4.3, while USVM uses ksmt 0.5.7
+ksmtVersion=0.5.7
 sootVersion=4.4.0-FORK-2
 kotlinVersion=1.8.0
 log4j2Version=2.13.3
-coroutinesVersion=1.6.3
-collectionsVersion=0.3.4
+# TODO usvm-sbft-merge: UtBot uses kotlinx.coroutines 1.6.3, while USVM uses kotlinx.coroutines 1.6.4
+coroutinesVersion=1.6.4
+# TODO usvm-sbft-merge: UtBot uses kotlinx.collections 0.3.4, while USVM uses kotlinx.collections 0.3.5
+collectionsVersion=0.3.5
 # after updating plugin version you should manually bump corresponding versions in plugin
 # as they cannot be set from properties
 # utbot-intellij/build.gradle.kts
@@ -56,7 +59,8 @@ intellijPluginVersion=1.13.1
 # every time you bump rd version:
 # 1. regenerate all models
 # 2. check if rider plugin works
-rdVersion=2023.1.2
+# TODO usvm-sbft-merge: UtBot engine uses RD 2023.1.2, while USVM uses RD 2023.2.0
+rdVersion=2023.2.0
 # to enable - add -PincludeRiderInBuild=true in build CLI
 includeRiderInBuild=false
 jacocoVersion=0.8.8
@@ -77,6 +81,7 @@ testNgVersion=7.6.0
 kamlVersion=0.51.0
 jacksonVersion=2.12.3
 kotlinxSerializationVersion=1.5.0
+# TODO usvm-sbft: USVM uses slf4j 1.6.1
 slf4jVersion=1.7.36
 eclipseAetherVersion=1.1.0
 mavenWagonVersion=3.5.1
@@ -96,6 +101,7 @@ commonsLoggingVersion=1.2
 commonsIOVersion=2.11.0
 javaxVersion=2.2
 jakartaVersion=3.1.0
+jacoDbVersion=1.3.0
 
 # use latest Java 8 compaitable Spring and Spring Boot versions
 springVersion=5.3.28

settings.gradle.kts

@@ -91,4 +91,8 @@ if (projectType == ultimateEdition) {
 }
 
 include("utbot-light")
+
 include("utbot-intellij-main")
+
+// TODO usvm-sbft-merge: add if here if we want merge contest it into main
+includeBuild("../usvm")

utbot-framework-api/src/main/kotlin/org/utbot/framework/plugin/api/Api.kt

@@ -148,6 +148,10 @@ abstract class UtExecution(
     val executableToCall get() = stateBefore.executableToCall
 }
 
+interface UtExecutionWithInstrumentation {
+    val instrumentation: List<UtInstrumentation>
+}
+
 /**
  * Symbolic execution.
  *
@@ -163,15 +167,15 @@ class UtSymbolicExecution(
     stateBefore: EnvironmentModels,
     stateAfter: EnvironmentModels,
     result: UtExecutionResult,
-    val instrumentation: List<UtInstrumentation>,
+    override val instrumentation: List<UtInstrumentation>,
     val path: MutableList<Step>,
     val fullPath: List<Step>,
     coverage: Coverage? = null,
     summary: List<DocStatement>? = null,
     testMethodName: String? = null,
     displayName: String? = null,
     /** Convenient view of the full symbolic path */ val symbolicSteps: List<SymbolicStep> = listOf(),
-) : UtExecution(stateBefore, stateAfter, result, coverage, summary, testMethodName, displayName) {
+) : UtExecution(stateBefore, stateAfter, result, coverage, summary, testMethodName, displayName), UtExecutionWithInstrumentation {
     /**
      * By design the 'before' and 'after' states contain info about the same fields.
      * It means that it is not possible for a field to be present at 'before' and to be absent at 'after'.

utbot-framework-test/build.gradle

@@ -42,8 +42,9 @@ dependencies {
     testImplementation group: 'org.mockito', name: 'mockito-inline', version: mockitoInlineVersion
     testImplementation group: 'org.apache.logging.log4j', name: 'log4j-core', version: log4j2Version
 
-    implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-core', version: ksmtVersion
-    implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-z3', version: ksmtVersion
+    // TODO sbft-usvm-merge: UtBot engine expects `com.github.UnitTestBot.ksmt` here
+    implementation group: 'io.ksmt', name: 'ksmt-core', version: ksmtVersion
+    implementation group: 'io.ksmt', name: 'ksmt-z3', version: ksmtVersion
 }
 
 // This is required to avoid conflict between SpringBoot standard logger and the logger of our project.

utbot-framework/build.gradle

@@ -42,8 +42,9 @@ dependencies {
     implementation group: 'org.junit.jupiter', name: 'junit-jupiter-params', version: '5.8.1'
     implementation group: 'org.junit.jupiter', name: 'junit-jupiter-engine', version: '5.8.1'
 
-    implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-core', version: ksmtVersion
-    implementation group: 'com.github.UnitTestBot.ksmt', name: 'ksmt-z3', version: ksmtVersion
+    // TODO sbft-usvm-merge: UtBot engine expects `com.github.UnitTestBot.ksmt` here
+    implementation group: 'io.ksmt', name: 'ksmt-core', version: ksmtVersion
+    implementation group: 'io.ksmt', name: 'ksmt-z3', version: ksmtVersion
 
     fetchInstrumentationJar project(path: ':utbot-instrumentation', configuration: 'instrumentationArchive')
 }

utbot-framework/src/main/kotlin/org/utbot/framework/codegen/tree/CgMethodConstructor.kt

@@ -99,6 +99,7 @@ import org.utbot.framework.plugin.api.UtExecution
 import org.utbot.framework.plugin.api.UtExecutionFailure
 import org.utbot.framework.plugin.api.UtExecutionResult
 import org.utbot.framework.plugin.api.UtExecutionSuccess
+import org.utbot.framework.plugin.api.UtExecutionWithInstrumentation
 import org.utbot.framework.plugin.api.UtExplicitlyThrownException
 import org.utbot.framework.plugin.api.UtLambdaModel
 import org.utbot.framework.plugin.api.UtModel
@@ -201,8 +202,7 @@ open class CgMethodConstructor(val context: CgContext) : CgContextOwner by conte
 
     protected fun setupInstrumentation() {
         val instrumentation = when (val execution = currentExecution) {
-            is UtSymbolicExecution -> execution.instrumentation
-            is UtFuzzedExecution -> execution.instrumentation
+            is UtExecutionWithInstrumentation -> execution.instrumentation
             else -> return
         }
         if (instrumentation.isEmpty()) return

utbot-java-fuzzing/src/main/kotlin/org/utbot/fuzzer/UtFuzzedExecution.kt

@@ -22,8 +22,8 @@ class UtFuzzedExecution(
     displayName: String? = null,
     val fuzzingValues:  List<FuzzedValue>? = null,
     val fuzzedMethodDescription:  FuzzedMethodDescription? = null,
-    val instrumentation: List<UtInstrumentation> = emptyList(),
-) : UtExecution(stateBefore, stateAfter, result, coverage, summary, testMethodName, displayName) {
+    override val instrumentation: List<UtInstrumentation> = emptyList(),
+) : UtExecution(stateBefore, stateAfter, result, coverage, summary, testMethodName, displayName), UtExecutionWithInstrumentation {
     /**
      * By design the 'before' and 'after' states contain info about the same fields.
      * It means that it is not possible for a field to be present at 'before' and to be absent at 'after'.

utbot-junit-contest/build.gradle

@@ -139,6 +139,19 @@ dependencies {
     implementation group: 'org.mockito', name: 'mockito-core', version: mockitoVersion
     implementation group: 'org.mockito', name: 'mockito-inline', version: mockitoInlineVersion
     implementation 'junit:junit:4.13.2'
+
+    implementation('org.usvm:usvm-core')
+    implementation('org.usvm:usvm-jvm')
+    implementation('org.usvm:usvm-jvm-instrumentation')
+
+    implementation group: "org.jacodb", name: "jacodb-core", version: jacoDbVersion
+    implementation group: "org.jacodb", name: "jacodb-analysis", version: jacoDbVersion
+    implementation group: "org.jacodb", name: "jacodb-approximations", version: jacoDbVersion
+
+    // TODO uvms-sbft-hack: UtBot has `fastutil:8.3.0` on the classpath that overrides classes from
+    //  `fastutil-core:8.5.11` that USVM adds. Solution: bump `fastutil` version to `8.5.11`
+    runtimeOnly("it.unimi.dsi:fastutil:8.5.11")
+
     testImplementation fileTree(dir: 'src/main/resources/projects/', include: '*/*.jar')
     testImplementation files('src/main/resources/evosuite/evosuite-1.2.0.jar')
     testImplementation files('src/main/resources/evosuite/evosuite-standalone-runtime-1.2.0.jar')

utbot-junit-contest/src/main/kotlin/org/utbot/contest/Contest.kt

@@ -417,7 +417,7 @@ fun runGeneration(
     statsForClass
 }
 
-private fun prepareClass(javaClazz: Class<*>, methodNameFilter: String?): List<ExecutableId> {
+fun prepareClass(javaClazz: Class<*>, methodNameFilter: String?): List<ExecutableId> {
     //1. all methods from cut
     val methods = javaClazz.declaredMethods
         .filterNot { it.isAbstract }
@@ -491,11 +491,13 @@ internal fun File.toUrl(): URL = toURI().toURL()
 
 internal fun testMethodName(name: String, num: Int): String = "test${name.capitalize()}$num"
 
+// TODO usvm-sbft: does SBFT allow to generate tests for private methods and constructors
+//  If no, add more filtering here
 internal val Method.isVisibleFromGeneratedTest: Boolean
     get() = (this.modifiers and Modifier.ABSTRACT) == 0
             && (this.modifiers and Modifier.NATIVE) == 0
 
-private fun StatsForClass.updateCoverage(newCoverage: Coverage, isNewClass: Boolean, fromFuzzing: Boolean) {
+fun StatsForClass.updateCoverage(newCoverage: Coverage, isNewClass: Boolean, fromFuzzing: Boolean) {
     coverage.update(newCoverage, isNewClass)
     // other coverage type updates by empty coverage to respect new class
     val emptyCoverage = newCoverage.copy(

utbot-junit-contest/src/main/kotlin/org/utbot/contest/ContestEstimator.kt

@@ -26,6 +26,7 @@ import org.utbot.contest.Paths.evosuiteReportFile
 import org.utbot.contest.Paths.jarsDir
 import org.utbot.contest.Paths.moduleTestDir
 import org.utbot.contest.Paths.outputDir
+import org.utbot.contest.usvm.runUsvmGeneration
 import org.utbot.features.FeatureExtractorFactoryImpl
 import org.utbot.features.FeatureProcessorWithStatesRepetitionFactory
 import org.utbot.framework.PathSelectorType
@@ -125,10 +126,20 @@ object Paths {
 }
 
 @Suppress("unused")
-enum class Tool {
-    UtBot {
-        @OptIn(ObsoleteCoroutinesApi::class)
-        @Suppress("EXPERIMENTAL_API_USAGE")
+interface Tool {
+    abstract class UtBotBasedTool : Tool {
+        abstract fun runGeneration(
+            project: ProjectToEstimate,
+            cut: ClassUnderTest,
+            timeLimit: Long,
+            fuzzingRatio: Double,
+            methodNameFilter: String?,
+            statsForProject: StatsForProject,
+            compiledTestDir: File,
+            classFqn: String,
+            expectedExceptions: ExpectedExceptionsForClass
+        ) : StatsForClass
+
         override fun run(
             project: ProjectToEstimate,
             cut: ClassUnderTest,
@@ -142,19 +153,21 @@ enum class Tool {
         ) = withUtContext(ContextManager.createNewContext(project.classloader)) {
             val classStats: StatsForClass = try {
                 runGeneration(
-                    project.name,
+                    project,
                     cut,
                     timeLimit,
                     fuzzingRatio,
-                    project.sootClasspathString,
-                    runFromEstimator = true,
-                    expectedExceptions,
-                    methodNameFilter
+                    methodNameFilter,
+                    statsForProject,
+                    compiledTestDir,
+                    classFqn,
+                    expectedExceptions
                 )
             } catch (e: CancellationException) {
                 logger.info { "[$classFqn] finished with CancellationException" }
                 return
             } catch (e: Throwable) {
+                logger.error(e) { "ISOLATION: $e" }
                 logger.info { "ISOLATION: $e" }
                 logger.info { "continue without compilation" }
                 return
@@ -198,8 +211,61 @@ enum class Tool {
         override fun moveProducedFilesIfNeeded() {
             // don't do anything
         }
-    },
-    EvoSuite {
+    }
+
+    object UtBot : UtBotBasedTool() {
+        @OptIn(ObsoleteCoroutinesApi::class)
+        @Suppress("EXPERIMENTAL_API_USAGE")
+        override fun runGeneration(
+            project: ProjectToEstimate,
+            cut: ClassUnderTest,
+            timeLimit: Long,
+            fuzzingRatio: Double,
+            methodNameFilter: String?,
+            statsForProject: StatsForProject,
+            compiledTestDir: File,
+            classFqn: String,
+            expectedExceptions: ExpectedExceptionsForClass
+        ): StatsForClass {
+            return runGeneration(
+                project.name,
+                cut,
+                timeLimit,
+                fuzzingRatio,
+                project.sootClasspathString,
+                runFromEstimator = true,
+                expectedExceptions,
+                methodNameFilter
+            )
+        }
+    }
+
+    object USVM : UtBotBasedTool() {
+        @OptIn(ObsoleteCoroutinesApi::class)
+        @Suppress("EXPERIMENTAL_API_USAGE")
+        override fun runGeneration(
+            project: ProjectToEstimate,
+            cut: ClassUnderTest,
+            timeLimit: Long,
+            fuzzingRatio: Double,
+            methodNameFilter: String?,
+            statsForProject: StatsForProject,
+            compiledTestDir: File,
+            classFqn: String,
+            expectedExceptions: ExpectedExceptionsForClass
+        ): StatsForClass = runUsvmGeneration(
+            project.name,
+            cut,
+            timeLimit,
+            fuzzingRatio,
+            project.sootClasspathString,
+            runFromEstimator = true,
+            expectedExceptions,
+            methodNameFilter
+        )
+    }
+
+    object EvoSuite : Tool {
         override fun run(
             project: ProjectToEstimate,
             cut: ClassUnderTest,
@@ -272,7 +338,7 @@ enum class Tool {
         }
     };
 
-    abstract fun run(
+    fun run(
         project: ProjectToEstimate,
         cut: ClassUnderTest,
         timeLimit: Long,
@@ -284,7 +350,7 @@ enum class Tool {
         expectedExceptions: ExpectedExceptionsForClass
     )
 
-    abstract fun moveProducedFilesIfNeeded()
+    fun moveProducedFilesIfNeeded()
 }
 
 fun main(args: Array<String>) {
@@ -295,7 +361,7 @@ fun main(args: Array<String>) {
     val tools: List<Tool>
 
     // very special case when you run your project directly from IntellijIDEA omitting command line arguments
-    if (args.isEmpty() && System.getProperty("os.name")?.run { contains("win", ignoreCase = true) } == true) {
+    if (args.isEmpty()) {
         processedClassesThreshold = 9999 //change to change number of classes to run
         val timeLimit = 20 // increase if you want to debug something
         val fuzzingRatio = 0.1 // sets fuzzing ratio to total test generation
@@ -319,7 +385,8 @@ fun main(args: Array<String>) {
         // config for SBST 2022
         methodFilter = null
         projectFilter = listOf("fastjson-1.2.50", "guava-26.0", "seata-core-0.5.0", "spoon-core-7.0.0")
-        tools = listOf(Tool.UtBot)
+        // TODO usvm-sbft-merge: add if here if we want merge contest it into main
+        tools = listOf(Tool.USVM)
 
         estimatorArgs = arrayOf(
             classesLists,
@@ -339,7 +406,8 @@ fun main(args: Array<String>) {
         processedClassesThreshold = 9999
         methodFilter = null
         projectFilter = null
-        tools = listOf(Tool.UtBot)
+        // TODO usvm-sbft-merge: add if here if we want merge contest it into main
+        tools = listOf(Tool.USVM)
     }
 
     JdkInfoService.jdkInfoProvider = ContestEstimatorJdkInfoProvider(javaHome)