From 2f7332b2f9bba145095171741b2086c15ea88964 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 20 Oct 2025 20:10:26 -0300 Subject: [PATCH 01/46] set version to 0.6.1-SNAPSHOT --- CITATION.cff | 2 +- README.md | 6 +++--- build.sbt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index 2b77079c..160f2112 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -18,6 +18,6 @@ authors: given-names: "Eric" orcid: "https://orcid.org/0000-0003-3470-3647" title: "SVFA-Scala: an implementation of SVFA for Java" -version: 0.6.0 +version: 0.6.1-SNAPSHOT date-released: 2025-09-06 url: "https://github.com/PAMunb/svfa" diff --git a/README.md b/README.md index 8dd29d5c..630bbb4b 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ This project follows a **modular architecture** with three focused modules: Add to your `build.sbt`: ```scala resolvers += Resolver.githubPackages("PAMunb", "svfa") -libraryDependencies += "br.unb.cic" %% "svfa-core" % "0.6.0" +libraryDependencies += "br.unb.cic" %% "svfa-core" % "0.6.1-SNAPSHOT" ``` #### Using svfa-core in Java/Maven Projects @@ -40,7 +40,7 @@ Add to your `pom.xml`: br.unb.cic svfa-core_2.12 - 0.6.0 + 0.6.1-SNAPSHOT ``` @@ -59,7 +59,7 @@ repositories { } dependencies { - implementation 'br.unb.cic:svfa-core_2.12:0.6.0' + implementation 'br.unb.cic:svfa-core_2.12:0.6.1-SNAPSHOT' } ``` diff --git a/build.sbt b/build.sbt index e73b1dc4..9d622e32 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ ThisBuild / scalaVersion := "2.12.20" ThisBuild / organization := "br.unb.cic" -ThisBuild / version := "0.6.0" +ThisBuild / version := "0.6.1-SNAPSHOT" // Global settings ThisBuild / publishConfiguration := publishConfiguration.value.withOverwrite(true) From 01594af0da75141bf0bda2e3c13e4a2a1fe512ec Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sun, 26 Oct 2025 21:39:34 -0300 Subject: [PATCH 02/46] remove time computation (partial) --- .../src/test/scala/br/unb/cic/metrics/TestResult.scala | 6 +++--- .../src/test/scala/br/unb/cic/metrics/TestResult.scala | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/core/src/test/scala/br/unb/cic/metrics/TestResult.scala b/modules/core/src/test/scala/br/unb/cic/metrics/TestResult.scala index 11641a8c..689bcca6 100644 --- a/modules/core/src/test/scala/br/unb/cic/metrics/TestResult.scala +++ b/modules/core/src/test/scala/br/unb/cic/metrics/TestResult.scala @@ -191,9 +191,9 @@ trait TestResult { ) val header = - "| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Execution Time |" + "| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score |" val sep = - "|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:--------------:|" + "|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|" println(header) println(sep) var totalFound = 0 @@ -229,7 +229,7 @@ trait TestResult { val totalF1 = f1Score() val totalStatus = s"${totalPassed}/${totalTests}" println( - f"| TOTAL | ${totalFound}%5d | ${totalExpected}%8d | ${totalStatus}%6s | ${totalTP}%2d | ${totalFP}%2d | ${totalFN}%3d | ${totalPrec}%9.2f | ${totalRec}%6.2f | ${totalF1}%7.2f | ${totalExecutionTime}%9.2f ms |" + f"| TOTAL | ${totalFound}%5d | ${totalExpected}%8d | ${totalStatus}%6s | ${totalTP}%2d | ${totalFP}%2d | ${totalFN}%3d | ${totalPrec}%9.2f | ${totalRec}%6.2f | ${totalF1}%7.2f |" ) } } diff --git a/modules/securibench/src/test/scala/br/unb/cic/metrics/TestResult.scala b/modules/securibench/src/test/scala/br/unb/cic/metrics/TestResult.scala index 6b8d8654..4169d502 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/metrics/TestResult.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/metrics/TestResult.scala @@ -191,9 +191,9 @@ trait TestResult { ) val header = - "| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Execution Time |" + "| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score |" val sep = - "|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:--------------:|" + "|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|" println(header) println(sep) var totalFound = 0 @@ -212,7 +212,7 @@ trait TestResult { val f1 = f1Score(testName) val shortTestName = testName.split('.').last.padTo(14, ' ') println( - f"| $shortTestName| ${m.found}%5d | ${m.expected}%8d | ${status}%6s | ${m.truePositives}%2d | ${m.falsePositives}%2d | ${m.falseNegatives}%3d | ${prec}%9.2f | ${rec}%6.2f | ${f1}%7.2f | ${m.executionTime}%9.2f ms |" + f"| $shortTestName| ${m.found}%5d | ${m.expected}%8d | ${status}%6s | ${m.truePositives}%2d | ${m.falsePositives}%2d | ${m.falseNegatives}%3d | ${prec}%9.2f | ${rec}%6.2f | ${f1}%7.2f |" ) totalFound += m.found totalExpected += m.expected @@ -229,7 +229,7 @@ trait TestResult { val totalF1 = f1Score() val totalStatus = s"${totalPassed}/${totalTests}" println( - f"| TOTAL | ${totalFound}%5d | ${totalExpected}%8d | ${totalStatus}%6s | ${totalTP}%2d | ${totalFP}%2d | ${totalFN}%3d | ${totalPrec}%9.2f | ${totalRec}%6.2f | ${totalF1}%7.2f | ${totalExecutionTime}%9.2f ms |" + f"| TOTAL | ${totalFound}%5d | ${totalExpected}%8d | ${totalStatus}%6s | ${totalTP}%2d | ${totalFP}%2d | ${totalFN}%3d | ${totalPrec}%9.2f | ${totalRec}%6.2f | ${totalF1}%7.2f |" ) } } From 4cd5b359821e5147ca3b3f87ee7c148c7de40093 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sun, 26 Oct 2025 20:39:29 -0300 Subject: [PATCH 03/46] metrics in v0.6.1 --- .../jsvfa/jsvfa-metrics-v0.6.1.md | 236 ++++++++++++++++++ 1 file changed, 236 insertions(+) create mode 100644 modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md new file mode 100644 index 00000000..26f0e261 --- /dev/null +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -0,0 +1,236 @@ + +> SUMMARY (*computed in October 2025.*) + +- **securibench.micro** - failed: 51, passed: 74 of 125 tests - (59.2%) + + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | +|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| +| Aliasing | 8 | 26 | 1/9 | 1 | 2 | 20 | 0.33 | 0.05 | 0.09 | 11.11% | +| Arrays | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | 50% | +| Basic | 56 | 60 | 37/42 | 53 | 1 | 5 | 0.98 | 0.91 | 0.94 | 88.1% | +| Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | 35.71% | +| Datastructures | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | 66.67% | +| Factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | 66.67% | +| Inter | 13 | 18 | 9/14 | 10 | 0 | 5 | 1.00 | 0.67 | 0.80 | 64.29% | +| Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | 0% | +| StrongUpdates | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | 60% | +| Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | +| Reflection | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | 0% | +| Sanitizers | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | 33.33% | +| TOTAL | 118 | 155 | 74/125 | 87 | 15 | 52 | 0.85 | 0.63 | 0.72 | 59.2% | + + +> Details + +- **securibench.micro.aliasing** - failed: 8, passed: 1 of 9 tests - (11.11%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:---------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing6 | 1 | 7 | ❌ | 0 | 0 | 6 | 0.00 | 0.00 | 0.00 | +| Aliasing7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Aliasing8 | 3 | 7 | ❌ | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | +| Aliasing9 | 0 | 7 | ❌ | 0 | 0 | 7 | 0.00 | 0.00 | 0.00 | +| TOTAL | 8 | 26 | 1/9 | 1 | 2 | 20 | 0.33 | 0.05 | 0.09 | + + +- **securibench.micro.arrays** - failed: 5, passed: 5 of 10 tests - (50.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Arrays1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays2 | 3 | 1 | ❌ | 0 | 2 | 0 | 0.00 | 0.00 | 0.00 | +| Arrays3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Arrays6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays8 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Arrays9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays10 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | + + +- **securibench.micro.basic** - failed: 5, passed: 37 of 42 tests - (88.1%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Basic0 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic5 | 3 | 3 | ✅ | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic11 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic12 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic15 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic16 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic17 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic18 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic19 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic20 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic21 | 4 | 4 | ✅ | 4 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic22 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic23 | 3 | 3 | ✅ | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic24 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic25 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic26 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic27 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic28 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic29 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic30 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic31 | 0 | 2 | ❌ | 0 | 0 | 2 | 0.00 | 0.00 | 0.00 | +| Basic32 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic33 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic34 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic35 | 6 | 6 | ✅ | 6 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic36 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic37 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic38 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Basic39 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic41 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic42 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 56 | 60 | 37/42 | 53 | 1 | 5 | 0.98 | 0.91 | 0.94 | + + +- **securibench.micro.collections** - failed: 9, passed: 5 of 14 tests - (35.71%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Collections1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections3 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections8 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections10 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Collections11 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | + + +- **securibench.micro.datastructures** - failed: 2, passed: 4 of 6 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:---------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Datastructures1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures4 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Datastructures5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Datastructures6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | + + +- **securibench.micro.factories** - failed: 1, passed: 2 of 3 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Factories1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Factories2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Factories3 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| TOTAL | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | + + +- **securibench.micro.inter** - failed: 5, passed: 9 of 14 tests - (64.29%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Inter1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter2 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter11 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 13 | 18 | 9/14 | 10 | 0 | 5 | 1.00 | 0.67 | 0.80 | + + +- **securibench.micro.session** - failed: 3, passed: 0 of 3 tests - (0.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Session1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Session2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Session3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | + + +- **securibench.micro.strong_updates** - failed: 2, passed: 3 of 5 tests - (60.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| StrongUpdates1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates2 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| StrongUpdates5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| TOTAL | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | + + +> Extra Tests + +These tests are not executed by Flowdroid + +- **securibench.micro.pred** - failed: 3, passed: 6 of 9 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Pred1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| Pred2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred5 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred6 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | + + +- **securibench.micro.reflection** - failed: 4, passed: 0 of 4 tests - (0.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Refl1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Refl2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Refl3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Refl4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | + + +- **securibench.micro.sanitizers** - failed: 4, passed: 2 of 6 tests - (33.33%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Sanitizers1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Sanitizers2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Sanitizers3 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| Sanitizers4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Sanitizers5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Sanitizers6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | \ No newline at end of file From 73fb395b19a65524643e5bddd98319c7105ac169 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 27 Oct 2025 15:34:38 -0300 Subject: [PATCH 04/46] metrics in v0.3.0 --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md index 60eedfe4..724a47c2 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md @@ -2,7 +2,7 @@ > SUMMARY (*computed in June 2023.*) -- **securibench.micro** - failed: 46, passed: 57 of 103 tests - (55.34%) +- **securibench.micro** - failed: 59, passed: 63 of 122 tests - (55.34%) | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | |:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| @@ -15,8 +15,10 @@ | inter | 10 | 18 | 7/14 | 8 | 0 | 8 | 1.00 | 0.50 | 0.67 | 50 | | session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | 0 | | strong_updates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | 80 | -| TOTAL | 94 | 126 | 57/103 | 75 | 8 | 40 | 0.90 | 0.65 | 0.75 | 55.34 | - +| Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | +| Reflection | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | 0% | +| Sanitizers | 0 | 4 | 0/6 | 0 | 0 | 6 | 0.00 | 0.00 | 0.00 | 0% | +| TOTAL | 102 | 139 | 63/122 | 80 | 11 | 50 | 0.88 | 0.62 | 0.72 | 51.64 | > Details From 5d400536288b785619e9eb28bea93260af701430 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 18:46:15 -0300 Subject: [PATCH 05/46] refactor logic in method reportConflicts --- .../main/scala/br/unb/cic/soot/graph/Graph.scala | 16 +--------------- .../main/scala/br/unb/cic/soot/svfa/SVFA.scala | 6 ++---- 2 files changed, 3 insertions(+), 19 deletions(-) diff --git a/modules/core/src/main/scala/br/unb/cic/soot/graph/Graph.scala b/modules/core/src/main/scala/br/unb/cic/soot/graph/Graph.scala index 7ab6c913..c89b7a1d 100644 --- a/modules/core/src/main/scala/br/unb/cic/soot/graph/Graph.scala +++ b/modules/core/src/main/scala/br/unb/cic/soot/graph/Graph.scala @@ -568,21 +568,7 @@ class Graph() { f(stmt) ) - def reportConflicts( - useUniquePaths: Boolean = false - ): scala.collection.Set[String] = { - val conflicts = findConflictingPaths() - - if (useUniquePaths) { - var conflictsByUniquePaths: Set[String] = Set.empty[String] - conflicts.foreach(path => { - conflictsByUniquePaths += s"source: ${path.head.show()} - sink: ${path.last.show()}" - }) - conflictsByUniquePaths - } else { - conflicts.map(p => p.toString) - } - } + def reportConflicts(): scala.collection.Set[List[GraphNode]] = findConflictingPaths() def findConflictingPaths(): scala.collection.Set[List[GraphNode]] = { if (fullGraph) { diff --git a/modules/core/src/main/scala/br/unb/cic/soot/svfa/SVFA.scala b/modules/core/src/main/scala/br/unb/cic/soot/svfa/SVFA.scala index 84cdbf0e..50e09328 100644 --- a/modules/core/src/main/scala/br/unb/cic/soot/svfa/SVFA.scala +++ b/modules/core/src/main/scala/br/unb/cic/soot/svfa/SVFA.scala @@ -27,10 +27,8 @@ abstract class SVFA extends SootConfiguration { svg.toDotModel() } - def reportConflictsSVG( - useUniquePaths: Boolean = false - ): collection.Set[String] = { - svg.reportConflicts(useUniquePaths) + def reportConflictsSVG(): scala.collection.Set[List[GraphNode]] = { + svg.reportConflicts() } def executionTime(): Double = { From ec8e43f96e1a0c7847c038e160bd1490dc4ce26c Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 18:46:29 -0300 Subject: [PATCH 06/46] enable test Aliasing6 --- .../unb/cic/securibench/deprecated/SecuribenchTestSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index 57ec6841..3f05fe8d 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -65,7 +65,7 @@ class SecuribenchTestSuite extends FunSuite { assert(svfa.reportConflictsSVG().size == expectedConflicts) } - ignore("in the class Aliasing6 we should detect 7 conflicts") { + test("in the class Aliasing6 we should detect 7 conflicts") { val testName = "Aliasing6" val expectedConflicts = 7 From 4a61c6e2e8165cd7a969e8d61df7b08ff21c9a66 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 22:06:43 -0300 Subject: [PATCH 07/46] add missing tests for Aliasing --- .../deprecated/SecuribenchTestSuite.scala | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index 57ec6841..fcc484ad 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -75,6 +75,36 @@ class SecuribenchTestSuite extends FunSuite { assert(svfa.reportConflictsSVG().size == expectedConflicts) } + ignore("in the class Aliasing7 we should detect 0 conflicts") { + val testName = "Aliasing7" + val expectedConflicts = 0 + + val svfa = new SecuribenchTest(s"securibench.micro.aliasing.$testName", "doGet") + svfa.buildSparseValueFlowGraph() + + assert(svfa.reportConflictsSVG().size == expectedConflicts) + } + + ignore("in the class Aliasing8 we should detect 7 conflicts") { + val testName = "Aliasing8" + val expectedConflicts = 7 + + val svfa = new SecuribenchTest(s"securibench.micro.aliasing.$testName", "doGet") + svfa.buildSparseValueFlowGraph() + + assert(svfa.reportConflictsSVG().size == expectedConflicts) + } + + ignore("in the class Aliasing9 we should detect 7 conflicts") { + val testName = "Aliasing9" + val expectedConflicts = 7 + + val svfa = new SecuribenchTest(s"securibench.micro.aliasing.$testName", "doGet") + svfa.buildSparseValueFlowGraph() + + assert(svfa.reportConflictsSVG().size == expectedConflicts) + } + /** ARRAY TESTs */ From 88784ed5085ba813520b01bee7df5c6fe8708c9d Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 22:10:08 -0300 Subject: [PATCH 08/46] merge all tests in a single file --- .../SecuribenchDeprecatedExtraTest.scala | 176 ----------------- .../deprecated/SecuribenchTestSuite.scala | 178 +++++++++++++++++- 2 files changed, 176 insertions(+), 178 deletions(-) delete mode 100644 modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedExtraTest.scala diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedExtraTest.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedExtraTest.scala deleted file mode 100644 index 59ea1389..00000000 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedExtraTest.scala +++ /dev/null @@ -1,176 +0,0 @@ -package br.unb.cic.securibench.deprecated - -import br.unb.cic.securibench.SecuribenchTest -import org.scalatest.FunSuite - -class SecuribenchDeprecatedExtraTest extends FunSuite { - - /** PRED TESTs - */ - - test( - "in the class Pred1 we should detect 0 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test( - "in the class Pred2 we should detect 1 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore( - "in the class Pred3 we should detect 0 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test( - "in the class Pred4 we should detect 1 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test( - "in the class Pred5 we should detect 1 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore( - "in the class Pred6 we should detect 0 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - ignore( - "in the class Pred7 we should detect 0 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred7", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test( - "in the class Pred8 we should detect 1 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred8", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test( - "in the class Pred9 we should detect 1 conflict(s) of a Pred test case" - ) { - val svfa = new SecuribenchTest("securibench.micro.pred.Pred9", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - /** REFLECTION TESTs - */ - ignore( - "in the class Refl1 we should detect 1 conflict(s) of a Reflection test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.reflection.Refl1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore( - "in the class Refl2 we should detect 1 conflict(s) of a Reflection test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.reflection.Refl2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore( - "in the class Refl3 we should detect 1 conflict(s) of a Reflection test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.reflection.Refl3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore( - "in the class Refl4 we should detect 1 conflict(s) of a Reflection test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.reflection.Refl4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - /** SANITIZERS TESTs - */ - ignore( - "in the class Sanitizers1 we should detect 1 conflict(s) of a Sanitizers test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.sanitizers.Sanitizers1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore( - "in the class Sanitizers2 we should detect 1 conflict(s) of a Sanitizers test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.sanitizers.Sanitizers2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test( - "in the class Sanitizers3 we should detect 0 conflict(s) of a Sanitizers test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.sanitizers.Sanitizers3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - ignore( - "in the class Sanitizers4 we should detect 2 conflict(s) of a Sanitizers test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.sanitizers.Sanitizers4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - ignore( - "in the class Sanitizers5 we should detect 1 conflict(s) of a Sanitizers test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.sanitizers.Sanitizers5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore( - "in the class Sanitizers6 we should detect 1 conflict(s) of a Sanitizers test case" - ) { - val svfa = - new SecuribenchTest("securibench.micro.sanitizers.Sanitizers6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index fcc484ad..26e3d00e 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -5,8 +5,9 @@ import org.scalatest.FunSuite class SecuribenchTestSuite extends FunSuite { - /** ALIASING TESTs - */ + /** + ALIASING TESTs + */ test( "in the class Aliasing1 we should detect 1 conflict of a simple aliasing test case" @@ -1321,4 +1322,177 @@ class SecuribenchTestSuite extends FunSuite { svfa.buildSparseValueFlowGraph() assert(svfa.reportConflictsSVG().size == expectedConflicts) } + + /** + * EXTRA TESTs + */ + + /** PRED TESTs + */ + + test( + "in the class Pred1 we should detect 0 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test( + "in the class Pred2 we should detect 1 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore( + "in the class Pred3 we should detect 0 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test( + "in the class Pred4 we should detect 1 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test( + "in the class Pred5 we should detect 1 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore( + "in the class Pred6 we should detect 0 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + ignore( + "in the class Pred7 we should detect 0 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred7", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test( + "in the class Pred8 we should detect 1 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred8", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test( + "in the class Pred9 we should detect 1 conflict(s) of a Pred test case" + ) { + val svfa = new SecuribenchTest("securibench.micro.pred.Pred9", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** REFLECTION TESTs + */ + ignore( + "in the class Refl1 we should detect 1 conflict(s) of a Reflection test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.reflection.Refl1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore( + "in the class Refl2 we should detect 1 conflict(s) of a Reflection test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.reflection.Refl2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore( + "in the class Refl3 we should detect 1 conflict(s) of a Reflection test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.reflection.Refl3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore( + "in the class Refl4 we should detect 1 conflict(s) of a Reflection test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.reflection.Refl4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** SANITIZERS TESTs + */ + ignore( + "in the class Sanitizers1 we should detect 1 conflict(s) of a Sanitizers test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.sanitizers.Sanitizers1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore( + "in the class Sanitizers2 we should detect 1 conflict(s) of a Sanitizers test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.sanitizers.Sanitizers2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test( + "in the class Sanitizers3 we should detect 0 conflict(s) of a Sanitizers test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.sanitizers.Sanitizers3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + ignore( + "in the class Sanitizers4 we should detect 2 conflict(s) of a Sanitizers test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.sanitizers.Sanitizers4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + ignore( + "in the class Sanitizers5 we should detect 1 conflict(s) of a Sanitizers test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.sanitizers.Sanitizers5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore( + "in the class Sanitizers6 we should detect 1 conflict(s) of a Sanitizers test case" + ) { + val svfa = + new SecuribenchTest("securibench.micro.sanitizers.Sanitizers6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } } From 66ac38fa8ee212701a7a0881be786bd308ee4989 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 22:15:34 -0300 Subject: [PATCH 09/46] enable Sanitizers1 --- .../unb/cic/securibench/deprecated/SecuribenchTestSuite.scala | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index 26e3d00e..c9c5f1f4 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -1324,7 +1324,9 @@ class SecuribenchTestSuite extends FunSuite { } /** + * * EXTRA TESTs + * */ /** PRED TESTs @@ -1442,7 +1444,7 @@ class SecuribenchTestSuite extends FunSuite { /** SANITIZERS TESTs */ - ignore( + test( "in the class Sanitizers1 we should detect 1 conflict(s) of a Sanitizers test case" ) { val svfa = From 46b61ee25d0e99e8321b48ed7713e8fc23154516 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 22:23:56 -0300 Subject: [PATCH 10/46] enable StrongUpdates4 and Inter11 --- .../unb/cic/securibench/deprecated/SecuribenchTestSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index c9c5f1f4..ad3c5dc8 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -1162,7 +1162,7 @@ class SecuribenchTestSuite extends FunSuite { } // FLAKY - ignore( + test( "in the class Inter11 we should detect 1 conflict of a simple inter test case" ) { val testName = "Inter11" @@ -1295,7 +1295,7 @@ class SecuribenchTestSuite extends FunSuite { } // It only fails in the Github action pipeline - ignore( + test( "in the class StrongUpdates4 we should detect 1 conflict of a simple strong update test case" ) { val testName = "StrongUpdates4" From 06a8370c3b2d69c3187bb83cf7522f213686f9c9 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 22:30:24 -0300 Subject: [PATCH 11/46] set StrongUpdates4 as FLAKY --- .../cic/securibench/deprecated/SecuribenchTestSuite.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index ad3c5dc8..68346acd 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -303,7 +303,6 @@ class SecuribenchTestSuite extends FunSuite { assert(svfa.reportConflictsSVG().size == expectedConflicts) } - // FLAKY test( "in the class Basic6 we should detect 1 conflict of a complex derived string test" ) { @@ -1161,7 +1160,6 @@ class SecuribenchTestSuite extends FunSuite { assert(svfa.reportConflictsSVG().size == expectedConflicts) } -// FLAKY test( "in the class Inter11 we should detect 1 conflict of a simple inter test case" ) { @@ -1294,8 +1292,8 @@ class SecuribenchTestSuite extends FunSuite { assert(svfa.reportConflictsSVG().size == expectedConflicts) } - // It only fails in the Github action pipeline - test( + // FLAKY: It only fails in the Github action pipeline + ignore( "in the class StrongUpdates4 we should detect 1 conflict of a simple strong update test case" ) { val testName = "StrongUpdates4" From a13d6dfa7e3b01ea537a087ff4dfb05052b3985d Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 22:34:37 -0300 Subject: [PATCH 12/46] set Inter11 as FLAKY --- .../unb/cic/securibench/deprecated/SecuribenchTestSuite.scala | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index 68346acd..1e66c151 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -1160,7 +1160,8 @@ class SecuribenchTestSuite extends FunSuite { assert(svfa.reportConflictsSVG().size == expectedConflicts) } - test( + // FLAKY: It only fails in the Github action pipeline + ignore( "in the class Inter11 we should detect 1 conflict of a simple inter test case" ) { val testName = "Inter11" From 924d5e111ee98e62397c519f48837de8b3d52f3f Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 22:54:40 -0300 Subject: [PATCH 13/46] fix tests in AndroidTaintBenchSuiteExperiment2Test --- ...ndroidTaintBenchSuiteExperiment2Test.scala | 76 +++++++++---------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/modules/taintbench/src/test/scala/br/unb/cic/android/AndroidTaintBenchSuiteExperiment2Test.scala b/modules/taintbench/src/test/scala/br/unb/cic/android/AndroidTaintBenchSuiteExperiment2Test.scala index 2588f22f..ca2d704c 100644 --- a/modules/taintbench/src/test/scala/br/unb/cic/android/AndroidTaintBenchSuiteExperiment2Test.scala +++ b/modules/taintbench/src/test/scala/br/unb/cic/android/AndroidTaintBenchSuiteExperiment2Test.scala @@ -23,7 +23,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -36,7 +36,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -49,7 +49,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -62,7 +62,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -75,7 +75,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -88,7 +88,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -101,7 +101,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -114,7 +114,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -127,7 +127,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -140,7 +140,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -153,7 +153,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -166,7 +166,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -179,7 +179,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -192,7 +192,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -205,7 +205,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -218,7 +218,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -231,7 +231,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -244,7 +244,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -258,7 +258,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -271,7 +271,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -284,7 +284,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -297,7 +297,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -310,7 +310,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -323,7 +323,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -336,7 +336,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -349,7 +349,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -362,7 +362,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -375,7 +375,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -388,7 +388,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -401,7 +401,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -415,7 +415,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -428,7 +428,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -444,7 +444,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -460,7 +460,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -473,7 +473,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -486,7 +486,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -499,7 +499,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } @@ -512,7 +512,7 @@ class AndroidTaintBenchSuiteExperiment2Test extends FunSuite createJsonReport(svfa.conflictPaths() , nameAPK, "modules/taintbench/src/docs-metrics/experiment-II/findings") - val found = svfa.reportConflictsSVG(true).size + val found = svfa.reportConflictsSVG().size assert(found == expected) } From 42394e4d0b5262dae4ef589aaf39b5c198e27f72 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 22:57:52 -0300 Subject: [PATCH 14/46] update metrics for aliasing --- .../jsvfa/jsvfa-metrics-v0.6.1.md | 37 ++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 26f0e261..46f46760 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -1,12 +1,11 @@ > SUMMARY (*computed in October 2025.*) -- **securibench.micro** - failed: 51, passed: 74 of 125 tests - (59.2%) - +- **securibench.micro** - failed: 50, passed: 75 of 125 tests - (60.0%) | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | |:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| -| Aliasing | 8 | 26 | 1/9 | 1 | 2 | 20 | 0.33 | 0.05 | 0.09 | 11.11% | +| Aliasing | 25 | 26 | 2/9 | 8 | 9 | 10 | 0.47 | 0.44 | 0.45 | 22.22% | | Arrays | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | 50% | | Basic | 56 | 60 | 37/42 | 53 | 1 | 5 | 0.98 | 0.91 | 0.94 | 88.1% | | Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | 35.71% | @@ -23,20 +22,24 @@ > Details -- **securibench.micro.aliasing** - failed: 8, passed: 1 of 9 tests - (11.11%) - -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | -|:---------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| -| Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Aliasing6 | 1 | 7 | ❌ | 0 | 0 | 6 | 0.00 | 0.00 | 0.00 | -| Aliasing7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Aliasing8 | 3 | 7 | ❌ | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | -| Aliasing9 | 0 | 7 | ❌ | 0 | 0 | 7 | 0.00 | 0.00 | 0.00 | -| TOTAL | 8 | 26 | 1/9 | 1 | 2 | 20 | 0.33 | 0.05 | 0.09 | +[//]: # (* wrong counting) + +[//]: # (** array indexes) + +- **securibench.micro.aliasing** - failed: 7, passed: 2 of 9 tests - (22.22%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:---------:|:-----:|:--------:|:-------:|:--:|:--:|:-----|:---------:|:-------:|:-------:| +| Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing6 | 7 | 7 | ✅ | 7 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Aliasing7 | 7 | 0 | ❌ | 0 | 7 | 0 | 0.00 | 0.00 | 0.00 | +| Aliasing8 | 8 | 7 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Aliasing9 | 0 | 7 | ❌ | 0 | 0 | 7 | 0.00 | 0.00 | 0.00 | +| TOTAL | 25 | 26 | 2/9 | 8 | 9 | 10 | 0.47 | 0.44 | 0.45 | - **securibench.micro.arrays** - failed: 5, passed: 5 of 10 tests - (50.0%) From cc8e745e345f5f9b45569981e68b9fb4b1de4ff6 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 4 Nov 2025 23:06:15 -0300 Subject: [PATCH 15/46] Add comments for aliasing tests --- .../docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 46f46760..2dfc7dce 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -22,23 +22,23 @@ > Details -[//]: # (* wrong counting) +[//]: # () -[//]: # (** array indexes) +[//]: # ) - **securibench.micro.aliasing** - failed: 7, passed: 2 of 9 tests - (22.22%) | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:---------:|:-----:|:--------:|:-------:|:--:|:--:|:-----|:---------:|:-------:|:-------:| | Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * wrong counting +| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | ** array indexes +| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * wrong counting +| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | > | Aliasing6 | 7 | 7 | ✅ | 7 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Aliasing7 | 7 | 0 | ❌ | 0 | 7 | 0 | 0.00 | 0.00 | 0.00 | -| Aliasing8 | 8 | 7 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Aliasing9 | 0 | 7 | ❌ | 0 | 0 | 7 | 0.00 | 0.00 | 0.00 | +| Aliasing7 | 7 | 0 | ❌ | 0 | 7 | 0 | 0.00 | 0.00 | 0.00 | ** array indexes +| Aliasing8 | 8 | 7 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * wrong counting +| Aliasing9 | 0 | 7 | ❌ | 0 | 0 | 7 | 0.00 | 0.00 | 0.00 | ** array indexes | TOTAL | 25 | 26 | 2/9 | 8 | 9 | 10 | 0.47 | 0.44 | 0.45 | From 3bb8367eafd215d35981a31429b9ddbf401f671e Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Thu, 6 Nov 2025 09:48:17 -0300 Subject: [PATCH 16/46] add comments about logic --- .../br/unb/cic/soot/svfa/jimple/JSVFA.scala | 61 +++++++++++++------ 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/JSVFA.scala b/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/JSVFA.scala index a9692466..0bf01409 100644 --- a/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/JSVFA.scala +++ b/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/JSVFA.scala @@ -317,7 +317,7 @@ abstract class JSVFA case (p: Local, q: ArrayRef) => // p = q[i] loadArrayRule(assignStmt.stmt, q, method, defs) case (p: Local, q: InvokeExpr) => - invokeRule(assignStmt, q, method, defs) // call a method + invokeRule(assignStmt, q, method, defs) // p = myObject.method() : call a method and assign its return value to a local variable case (p: Local, q: Local) => copyRule(assignStmt.stmt, q, method, defs) case (p: Local, _) => copyRuleInvolvingExpressions(assignStmt.stmt, method, defs) @@ -367,10 +367,18 @@ abstract class JSVFA }) } - /** Handles invocation rules for a call statement by traversing the call - * graph. This method avoids infinite recursion and limits the traversal - * depth for performance. - */ + /** + * Handles invocation rules for a call statement by traversing the call + * graph. This method avoids infinite recursion and limits the traversal + * depth for performance. + * + * i.e: + * + * myObject.method() + * myObject.method(q) + * this.method() + * this.method(q) + */ private def invokeRule( callStmt: Statement, exp: InvokeExpr, @@ -1039,14 +1047,28 @@ abstract class JSVFA case v => v } - /** CASE 1: UPDATE EDGE(S) "FROM" each stmt where the variable, passed as an - * argument, is defined. "TO" stmt where the method is called (call-site - * stmt). - * - * CASE 2: ??? - * - * CASE 2: ??? - */ + /** CASE #1: UPDATE EDGE(S) "FROM" each stmt where the variable, passed as an + * argument, is defined. "TO" stmt where the method is called (call-site + * stmt). i.e: [s1 -> s2] + * + * ------------------- + * s1: p = ... + * s2: myObj.method(p) + * ------------------- + * + * CASE #2: + * TO-DO + * + * CASE #3: UPDATE EDGE(S) "FROM" from definition of base object "TO" where it + * calls any of its methods. The expression must be type (invoke). + * i.e: [s1 -> s2] + * + * ------------------- + * s1: myObj = new Object() + * s2: myObj.method() + * ------------------- + * + */ private def defsToCallOfSinkMethod( stmt: Statement, exp: InvokeExpr, @@ -1054,7 +1076,7 @@ abstract class JSVFA defs: SimpleLocalDefs ) = { - // CASE 1 + // CASE #1 exp.getArgs .stream() .filter(a => a.isInstanceOf[Local]) @@ -1069,22 +1091,21 @@ abstract class JSVFA updateGraph( source, target - ) // update 'edge(s)' FROM "declaration stmt(s) for args" TO "call-site stmt" (current stmt) + ) }) - // CASE 2 + // CASE #2 if (local.getType.isInstanceOf[ArrayType]) { val stores = arrayStores.getOrElseUpdate(local, List()) stores.foreach(sourceStmt => { val source = createNode(caller, sourceStmt) val target = createNode(caller, targetStmt) - updateGraph(source, target) // add comment + updateGraph(source, target) }) } }) - // CASE 3 - // edges from definition to base object of an invoke expression + // CASE #3 if (isFieldSensitiveAnalysis() && exp.isInstanceOf[InstanceInvokeExpr]) { if (exp.asInstanceOf[InstanceInvokeExpr].getBase.isInstanceOf[Local]) { val local = @@ -1095,7 +1116,7 @@ abstract class JSVFA .forEach(sourceStmt => { val source = createNode(caller, sourceStmt) val target = createNode(caller, targetStmt) - updateGraph(source, target) // add comment + updateGraph(source, target) }) } } From e27b701fcb7fb02706cd2fc790f1972c13ea215c Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Thu, 6 Nov 2025 19:27:25 -0300 Subject: [PATCH 17/46] to-do for method isValidContext --- .../core/src/main/scala/br/unb/cic/soot/graph/Graph.scala | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/modules/core/src/main/scala/br/unb/cic/soot/graph/Graph.scala b/modules/core/src/main/scala/br/unb/cic/soot/graph/Graph.scala index c89b7a1d..505ae327 100644 --- a/modules/core/src/main/scala/br/unb/cic/soot/graph/Graph.scala +++ b/modules/core/src/main/scala/br/unb/cic/soot/graph/Graph.scala @@ -523,6 +523,12 @@ class Graph() { val csOpenAndClose = csOpen ++ csClose + /** + TO-DO: Implement a better way to calculate the right csOpen and Close + because the right one can lead to a bug in some edges cases. + */ + + csOpenAndClose.foreach(open => { if (open.value.context.nonEmpty) { cs = cs + open.value.context.head From f73a7bbdd722094136b9d4c88a0bd57d4d0d077b Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Fri, 7 Nov 2025 08:18:07 -0300 Subject: [PATCH 18/46] remove extra aliasing tests --- .../securibench/micro/aliasing/Aliasing7.java | 52 ----------------- .../securibench/micro/aliasing/Aliasing8.java | 58 ------------------- .../securibench/micro/aliasing/Aliasing9.java | 51 ---------------- .../deprecated/SecuribenchTestSuite.scala | 30 ---------- 4 files changed, 191 deletions(-) delete mode 100644 modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing7.java delete mode 100644 modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing8.java delete mode 100644 modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing9.java diff --git a/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing7.java b/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing7.java deleted file mode 100644 index 4e6281a9..00000000 --- a/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing7.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - @author Benjamin Livshits - - $Id: Aliasing6.java,v 1.1 2006/04/21 17:14:27 livshits Exp $ - */ -package securibench.micro.aliasing; - -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import securibench.micro.BasicTestCase; -import securibench.micro.MicroTestCase; - -/** - * @servlet description="false positive of aliasing with copy propagation" - * @servlet vuln_count = "0" - * */ -public class Aliasing7 extends BasicTestCase implements MicroTestCase { - private static final String FIELD_NAME = "name"; - - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String[] names = new String[2]; - names[0] = req.getParameter(FIELD_NAME); - names[1] = "Foo"; - Object - o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15, o16, o17, o18, o19, o20, - o21, o22, o23, o24, o25, o26, o27, o28, o29, o30, o31, o32, o33, o34, o35, o36, o37, o38, o39, o40; - o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = o10 = o11 = o12 = o13 = o14 = o15 = o16 = o17 = o18 = o19 = o20 = - o21 = o22 = o23 = o24 = o25 = o26 = o27 = o28 = o29 = o30 = o31 = o32 = o33 = o34 = o35 = o36 = o37 = o38 = o39 = o40 = - names[1]; - - PrintWriter writer = resp.getWriter(); - writer.println(o1); /* OK */ - writer.println(o2); /* OK */ - writer.println(o3); /* OK */ - writer.println(o4); /* OK */ - writer.println(o32); /* OK */ - writer.println(o37); /* OK */ - writer.println(o40); /* OK */ - } - - public String getDescription() { - return "false positive of aliasing with copy propagation"; - } - - public int getVulnerabilityCount() { - return 0; - } -} \ No newline at end of file diff --git a/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing8.java b/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing8.java deleted file mode 100644 index eedee156..00000000 --- a/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing8.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - @author Benjamin Livshits - - $Id: Aliasing6.java,v 1.1 2006/04/21 17:14:27 livshits Exp $ - */ -package securibench.micro.aliasing; - -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import securibench.micro.BasicTestCase; -import securibench.micro.MicroTestCase; - -/** - * @servlet description="aliasing in an array index" - * @servlet vuln_count = "1" - * */ -public class Aliasing8 extends BasicTestCase implements MicroTestCase { - private static final String FIELD_NAME = "name"; - - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String[] names = new String[2]; - names[0] = req.getParameter(FIELD_NAME); - names[1] = "Foo"; - Object - o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15, o16, o17, o18, o19, o20, - o21, o22, o23, o24, o25, o26, o27, o28, o29, o30, o31, o32, o33, o34, o35, o36, o37, o38, o39, o40; - o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = o10 = o11 = o12 = o13 = o14 = o15 = o16 = o17 = o18 = o19 = o20 = - o21 = o22 = o23 = o24 = o25 = o26 = o27 = o28 = o29 = o30 = o31 = o32 = o33 = o34 = o35 = o36 = o37 = o38 = o39 = o40 = - names[1]; - - o2 = names[0]; - - PrintWriter writer = resp.getWriter(); - writer.println(o1); /* BAD */ - writer.println(o2); /* BAD */ - writer.println(o3); /* BAD */ - writer.println(o4); /* BAD */ - writer.println(o32); /* BAD */ - writer.println(o37); /* BAD */ - writer.println(o40); /* BAD */ - - sink(names); /* This should be consider a leak too, because one array element is tainted */ - } - - public String getDescription() { - return "aliasing in an array index"; - } - - public int getVulnerabilityCount() { - return 7; - } - - public void sink(String[] data) { } -} \ No newline at end of file diff --git a/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing9.java b/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing9.java deleted file mode 100644 index 73635096..00000000 --- a/modules/securibench/src/test/java/securibench/micro/aliasing/Aliasing9.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - @author Benjamin Livshits - - $Id: Aliasing6.java,v 1.1 2006/04/21 17:14:27 livshits Exp $ - */ -package securibench.micro.aliasing; - -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import securibench.micro.BasicTestCase; -import securibench.micro.MicroTestCase; - -/** - * @servlet description="interprocedural aliasing in an array index" - * @servlet vuln_count = "1" - * */ -public class Aliasing9 extends BasicTestCase implements MicroTestCase { - private static final String FIELD_NAME = "name"; - - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String[] names = source(req); - Object o1, o2; - o1 = o2 = names[1]; - - o2 = names[0]; - - PrintWriter writer = resp.getWriter(); - writer.println(o1); /* OK */ - writer.println(o2); /* BAD */ - } - - public String[] source(HttpServletRequest req) { - String[] names = new String[2]; - names[0] = req.getParameter(FIELD_NAME); - names[1] = "Foo"; - - return names; - } - - public String getDescription() { - return "aliasing with copy propagation"; - } - - public int getVulnerabilityCount() { - return 7; - } -} \ No newline at end of file diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index fad55639..e2f7c806 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -76,36 +76,6 @@ class SecuribenchTestSuite extends FunSuite { assert(svfa.reportConflictsSVG().size == expectedConflicts) } - ignore("in the class Aliasing7 we should detect 0 conflicts") { - val testName = "Aliasing7" - val expectedConflicts = 0 - - val svfa = new SecuribenchTest(s"securibench.micro.aliasing.$testName", "doGet") - svfa.buildSparseValueFlowGraph() - - assert(svfa.reportConflictsSVG().size == expectedConflicts) - } - - ignore("in the class Aliasing8 we should detect 7 conflicts") { - val testName = "Aliasing8" - val expectedConflicts = 7 - - val svfa = new SecuribenchTest(s"securibench.micro.aliasing.$testName", "doGet") - svfa.buildSparseValueFlowGraph() - - assert(svfa.reportConflictsSVG().size == expectedConflicts) - } - - ignore("in the class Aliasing9 we should detect 7 conflicts") { - val testName = "Aliasing9" - val expectedConflicts = 7 - - val svfa = new SecuribenchTest(s"securibench.micro.aliasing.$testName", "doGet") - svfa.buildSparseValueFlowGraph() - - assert(svfa.reportConflictsSVG().size == expectedConflicts) - } - /** ARRAY TESTs */ From 184c26410070582bb1edc1ff9593fc51fd2f8061 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Fri, 7 Nov 2025 08:18:13 -0300 Subject: [PATCH 19/46] recompute metrics --- .../jsvfa/jsvfa-metrics-v0.6.1.md | 33 +++++++++---------- 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 2dfc7dce..cdcd8b4a 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -1,11 +1,11 @@ -> SUMMARY (*computed in October 2025.*) +> SUMMARY (*computed in November 2025.*) -- **securibench.micro** - failed: 50, passed: 75 of 125 tests - (60.0%) +- **securibench.micro** - failed: 47, passed: 75 of 122 tests - (61.48%) | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | |:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| -| Aliasing | 25 | 26 | 2/9 | 8 | 9 | 10 | 0.47 | 0.44 | 0.45 | 22.22% | +| Aliasing | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | 33.33% | | Arrays | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | 50% | | Basic | 56 | 60 | 37/42 | 53 | 1 | 5 | 0.98 | 0.91 | 0.94 | 88.1% | | Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | 35.71% | @@ -17,7 +17,7 @@ | Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | | Reflection | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | 0% | | Sanitizers | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | 33.33% | -| TOTAL | 118 | 155 | 74/125 | 87 | 15 | 52 | 0.85 | 0.63 | 0.72 | 59.2% | +| TOTAL | 120 | 141 | 75/122 | 95 | 14 | 35 | 0.87 | 0.73 | 0.79 | 61.48% | > Details @@ -26,20 +26,17 @@ [//]: # ) -- **securibench.micro.aliasing** - failed: 7, passed: 2 of 9 tests - (22.22%) - -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | -|:---------:|:-----:|:--------:|:-------:|:--:|:--:|:-----|:---------:|:-------:|:-------:| -| Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * wrong counting -| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | ** array indexes -| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * wrong counting -| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | > -| Aliasing6 | 7 | 7 | ✅ | 7 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Aliasing7 | 7 | 0 | ❌ | 0 | 7 | 0 | 0.00 | 0.00 | 0.00 | ** array indexes -| Aliasing8 | 8 | 7 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * wrong counting -| Aliasing9 | 0 | 7 | ❌ | 0 | 0 | 7 | 0.00 | 0.00 | 0.00 | ** array indexes -| TOTAL | 25 | 26 | 2/9 | 8 | 9 | 10 | 0.47 | 0.44 | 0.45 | +- **securibench.micro.aliasing** - failed: 4, passed: 2 of 6 tests - (33.33%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:---------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * wrong counting +| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | ** array indexes +| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * wrong counting +| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing6 | 7 | 7 | ✅ | 7 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | - **securibench.micro.arrays** - failed: 5, passed: 5 of 10 tests - (50.0%) From 2f6b3b5a904a5b861365260f894d0832d3e33878 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Thu, 20 Nov 2025 21:30:07 -0300 Subject: [PATCH 20/46] update total for metrics v0.3.0 --- .../securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md index 724a47c2..207910cf 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md @@ -2,7 +2,7 @@ > SUMMARY (*computed in June 2023.*) -- **securibench.micro** - failed: 59, passed: 63 of 122 tests - (55.34%) +- **securibench.micro** - failed: 59, passed: 63 of 122 tests - (51.63%) | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | |:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| From f39f1257bda48d07b4584cd86eb9f30faa1d1f48 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Thu, 20 Nov 2025 21:30:31 -0300 Subject: [PATCH 21/46] update metrics for Basic category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index cdcd8b4a..001ab085 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -7,7 +7,7 @@ |:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| | Aliasing | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | 33.33% | | Arrays | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | 50% | -| Basic | 56 | 60 | 37/42 | 53 | 1 | 5 | 0.98 | 0.91 | 0.94 | 88.1% | +| Basic | 57 | 60 | 38/42 | 55 | 1 | 4 | 0.98 | 0.93 | 0.95 | 90.48% | | Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | 35.71% | | Datastructures | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | 66.67% | | Factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | 66.67% | @@ -56,7 +56,7 @@ | TOTAL | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | -- **securibench.micro.basic** - failed: 5, passed: 37 of 42 tests - (88.1%) +- **securibench.micro.basic** - failed: 4, passed: 38 of 42 tests - (90.48%) | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:-------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| @@ -88,7 +88,7 @@ | Basic25 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic26 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic27 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic28 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic28 | 2 | 2 | ✅ | 0 | 0 | 2 | 1.00 | 1.00 | 1.00 | | Basic29 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic30 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic31 | 0 | 2 | ❌ | 0 | 0 | 2 | 0.00 | 0.00 | 0.00 | @@ -102,7 +102,7 @@ | Basic39 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic41 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic42 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| TOTAL | 56 | 60 | 37/42 | 53 | 1 | 5 | 0.98 | 0.91 | 0.94 | +| TOTAL | 57 | 60 | 38/42 | 55 | 1 | 4 | 0.98 | 0.93 | 0.95 | - **securibench.micro.collections** - failed: 9, passed: 5 of 14 tests - (35.71%) From 24ac939dc5c7b5a79b5f7b0bab4c52734c0daf4b Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Thu, 20 Nov 2025 21:40:29 -0300 Subject: [PATCH 22/46] update metrics for Inter category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 001ab085..3c75d88d 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -11,7 +11,7 @@ | Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | 35.71% | | Datastructures | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | 66.67% | | Factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | 66.67% | -| Inter | 13 | 18 | 9/14 | 10 | 0 | 5 | 1.00 | 0.67 | 0.80 | 64.29% | +| Inter | 12 | 18 | 8/14 | 9 | 0 | 6 | 1.00 | 0.60 | 0.75 | 57.14% | | Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | 0% | | StrongUpdates | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | 60% | | Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | @@ -149,7 +149,7 @@ | TOTAL | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | -- **securibench.micro.inter** - failed: 5, passed: 9 of 14 tests - (64.29%) +- **securibench.micro.inter** - failed: 6, passed: 8 of 14 tests - (57.14%) | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:-------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| @@ -163,11 +163,11 @@ | Inter8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | | Inter10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter11 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter11 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | *flaky | Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | | Inter13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| TOTAL | 13 | 18 | 9/14 | 10 | 0 | 5 | 1.00 | 0.67 | 0.80 | +| TOTAL | 12 | 18 | 8/14 | 9 | 0 | 6 | 1.00 | 0.60 | 0.75 | - **securibench.micro.session** - failed: 3, passed: 0 of 3 tests - (0.0%) From 5c2cac76585f7c1ba4edbf44414b035426e89c63 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Fri, 21 Nov 2025 02:45:38 -0300 Subject: [PATCH 23/46] add info to readme --- README.md | 31 ++++++++++++++++++++++--------- 1 file changed, 22 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 630bbb4b..5a03c9af 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ This project follows a **modular architecture** with three focused modules: - **`core`**: Essential SVFA framework + Android analysis support - **`securibench`**: Java security vulnerability analysis benchmarks - **`taintbench`**: Android malware analysis benchmarks +- **`scripts`**: snippets of code use to automatize metric computation ## Quick Start @@ -110,12 +111,24 @@ sbt testTaintbench ## API Usage -Implement a class that extends the `JSVFA class` (see examples in the tests). You must provide implementations for: +Implement a class that extends the `JSVFA class` (i.e., scala/br/unb/cic/svfa/JSVFATest.scala) + +You must provide implementations for: * `getEntryPoints()` - Set up the "main" methods (returns List of Soot methods) * `sootClassPath()` - Set up the soot classpath (returns String) * `analyze(unit)` - Identify node types (source, sink, simple node) in the graph +The framework implements a flexible approach (`trait`) to store the set of node types (source, sink) +Then, this trait is available to be manipulated in method `analyze(unit)` + +```scala +trait SecuribenchSpec { + val sinkList: Seq[String] = List() + + val sourceList: Seq[String] = List() +``` + ### Example Usage ```scala @@ -138,15 +151,15 @@ class MyAnalysis extends JSVFATest { ## Available Commands -| Command | Description | -|---------|-------------| -| `sbt testCore` | Run core SVFA tests | +| Command | Description | +|-----------------------|--------------------------------------------------| +| `sbt testCore` | Run core SVFA tests | | `sbt testSecuribench` | Run security vulnerability tests (93 test cases) | -| `sbt testTaintbench` | Run Android malware tests | -| `sbt testRoidsec` | Run specific Roidsec test | -| `sbt compileAll` | Compile all modules | -| `sbt publishCore` | Publish core module to GitHub Packages | -| `sbt publishAllLocal` | Publish all modules to local Maven repository | +| `sbt testTaintbench` | Run Android malware tests | +| `sbt testRoidsec` | Run specific Roidsec test | +| `sbt compileAll` | Compile all modules | +| `sbt publishCore` | Publish core module to GitHub Packages | +| `sbt publishAllLocal` | Publish all modules to local Maven repository | ## Scripts From 0c25e64b6cc1be65d95e6be9e6d449de126d046e Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sat, 22 Nov 2025 13:18:02 -0300 Subject: [PATCH 24/46] update joana metrics --- .../src/docs-metrics/joana/joana-metrics.md | 239 +++++++++++------- 1 file changed, 142 insertions(+), 97 deletions(-) diff --git a/modules/securibench/src/docs-metrics/joana/joana-metrics.md b/modules/securibench/src/docs-metrics/joana/joana-metrics.md index b84e7d2e..89056fbc 100644 --- a/modules/securibench/src/docs-metrics/joana/joana-metrics.md +++ b/modules/securibench/src/docs-metrics/joana/joana-metrics.md @@ -1,39 +1,42 @@ -## Joana Metrics +## JOANA Metrics -> SUMMARY +> SUMMARY (*computed in November 2025.*) -- **Securibench** - failed: 32, passed: 71 of 103 tests. `(68.93%)` +- **Securibench** - failed: 37, passed: 85 of 122 tests. `(69.67%)` -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | -|----------------|-------|----------|--------|----|----|----|-----------|--------|------| -| Aliasing | 6 | 11 | 2/6 | 2 | 2 | 7 | 0.50 | 0.22 | 0.31 | -| Arrays | 10 | 9 | 9/10 | 9 | 1 | 0 | 0.90 | 1.00 | 0.95 | -| Basic | 45 | 61 | 25/42 | 26 | 6 | 22 | 0.81 | 0.54 | 0.65 | -| Collections | 15 | 15 | 14/14 | 15 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Datastructures | 6 | 5 | 5/6 | 5 | 1 | 0 | 0.83 | 1.00 | 0.91 | -| Factories | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter | 13 | 18 | 9/14 | 9 | 0 | 5 | 1.00 | 0.64 | 0.78 | -| Session | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| StrongUpdates | 5 | 1 | 1/5 | 1 | 4 | 0 | 0.20 | 1.00 | 0.33 | -| TOTAL | 106 | 126 | 71/103 | 73 | 14 | 34 | 0.84 | 0.68 | 0.75 | +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | Pass rate | +|----------------|-------|----------|--------|----|----|----|-----------|--------|-------|-----------| +| Aliasing | 6 | 11 | 2/6 | 2 | 2 | 7 | 0.50 | 0.22 | 0.31 | 33.33% | +| Arrays | 10 | 9 | 9/10 | 9 | 1 | 0 | 0.90 | 1.00 | 0.95 | 90% | +| Basic | 45 | 61 | 25/42 | 26 | 6 | 22 | 0.81 | 0.54 | 0.65 | 59.52% | +| Collections | 15 | 14 | 13/14 | 14 | 1 | 0 | 0.93 | 1.00 | 0.96 | 92.86% | +| Datastructures | 6 | 5 | 5/6 | 5 | 1 | 0 | 0.83 | 1.00 | 0.91 | 83.33% | +| Factories | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | 100% | +| Inter | 13 | 16 | 11/14 | 11 | 0 | 3 | 1.00 | 0.79 | 0.88 | 78.57% | +| Session | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | 100% | +| StrongUpdates | 5 | 1 | 1/5 | 1 | 4 | 0 | 0.20 | 1.00 | 0.33 | 20% | +| Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | +| Reflection | 3 | 4 | 3/4 | 3 | 0 | 1 | 1.00 | 0.75 | 0.86 | 75% | +| Sanitizer | 6 | 6 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | 66.67% | +| TOTAL | 123 | 138 | 85/122 | 86 | 19 | 34 | 0.82 | 0.72 | 0.77 | 69.67% | > DETAILS - **AliasingTest** - failed: 4, passed: 2 of 6 tests. `(33.33%)` -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | -|-----------|-------|----------|--------|----|----|----|-----------|--------|------| -| Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Aliasing2 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Aliasing3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Aliasing4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Aliasing5 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Aliasing6 | 1 | 7 | ❌ | 0 | 0 | 6 | 0.00 | 0.00 | 0.00 | -| TOTAL | 6 | 11 | 2/6 | 2 | 2 | 7 | 0.50 | 0.22 | 0.31 | +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | +|------------|-------|----------|--------|----|----|----|-----------|--------|------| +| Aliasing1 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Aliasing2 | 1 | 0 | FAIL | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Aliasing3 | 1 | 0 | FAIL | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Aliasing4 | 1 | 2 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing5 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Aliasing6 | 1 | 7 | FAIL | 0 | 0 | 6 | 0.00 | 0.00 | 0.00 | +| TOTAL | 6 | 11 | 2/6 | 2 | 2 | 7 | 0.50 | 0.22 | 0.31 | -- **ArraysTest** - failed: 1, passed: 9 of 10 tests. `(90.00%)` +- **ArraysTest** - failed: 1, passed: 9 of 10 tests. `(90%)` | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | |----------|-------|----------|--------|----|----|----|-----------|--------|------| @@ -52,54 +55,54 @@ - **BasicTest** - failed: 17, passed: 25 of 42 tests. `(59.52%)` -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | -|----------|-------|----------|--------|----|----|----|-----------|--------|------| -| Basic1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic5 | 1 | 3 | ❌ | 0 | 0 | 2 | 0.00 | 0.00 | 0.00 | -| Basic6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic11 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic12 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic14 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic15 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic16 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic17 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic18 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic19 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic20 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic21 | 0 | 4 | ❌ | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | -| Basic22 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic23 | 2 | 3 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic24 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic25 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic26 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic27 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic28 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic29 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic30 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic31 | 0 | 3 | ❌ | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | -| Basic32 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic33 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic34 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic35 | 12 | 6 | ❌ | 0 | 6 | 0 | 0.00 | 0.00 | 0.00 | -| Basic36 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic37 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic38 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic39 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic40 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic41 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Basic42 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| TOTAL | 45 | 61 | 25/42 | 26 | 6 | 22 | 0.81 | 0.54 | 0.65 | - - -- **CollectionTest** - failed: 0, passed: 14 of 14 tests. `(100.00%)` +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | +|----------|--------|-----------|---------|----|----|----|-----------|--------|------| +| Basic1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic5 | 1 | 3 | ❌ | 0 | 0 | 2 | 0.00 | 0.00 | 0.00 | +| Basic6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic11 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic12 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic14 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic15 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic16 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic17 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic18 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic19 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic20 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic21 | 0 | 4 | ❌ | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | +| Basic22 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic23 | 2 | 3 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic24 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic25 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic26 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic27 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic28 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic29 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic30 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic31 | 0 | 3 | ❌ | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | +| Basic32 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic33 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic34 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic35 | 12 | 6 | ❌ | 0 | 6 | 0 | 0.00 | 0.00 | 0.00 | +| Basic36 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic37 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic38 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic39 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic40 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic41 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic42 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 45 | 61 | 25/42 | 26 | 6 | 22 | 0.81 | 0.54 | 0.65 | + + +- **CollectionTest** - failed: 1, passed: 13 of 14 tests. `(92.86%)` | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | |---------------|-------|----------|--------|----|----|----|-----------|--------|------| @@ -111,13 +114,13 @@ | Collections6 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Collections7 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Collections8 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Collections9 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections9 | 1 | 0 | FAIL | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | *FLAKY | Collections10 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Collections11 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Collections12 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Collections13 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Collections14 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| TOTAL | 15 | 15 | 14/14 | 15 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 15 | 14 | 13/14 | 14 | 1 | 0 | 0.93 | 1.00 | 0.96 | - **DataStructureTest** - failed: 1, passed: 5 of 6 tests. `(83.33%)` @@ -133,7 +136,7 @@ | TOTAL | 6 | 5 | 5/6 | 5 | 1 | 0 | 0.83 | 1.00 | 0.91 | -- **FactoryTest** - failed: 0, passed: 3 of 3 tests. `(100.00%)` +- **FactoryTest** - failed: 0, passed: 3 of 3 tests. `(100%)` | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | |------------|-------|----------|--------|----|----|----|-----------|--------|------| @@ -143,28 +146,28 @@ | TOTAL | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | -- **InterTest** - failed: 5, passed: 9 of 14 tests. `(64.29%)` +- **InterTest** - failed: 3, passed: 11 of 14 tests. `(78.57%)` -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | -|---------|-------|----------|--------|----|----|----|-----------|--------|------| -| Inter1 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter2 | 1 | 2 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Inter3 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter4 | 1 | 2 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Inter5 | 1 | 2 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Inter6 | 0 | 1 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Inter7 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter8 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter9 | 1 | 2 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Inter10 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter11 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter12 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter13 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter14 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| TOTAL | 13 | 18 | 9/14 | 9 | 0 | 5 | 1.00 | 0.64 | 0.78 | +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | +|---------|--------|-----------|---------|-----|-----|-----|------------|---------|------| +| Inter1 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter2 | 1 | 2 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter3 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter4 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter5 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter6 | 0 | 1 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter7 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter8 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter9 | 1 | 2 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter10 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter11 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter12 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter13 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter14 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 13 | 16 | 11/14 | 11 | 0 | 3 | 1.00 | 0.79 | 0.88 | -- **SessionTest** - failed: 0, passed: 3 of 3 tests. `(100.00%)` +- **SessionTest** - failed: 0, passed: 3 of 3 tests. `(100%)` | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | |----------|-------|----------|--------|----|----|----|-----------|--------|------| @@ -174,7 +177,7 @@ | TOTAL | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | -- **StrongUpdateTest** - failed: 4, passed: 1 of 5 tests. `(20.00%)` +- **StrongUpdateTest** - failed: 4, passed: 1 of 5 tests. `(20%)` | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | |----------------|-------|----------|--------|----|----|----|-----------|--------|------| @@ -183,4 +186,46 @@ | StrongUpdates3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | | StrongUpdates4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | StrongUpdates5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| TOTAL | 5 | 1 | 1/5 | 1 | 4 | 0 | 0.20 | 1.00 | 0.33 | \ No newline at end of file +| TOTAL | 5 | 1 | 1/5 | 1 | 4 | 0 | 0.20 | 1.00 | 0.33 | + + +> Extras + +- **PredTest** - failed: 3, passed: 6 of 9 tests. `(66.67%)` + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | +|--------|---------|-----------|---------|-----|-----|-----|------------|---------|------| +| Pred1 | 0 | 0 | PASS | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| Pred2 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred3 | 1 | 0 | FAIL | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred4 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred5 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred6 | 1 | 0 | FAIL | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred7 | 1 | 0 | FAIL | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred8 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred9 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | + + +- **ReflectionTest** - failed: 1, passed: 3 of 4 tests. `(75%)` + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | +|--------|--------|-----------|---------|-----|-----|-----|------------|---------|------| +| Refl1 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Refl2 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Refl3 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Refl4 | 0 | 1 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 3 | 4 | 3/4 | 3 | 0 | 1 | 1.00 | 0.75 | 0.86 | + + +- **SanitizersTest** - failed: 2, passed: 4 of 6 tests. `(66.67%)` + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | +|-------------|---------|------------|----------|------|------|------|------------|---------|------| +| Sanitizers1 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Sanitizers2 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Sanitizers3 | 1 | 0 | FAIL | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Sanitizers4 | 1 | 2 | FAIL | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Sanitizers5 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Sanitizers6 | 1 | 1 | PASS | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 6 | 6 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | \ No newline at end of file From 9bc25d630d63c5071f61f27f0fb90e52a6a67abb Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sat, 22 Nov 2025 13:24:03 -0300 Subject: [PATCH 25/46] update securibench section information --- README.md | 133 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 73 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 5a03c9af..7fdeb53a 100644 --- a/README.md +++ b/README.md @@ -203,45 +203,52 @@ This project integrates 2 well-known benchmarks. ### Securibench -This benchmark was integrated because it is also used in the [FlowDroid Project](https://github.com/secure-software-engineering/FlowDroid) and tests cases are in `src/test/java/securibench`. - -#### JSVFA metrics (old) - -> failed: 46, passed: 57 of 103 tests - (55.34%) - -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | -|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| -| aliasing | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | -| arrays | 0 | 9 | 1/10 | 0 | 0 | 9 | 0.00 | 0.00 | 0.00 | -| basic | 60 | 60 | 36/42 | 52 | 3 | 3 | 0.95 | 0.95 | 0.95 | -| collections | 3 | 15 | 1/14 | 1 | 1 | 13 | 0.50 | 0.07 | 0.12 | -| datastructures | 7 | 5 | 4/6 | 4 | 2 | 0 | 0.67 | 1.00 | 0.80 | -| factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | -| inter | 10 | 18 | 7/14 | 8 | 0 | 8 | 1.00 | 0.50 | 0.67 | -| session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | -| strong_updates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| TOTAL | 94 | 126 | 57/103 | 75 | 8 | 40 | 0.90 | 0.65 | 0.75 | - -To have detailed information about each group of tests run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md) (*computed in March 2023.*) - -#### JSVFA 2.0 metrics (v0.3.4) - -> failed: 38, passed: 65 of 103 tests - (63.11%) - -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | -|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| -| Aliasing | 4 | 12 | 1/6 | 1 | 1 | 9 | 0.50 | 0.10 | 0.17 | -| Arrays | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | -| Basic | 59 | 60 | 37/42 | 53 | 2 | 3 | 0.96 | 0.95 | 0.95 | -| Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | -| Datastructures | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | -| Factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | -| Inter | 12 | 18 | 8/14 | 9 | 0 | 6 | 1.00 | 0.60 | 0.75 | -| Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | -| StrongUpdates | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | -| TOTAL | 106 | 126 | 65/103 | 80 | 12 | 32 | 0.87 | 0.71 | 0.78 | - -To have detailed information about each group of tests run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.4.md) (*computed in September 2025.*) +This benchmark was integrated because it is also used in the [FlowDroid Project](https://github.com/secure-software-engineering/FlowDroid) and this integration +is implemented in `securibench` module. + +#### Old metrics (v0.3.0) + +> failed: 59, passed: 63 of 122 tests - (51.63%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | +|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| +| aliasing | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | 33.33 | +| arrays | 0 | 9 | 1/10 | 0 | 0 | 9 | 0.00 | 0.00 | 0.00 | 10 | +| basic | 60 | 60 | 36/42 | 52 | 3 | 3 | 0.95 | 0.95 | 0.95 | 85.71 | +| collections | 3 | 15 | 1/14 | 1 | 1 | 13 | 0.50 | 0.07 | 0.12 | 7.14 | +| datastructures | 7 | 5 | 4/6 | 4 | 2 | 0 | 0.67 | 1.00 | 0.80 | 66.67 | +| factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | 66.67 | +| inter | 10 | 18 | 7/14 | 8 | 0 | 8 | 1.00 | 0.50 | 0.67 | 50 | +| session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | 0 | +| strong_updates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | 80 | +| Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | +| Reflection | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | 0% | +| Sanitizers | 0 | 4 | 0/6 | 0 | 0 | 6 | 0.00 | 0.00 | 0.00 | 0% | +| TOTAL | 102 | 139 | 63/122 | 80 | 11 | 50 | 0.88 | 0.62 | 0.72 | 51.64 | + +To have detailed information about each test category run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md) (*computed in June 2023.*) + +#### New metrics (v0.6.1) + +> failed: 47, passed: 75 of 122 tests - (61.48%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | +|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| +| Aliasing | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | 33.33% | +| Arrays | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | 50% | +| Basic | 57 | 60 | 38/42 | 55 | 1 | 4 | 0.98 | 0.93 | 0.95 | 90.48% | +| Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | 35.71% | +| Datastructures | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | 66.67% | +| Factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | 66.67% | +| Inter | 12 | 18 | 8/14 | 9 | 0 | 6 | 1.00 | 0.60 | 0.75 | 57.14% | +| Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | 0% | +| StrongUpdates | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | 60% | +| Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | +| Reflection | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | 0% | +| Sanitizers | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | 33.33% | +| TOTAL | 120 | 141 | 75/122 | 95 | 14 | 35 | 0.87 | 0.73 | 0.79 | 61.48% | + +To have detailed information about each test category run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md) (*computed in November 2025.*) #### FLOWDROID @@ -258,37 +265,43 @@ To have detailed information about each group of tests run, [see here.](modules/ | Inter | 15 | 16 | 13/14 | 15 | 0 | 1 | 1.00 | 0.94 | 0.97 | | Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | | StrongUpdates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Pred | - | - | -/- | - | - | - | - | - | - | ** NO EXECUTED +| Reflection | - | - | -/- | - | - | - | - | - | - | ** NO EXECUTED +| Sanitizers | - | - | -/- | - | - | - | - | - | - | ** NO EXECUTED | TOTAL | 98 | 126 | 67/103 | 77 | 9 | 37 | 0.90 | 0.68 | 0.77 | To have detailed information about each group of tests run, [see here.](modules/securibench/src/docs-metrics/flowdroid/flowdroid-metrics.md) #### JOANA -> failed: 32, passed: 71 of 103 tests. `(68.93%)` - -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | -|----------------|-------|----------|--------|----|----|----|-----------|--------|------| -| Aliasing | 6 | 11 | 2/6 | 2 | 2 | 7 | 0.50 | 0.22 | 0.31 | -| Arrays | 10 | 9 | 9/10 | 9 | 1 | 0 | 0.90 | 1.00 | 0.95 | -| Basic | 45 | 61 | 25/42 | 26 | 6 | 22 | 0.81 | 0.54 | 0.65 | -| Collections | 15 | 15 | 14/14 | 15 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Datastructures | 6 | 5 | 5/6 | 5 | 1 | 0 | 0.83 | 1.00 | 0.91 | -| Factories | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter | 13 | 18 | 9/14 | 9 | 0 | 5 | 1.00 | 0.64 | 0.78 | -| Session | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| StrongUpdates | 5 | 1 | 1/5 | 1 | 4 | 0 | 0.20 | 1.00 | 0.33 | -| TOTAL | 106 | 126 | 71/103 | 73 | 14 | 34 | 0.84 | 0.68 | 0.75 | +> failed: 37, passed: 85 of 122 tests. `(69.67%)` + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | Pass rate | +|----------------|-------|----------|--------|----|----|----|-----------|--------|-------|-----------| +| Aliasing | 6 | 11 | 2/6 | 2 | 2 | 7 | 0.50 | 0.22 | 0.31 | 33.33% | +| Arrays | 10 | 9 | 9/10 | 9 | 1 | 0 | 0.90 | 1.00 | 0.95 | 90% | +| Basic | 45 | 61 | 25/42 | 26 | 6 | 22 | 0.81 | 0.54 | 0.65 | 59.52% | +| Collections | 15 | 14 | 13/14 | 14 | 1 | 0 | 0.93 | 1.00 | 0.96 | 92.86% | +| Datastructures | 6 | 5 | 5/6 | 5 | 1 | 0 | 0.83 | 1.00 | 0.91 | 83.33% | +| Factories | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | 100% | +| Inter | 13 | 16 | 11/14 | 11 | 0 | 3 | 1.00 | 0.79 | 0.88 | 78.57% | +| Session | 3 | 3 | 3/3 | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | 100% | +| StrongUpdates | 5 | 1 | 1/5 | 1 | 4 | 0 | 0.20 | 1.00 | 0.33 | 20% | +| Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | +| Reflection | 3 | 4 | 3/4 | 3 | 0 | 1 | 1.00 | 0.75 | 0.86 | 75% | +| Sanitizer | 6 | 6 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | 66.67% | +| TOTAL | 123 | 138 | 85/122 | 86 | 19 | 34 | 0.82 | 0.72 | 0.77 | 69.67% | To have detailed information about each group of tests run, [see here.](modules/securibench/src/docs-metrics/joana/joana-metrics.md) -#### METRICS SUMMARY +### METRICS SUMMARY -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | -|:---------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|----------:| -| JSVFA | 94 | 126 | 57/103 | 75 | 8 | 40 | 0.90 | 0.65 | 0.75 | 55.34% | -| JSVFA 2.0 | 106 | 126 | 65/103 | 80 | 12 | 32 | 0.87 | 0.71 | 0.78 | 63.11% | -| Flowdroid | 98 | 126 | 67/103 | 77 | 9 | 37 | 0.90 | 0.68 | 0.77 | 65.05% | -| Joana | 106 | 126 | 71/103 | 73 | 14 | 34 | 0.84 | 0.68 | 0.75 | 68.93% | +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | +|:------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|----------:| +| JSVFA v0.3.0 | 102 | 139 | 63/122 | 80 | 11 | 50 | 0.88 | 0.62 | 0.72 | 51.64% | +| JSVFA v0.6.1 | 120 | 141 | 75/122 | 95 | 14 | 35 | 0.87 | 0.73 | 0.79 | 61.48% | +| Flowdroid | 98 | 126 | 67/103 | 77 | 9 | 37 | 0.90 | 0.68 | 0.77 | 65.05% | +| Joana | 123 | 138 | 85/122 | 86 | 19 | 34 | 0.82 | 0.72 | 0.77 | 69.67% | ### TAINTBENCH: From 9d7d7cddbf9e51462596618c7d320f49efd49671 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sat, 22 Nov 2025 15:01:08 -0300 Subject: [PATCH 26/46] update tainbench section information --- README.md | 88 +++++++++++++++++-------------------------------------- 1 file changed, 27 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 7fdeb53a..c6e81706 100644 --- a/README.md +++ b/README.md @@ -206,6 +206,17 @@ This project integrates 2 well-known benchmarks. This benchmark was integrated because it is also used in the [FlowDroid Project](https://github.com/secure-software-engineering/FlowDroid) and this integration is implemented in `securibench` module. +The result are presented in a table that contains the following information. + +- **Expected:** The amount of taint flows presented by TAINTBENCH +- **Actual:** The amount of taint flows detected by JSVFA +- **Status:** If the test PASS OR FAIL +- **TP:** True Positive +- **FP:** False Positive +- **Precision:** TP/(TP + FP) +- **Recall:** TP/P +- **F-score:** (2 x Precision x Recall)/(Precision + Recall) + #### Old metrics (v0.3.0) > failed: 59, passed: 63 of 122 tests - (51.63%) @@ -311,8 +322,11 @@ and it is introduced by [Paper: TaintBench: Automatic real-world malware benchma which in its result section presents six experiments to answer one of its RQ: *How effective are taint analysis tools on TaintBench compared to DroidBench*, where FLOWDROID and AMANDROID as the chosen tools. -In the next sections, we will focus in **Experiment 2** and **Experiment 3** and use our tool (JSVFA) to reproduce them. -After that, we will compare the already computed results for FLOWDROID to our results. +In the next sections, we will focus in **Experiment 2** and **Experiment 3** and use the latest version of JSVFA to reproduce them. +Both experiments compare the matches between the set of leaks reported by each APK execution TaintBench paper, referred to as `expected`, +against the number of leaks identified by JSVFA, which is termed `actual`. A `match` is when the source and sink are the same in both the actual and expected sets. +We have created a file `taintbench.properties` to set the configurations in module `taintbench`. + #### Environment Setup @@ -357,19 +371,6 @@ ANDROID_SDK=/path/to/android/sdk TAINT_BENCH=/path/to/taintbench sbt testRoidsec ANDROID_SDK=/path/to/android/sdk TAINT_BENCH=/path/to/taintbench sbt "testOnly br.unb.cic.android.RoidsecTest" ``` -- The result for each APK tested using JSVFA are presented in a table that contains the following information. - - **Expected:** The amount of taint flows presented by TAINTBENCH - - **Actual:** The amount of taint flows detected by JSVFA - - **Status:** If the test PASS OR FAIL - - **TP:** True Positive - - **FP:** False Positive - - **Precision:** TP/(TP + FP) - - **Recall:** TP/P - - **F-score:** (2 x Precision x Recall)/(Precision + Recall) -- We have created a file `taintbench.properties` in `src/test/resources` to set the configurations - -**Disclaimer**: Although TAINTBENCH contains 203 expected [taint flows](https://taintbench.github.io/taintbenchSuite/), -we have decided to use only 186 expected cases because the mentioned paper, uses as a reference, works with those amounts. #### EXPERIMENT I @@ -377,27 +378,12 @@ This case emulates **Experiment 2 - TB2** that states: >All tools are configured with sources and sinks defined in benchmark suite. -The mentioned sources and sinks can be found in [TB_SourcesAndSinks](https://github.com/TaintBench/TaintBench/blob/main/TB_SourcesAndSinks.txt), -and we have stored them in `src/test/scala/br/unb/cic/android/TaintBenchSpec.scala`. - -As a result, we got `36, passed: 3 of 39 tests` and comparing to FLOWDROID we computed the next metrics: -~~a better `precision(0.82)`, the same `recall(0.22)` value and a slightly better `F-score(0.35)`.~~ - -- JSVFA metrics, to have detailed information about each group of tests run, [see here.](modules/taintbench/src/docs-metrics/taintbench-experiment-I.md) - -| Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | -|:-----:|:--------:|:------:|:--:|:--:|:---:|:---------:|:------:|:-------:|:---------:| -| 60 | 203 | 3/39 | 4 | 12 | 155 | 0.25 | 0.03 | 0.05 | 7.69% | - -- FLOWDROID metrics from [Paper](https://doi.org/10.1007/s10664-021-10013-5) - -| Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | -|:-----:|:--------:|:------:|:--:|:--:|:--:|:---------:|:------:|:-------:|:---------:| -| 55 | 186 | ? | 41 | 14 | ? | 0.75 | 0.22 | 0.34 | ? | - -##### Observation -- From the 37 failing tests, 28 of them reported zero flows. +The mentioned sources and sinks can be found in [TB_SourcesAndSinks](https://github.com/TaintBench/TaintBench/blob/main/TB_SourcesAndSinks.txt). +As a result, it finds fewer leaks than the expected, and, it gets only 10 matches. +| Actual Findings | Expected Findings | Matches | +|:---------------:|:-----------------:|:-------:| +| 86 | 216 | 10 | #### EXPERIMENT II @@ -407,29 +393,16 @@ This case emulates **Experiment 3 - TB3** that configures: configure all tools. Each tool analyzes each benchmark app with the associated list of sources and sinks -The mentioned lists can be found in https://taintbench.github.io/taintbenchSuite/, and we have stored them by individual -files in `src/test/scala/br/unb/cic/android/specs`. - -As a result, we got `failed: 35, passed: 4 of 39 tests` and comparing to FLOWDROID -~~awe detect a several better amount of `TP(135)` but also a several amount of `FP(318)` and about metric, we got a significant less `precision(0.30)` due to the high amount of FP; however, -a good `recall(0.73)` value and a better `F-score(0.42)`.~~ - -- JSVFA metrics, to have detailed information about each group of tests run, [see here.](modules/taintbench/src/docs-metrics/taintbench-experiment-II.md) - -| Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | -|:-----:|:--------:|:------:|:--:|:---:|:--:|:---------:|:------:|:-------:|:---------:| -| 620 | 203 | 4/39 | 29 | 463 | 46 | 0.06 | 0.39 | 0.10 | 10.26% | +The mentioned lists can be found in [TB_SourcesAndSinks](https://taintbench.github.io/taintbenchSuite). +As a result, it finds more leaks than the expected, however, it still gets only 10 matches as in the last experiment. -- FLOWDROID metrics from Paper https://doi.org/10.1007/s10664-021-10013-5 +| Actual Findings | Expected Findings | Matches | +|:---------------:|:-----------------:|:-------:| +| 655 | 216 | 10 | -| Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | -|:-----:|:--------:|:------:|:--:|:--:|:--:|:---------:|:------:|:-------:|:---------:| -| 57 | 186 | ? | 43 | 14 | ? | 0.75 | 0.23 | 0.35 | ? | - -##### Observation -- We got a big amount of FP. ## Tasks + ### WIP - [ ] Add set up project documentation. - [ ] Fix bugs for Securibench in folders @@ -439,13 +412,6 @@ a good `recall(0.73)` value and a better `F-score(0.42)`.~~ - [ ] Strong Update - [ ] Aliasing -### TO-DO -- - -### DONE -- [X] Create Git Action flow. -- [X] Integrate Taintbench. - ## License This project is licensed under the MIT License. From 7a1b2e98e59ce6f7e0089c430a1c57fc147bfb76 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sat, 22 Nov 2025 15:10:36 -0300 Subject: [PATCH 27/46] update flowdroid metrics --- .../flowdroid/flowdroid-metrics.md | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/modules/securibench/src/docs-metrics/flowdroid/flowdroid-metrics.md b/modules/securibench/src/docs-metrics/flowdroid/flowdroid-metrics.md index 258e3280..769cd2e7 100644 --- a/modules/securibench/src/docs-metrics/flowdroid/flowdroid-metrics.md +++ b/modules/securibench/src/docs-metrics/flowdroid/flowdroid-metrics.md @@ -1,21 +1,24 @@ #### FLOWDROID metrics -### SUMMARY +### SUMMARY (*computed in November 2025.*) > failed: 36, passed: 67 of 103 tests. `(65.05%)` -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | -|----------------|-------|----------|--------|----|----|----|-----------|--------|------| -| Aliasing | 11 | 11 | 4/6 | 9 | 1 | 1 | 0.90 | 0.90 | 0.90 | -| Arrays | 14 | 9 | 6/10 | 6 | 5 | 0 | 0.55 | 1.00 | 0.71 | -| Basic | 38 | 61 | 26/42 | 33 | 1 | 24 | 0.97 | 0.58 | 0.73 | -| Collections | 14 | 15 | 11/14 | 12 | 1 | 2 | 0.92 | 0.86 | 0.89 | -| Datastructures | 5 | 5 | 4/6 | 3 | 1 | 1 | 0.75 | 0.75 | 0.75 | -| Factories | 1 | 3 | 1/3 | 1 | 0 | 2 | 1.00 | 0.33 | 0.50 | -| Inter | 15 | 18 | 11/14 | 13 | 0 | 3 | 1.00 | 0.81 | 0.90 | -| Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | -| StrongUpdates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| TOTAL | 98 | 126 | 67/103 | 77 | 9 | 37 | 0.90 | 0.68 | 0.77 | +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | Pass Rate | +|----------------|-------|----------|--------|----|----|----|-----------|--------|------|-----------| +| Aliasing | 11 | 11 | 4/6 | 9 | 1 | 1 | 0.90 | 0.90 | 0.90 | 66.67% | +| Arrays | 14 | 9 | 6/10 | 6 | 5 | 0 | 0.55 | 1.00 | 0.71 | 60% | +| Basic | 38 | 61 | 26/42 | 33 | 1 | 24 | 0.97 | 0.58 | 0.73 | 61.90% | +| Collections | 14 | 15 | 11/14 | 12 | 1 | 2 | 0.92 | 0.86 | 0.89 | 78.57% | +| Datastructures | 5 | 5 | 4/6 | 3 | 1 | 1 | 0.75 | 0.75 | 0.75 | 66.67% | +| Factories | 1 | 3 | 1/3 | 1 | 0 | 2 | 1.00 | 0.33 | 0.50 | 33.33% | +| Inter | 15 | 18 | 11/14 | 13 | 0 | 3 | 1.00 | 0.81 | 0.90 | 78.57% | +| Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | 0% | +| StrongUpdates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | 80% | +| Pred | - | - | - | - | - | - | - | - | - | - | +| Reflection | - | - | - | - | - | - | - | - | - | - | +| Sanitizers | - | - | - | - | - | - | - | - | - | - | +| **TOTAL** | 98 | 126 | 67/103 | 77 | 9 | 37 | 0.90 | 0.68 | 0.77 | 65.05% | According to Flowdroid Paper (https://www.bodden.de/pubs/far+14flowdroid.pdf), they computed the next values. @@ -38,7 +41,7 @@ According to Flowdroid Paper (https://www.bodden.de/pubs/far+14flowdroid.pdf), t | StrongUpdate | 0/0 | 0 | | **TOTAL** | 117/121 | 9 | -### Details +### DETAILS - ✅ : PASSED; ❌ : FAIL @@ -55,7 +58,7 @@ According to Flowdroid Paper (https://www.bodden.de/pubs/far+14flowdroid.pdf), t | TOTAL | 11 | 11 | 4/6 | 9 | 1 | 1 | 0.90 | 0.90 | 0.90 | -- **ArraysTest** - failed: 4, passed: 6 of 10 tests. `(60.00%)` +- **ArraysTest** - failed: 4, passed: 6 of 10 tests. `(60%)` | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | |----------|-------|----------|--------|----|----|----|-----------|--------|------| @@ -197,7 +200,7 @@ According to Flowdroid Paper (https://www.bodden.de/pubs/far+14flowdroid.pdf), t | TOTAL | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | -- **StrongUpdateTest** - failed: 1, passed: 4 of 5 tests. `(80.00%)` +- **StrongUpdateTest** - failed: 1, passed: 4 of 5 tests. `(80%)` | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | |----------------|-------|----------|--------|----|----|----|-----------|--------|------| From 177a09c7c37a1f082d46a313dac98229597cb24b Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sat, 22 Nov 2025 15:11:59 -0300 Subject: [PATCH 28/46] update securibench section information --- README.md | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index c6e81706..6ca9f67e 100644 --- a/README.md +++ b/README.md @@ -265,21 +265,21 @@ To have detailed information about each test category run, [see here.](modules/s - failed: 36, passed: 67 of 103 tests. `(65.05%)` -| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | -|----------------|-------|----------|--------|----|----|----|-----------|--------|------| -| Aliasing | 11 | 11 | 4/6 | 9 | 1 | 1 | 0.90 | 0.90 | 0.90 | -| Arrays | 14 | 9 | 6/10 | 6 | 5 | 0 | 0.55 | 1.00 | 0.71 | -| Basic | 38 | 61 | 26/42 | 33 | 1 | 24 | 0.97 | 0.58 | 0.73 | -| Collections | 14 | 14 | 12/14 | 12 | 1 | 1 | 0.92 | 0.92 | 0.92 | -| Datastructures | 5 | 5 | 4/6 | 3 | 1 | 1 | 0.75 | 0.75 | 0.75 | -| Factories | 1 | 3 | 1/3 | 1 | 0 | 2 | 1.00 | 0.33 | 0.50 | -| Inter | 15 | 16 | 13/14 | 15 | 0 | 1 | 1.00 | 0.94 | 0.97 | -| Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | -| StrongUpdates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Pred | - | - | -/- | - | - | - | - | - | - | ** NO EXECUTED -| Reflection | - | - | -/- | - | - | - | - | - | - | ** NO EXECUTED -| Sanitizers | - | - | -/- | - | - | - | - | - | - | ** NO EXECUTED -| TOTAL | 98 | 126 | 67/103 | 77 | 9 | 37 | 0.90 | 0.68 | 0.77 | +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F1 | Pass Rate | +|----------------|-------|----------|--------|----|----|----|-----------|--------|------|-----------| +| Aliasing | 11 | 11 | 4/6 | 9 | 1 | 1 | 0.90 | 0.90 | 0.90 | 66.67% | +| Arrays | 14 | 9 | 6/10 | 6 | 5 | 0 | 0.55 | 1.00 | 0.71 | 60% | +| Basic | 38 | 61 | 26/42 | 33 | 1 | 24 | 0.97 | 0.58 | 0.73 | 61.90% | +| Collections | 14 | 15 | 11/14 | 12 | 1 | 2 | 0.92 | 0.86 | 0.89 | 78.57% | +| Datastructures | 5 | 5 | 4/6 | 3 | 1 | 1 | 0.75 | 0.75 | 0.75 | 66.67% | +| Factories | 1 | 3 | 1/3 | 1 | 0 | 2 | 1.00 | 0.33 | 0.50 | 33.33% | +| Inter | 15 | 18 | 11/14 | 13 | 0 | 3 | 1.00 | 0.81 | 0.90 | 78.57% | +| Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | 0% | +| StrongUpdates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | 80% | +| Pred | - | - | - | - | - | - | - | - | - | - | * NO EXECUTED +| Reflection | - | - | - | - | - | - | - | - | - | - | * NO EXECUTED +| Sanitizers | - | - | - | - | - | - | - | - | - | - | * NO EXECUTED +| **TOTAL** | 98 | 126 | 67/103 | 77 | 9 | 37 | 0.90 | 0.68 | 0.77 | 65.05% | To have detailed information about each group of tests run, [see here.](modules/securibench/src/docs-metrics/flowdroid/flowdroid-metrics.md) From 66644a6c5067e4b1f5e7f35df4f5e91f695958c7 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sun, 23 Nov 2025 21:08:55 -0300 Subject: [PATCH 29/46] update readme --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6ca9f67e..16275c5a 100644 --- a/README.md +++ b/README.md @@ -379,7 +379,8 @@ This case emulates **Experiment 2 - TB2** that states: >All tools are configured with sources and sinks defined in benchmark suite. The mentioned sources and sinks can be found in [TB_SourcesAndSinks](https://github.com/TaintBench/TaintBench/blob/main/TB_SourcesAndSinks.txt). -As a result, it finds fewer leaks than the expected, and, it gets only 10 matches. +As a result, it finds fewer leaks than the expected, and, it gets only 10 matches, +which means, it has found 76 new leakages. | Actual Findings | Expected Findings | Matches | |:---------------:|:-----------------:|:-------:| @@ -395,6 +396,7 @@ of sources and sinks The mentioned lists can be found in [TB_SourcesAndSinks](https://taintbench.github.io/taintbenchSuite). As a result, it finds more leaks than the expected, however, it still gets only 10 matches as in the last experiment. +which means, it has found 645 new leakages. | Actual Findings | Expected Findings | Matches | |:---------------:|:-----------------:|:-------:| From 39de5cbcc9a5913c7301161df7829bed2e434e73 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Sun, 23 Nov 2025 21:09:07 -0300 Subject: [PATCH 30/46] wip: info about issues --- .../jsvfa/jsvfa-metrics-v0.6.1.md | 38 +++++++++++-------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 3c75d88d..6f754223 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -31,10 +31,10 @@ | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:---------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * wrong counting -| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | ** array indexes -| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * wrong counting -| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [i] +| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [ii] +| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [i] +| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | Aliasing6 | 7 | 7 | ✅ | 7 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | @@ -44,15 +44,15 @@ | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Arrays1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Arrays2 | 3 | 1 | ❌ | 0 | 2 | 0 | 0.00 | 0.00 | 0.00 | +| Arrays2 | 3 | 1 | ❌ | 0 | 2 | 0 | 0.00 | 0.00 | 0.00 | * issue [ii] | Arrays3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Arrays4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Arrays5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Arrays4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [ii] | Arrays6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Arrays7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Arrays8 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Arrays9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Arrays10 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays8 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [ii] +| Arrays9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [ii] +| Arrays10 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [ii] | TOTAL | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | @@ -91,17 +91,17 @@ | Basic28 | 2 | 2 | ✅ | 0 | 0 | 2 | 1.00 | 1.00 | 1.00 | | Basic29 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic30 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic31 | 0 | 2 | ❌ | 0 | 0 | 2 | 0.00 | 0.00 | 0.00 | +| Basic31 | 0 | 2 | ❌ | 0 | 0 | 2 | 0.00 | 0.00 | 0.00 | * issue [iii] | Basic32 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic33 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic34 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic35 | 6 | 6 | ✅ | 6 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic36 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic36 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] | Basic37 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic38 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Basic38 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [iii] | Basic39 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Basic41 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Basic42 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic42 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | TOTAL | 57 | 60 | 38/42 | 55 | 1 | 4 | 0.98 | 0.93 | 0.95 | @@ -233,4 +233,12 @@ These tests are not executed by Flowdroid | Sanitizers4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | | Sanitizers5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | | Sanitizers6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| TOTAL | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | \ No newline at end of file +| TOTAL | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | + + +### Common issues + +- [i] Wrong counting: Some tests from the Securibench benchmark are incorrectly labeled, leading to wrong expected values. +- [ii] Array Indexes: The actual implementation is unable to recognize tainted in specific indexes from an array. Currently, it marks all the array as tainted. +- [iii] Support Class Missing: Some tests use methods from securibench that are not mocked. +- [iv] The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs. From 409e562cdacf7d1a8483873fdbd9de315d08fa80 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 14:10:28 -0300 Subject: [PATCH 31/46] map error in collections category --- .../jsvfa/jsvfa-metrics-v0.6.1.md | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 6f754223..8ee30063 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -111,17 +111,17 @@ |:-------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Collections1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Collections2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Collections3 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections3 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] | Collections4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Collections5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Collections6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Collections7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Collections8 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Collections9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Collections10 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Collections5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Collections6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Collections7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Collections8 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Collections9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Collections10 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [iv] | Collections11 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Collections12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Collections13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Collections13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] | Collections14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | @@ -241,4 +241,7 @@ These tests are not executed by Flowdroid - [i] Wrong counting: Some tests from the Securibench benchmark are incorrectly labeled, leading to wrong expected values. - [ii] Array Indexes: The actual implementation is unable to recognize tainted in specific indexes from an array. Currently, it marks all the array as tainted. - [iii] Support Class Missing: Some tests use methods from securibench that are not mocked. -- [iv] The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs. +- [iv] Missing Context: The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs such as: + - Nested structures as HashMap, LinkedList, and others, + - Loop statement as "for" or "while", + - Parameters passed in the constructor, \ No newline at end of file From 8e0a3ae459edf6a50095ddcafa15fb406ca0c958 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 14:11:27 -0300 Subject: [PATCH 32/46] Datastructures --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 8ee30063..4e71850a 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -133,8 +133,8 @@ | Datastructures1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Datastructures2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Datastructures3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Datastructures4 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Datastructures5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Datastructures4 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [iv] +| Datastructures5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] | Datastructures6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | From 40375a9407eeb80e9fa3ed3d66014c5de041276e Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 14:12:50 -0300 Subject: [PATCH 33/46] map error in factories category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 4e71850a..024d3a04 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -145,7 +145,7 @@ |:----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Factories1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Factories2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Factories3 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Factories3 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [iv] | TOTAL | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | @@ -242,6 +242,6 @@ These tests are not executed by Flowdroid - [ii] Array Indexes: The actual implementation is unable to recognize tainted in specific indexes from an array. Currently, it marks all the array as tainted. - [iii] Support Class Missing: Some tests use methods from securibench that are not mocked. - [iv] Missing Context: The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs such as: - - Nested structures as HashMap, LinkedList, and others, - - Loop statement as "for" or "while", - - Parameters passed in the constructor, \ No newline at end of file + [a] Nested structures as HashMap, LinkedList, and others, + [b] Loop statement as "for" or "while", + [c] Parameters passed in the constructor, \ No newline at end of file From 66ee8d82f402633854b11e84c6237ac05f91183c Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 14:36:07 -0300 Subject: [PATCH 34/46] map error in inter category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 024d3a04..1b64f42b 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -156,15 +156,15 @@ | Inter1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter2 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Inter6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [i] +| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Inter6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] | Inter7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] | Inter10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter11 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | *flaky -| Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] | Inter13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 12 | 18 | 8/14 | 9 | 0 | 6 | 1.00 | 0.60 | 0.75 | @@ -244,4 +244,5 @@ These tests are not executed by Flowdroid - [iv] Missing Context: The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs such as: [a] Nested structures as HashMap, LinkedList, and others, [b] Loop statement as "for" or "while", - [c] Parameters passed in the constructor, \ No newline at end of file + [c] Parameters passed in the constructor, +- [v] Reflection: The actual implementation does not handle reflection feature. \ No newline at end of file From 0e080b0674f7713208436bdbdf490624965128ca Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 14:37:32 -0300 Subject: [PATCH 35/46] map error in session category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 1b64f42b..59987454 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -174,9 +174,9 @@ | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| -| Session1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Session2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Session3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Session1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iii] +| Session2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iii] +| Session3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iii] | TOTAL | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | From ec5e9b05da63ccb1d26c465d8fc44c06dc55f202 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 14:48:20 -0300 Subject: [PATCH 36/46] map error in strong_updates category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 59987454..14c4a7d1 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -186,9 +186,9 @@ |:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | StrongUpdates1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | | StrongUpdates2 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | -| StrongUpdates3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vi] | StrongUpdates4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| StrongUpdates5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vi] | TOTAL | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | @@ -245,4 +245,5 @@ These tests are not executed by Flowdroid [a] Nested structures as HashMap, LinkedList, and others, [b] Loop statement as "for" or "while", [c] Parameters passed in the constructor, -- [v] Reflection: The actual implementation does not handle reflection feature. \ No newline at end of file +- [v] Reflection: The actual implementation does not handle reflection feature. +- [vi] Global variables references: . \ No newline at end of file From 4cd3d5309a0431c5f6c65491c819cf10adda77d7 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 14:54:56 -0300 Subject: [PATCH 37/46] map error in pred category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 14c4a7d1..082ad18e 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -202,11 +202,11 @@ These tests are not executed by Flowdroid |:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Pred1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | | Pred2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Pred3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vii] | Pred4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Pred5 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Pred6 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | -| Pred7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred5 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred6 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vii] +| Pred7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vii] | Pred8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Pred9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | @@ -246,4 +246,5 @@ These tests are not executed by Flowdroid [b] Loop statement as "for" or "while", [c] Parameters passed in the constructor, - [v] Reflection: The actual implementation does not handle reflection feature. -- [vi] Global variables references: . \ No newline at end of file +- [vi] Global variables references: . +- [vii] Path for conditional: The logic always evaluate two paths for a conditional, regardless the condition being always set as True or False \ No newline at end of file From 23f467c10b9f8a6a0ce2c4cd57a5c752b6a83844 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 14:55:52 -0300 Subject: [PATCH 38/46] map error in reflection category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 082ad18e..a14a0787 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -216,10 +216,10 @@ These tests are not executed by Flowdroid | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| -| Refl1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Refl2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Refl3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Refl4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Refl1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] +| Refl2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] +| Refl3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] +| Refl4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] | TOTAL | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | From 0451ca347f2340836220085b6abfad9c5c81914a Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 15:03:56 -0300 Subject: [PATCH 39/46] map error in sanitizers category --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index a14a0787..ba23e356 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -228,11 +228,11 @@ These tests are not executed by Flowdroid | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:-----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Sanitizers1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Sanitizers2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Sanitizers2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [viii] | Sanitizers3 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | -| Sanitizers4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Sanitizers5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | -| Sanitizers6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Sanitizers4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [viii] +| Sanitizers5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iii] +| Sanitizers6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [viii] | TOTAL | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | @@ -247,4 +247,5 @@ These tests are not executed by Flowdroid [c] Parameters passed in the constructor, - [v] Reflection: The actual implementation does not handle reflection feature. - [vi] Global variables references: . -- [vii] Path for conditional: The logic always evaluate two paths for a conditional, regardless the condition being always set as True or False \ No newline at end of file +- [vii] Path for conditional: The current logic always evaluate two paths for a conditional, regardless the condition being always set as True or False +- [viii] Sanitizer method: The actual implementation does not handle intermediary method that Sanitizer uses. \ No newline at end of file From 99f2fc62daad1cad29595f2fab509f4c2962c3e3 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 15:23:48 -0300 Subject: [PATCH 40/46] typo --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index ba23e356..b44b0988 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -244,8 +244,8 @@ These tests are not executed by Flowdroid - [iv] Missing Context: The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs such as: [a] Nested structures as HashMap, LinkedList, and others, [b] Loop statement as "for" or "while", - [c] Parameters passed in the constructor, -- [v] Reflection: The actual implementation does not handle reflection feature. -- [vi] Global variables references: . -- [vii] Path for conditional: The current logic always evaluate two paths for a conditional, regardless the condition being always set as True or False + [c] Parameters passed in the constructor. +- [v] Reflection: The actual implementation does not handle reflection feature, +- [vi] Global variables references: There are edges cases not mapped about how to handle the "definition" of global variables, +- [vii] Path for conditional: The current logic always evaluate two paths for a conditional, regardless the condition being always set as True or Fals, - [viii] Sanitizer method: The actual implementation does not handle intermediary method that Sanitizer uses. \ No newline at end of file From 60077b2d14d91dd5aac300c5772bf6d6f7ccd082 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 15:32:27 -0300 Subject: [PATCH 41/46] Common issues --- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index b44b0988..24ba0e3d 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -245,7 +245,7 @@ These tests are not executed by Flowdroid [a] Nested structures as HashMap, LinkedList, and others, [b] Loop statement as "for" or "while", [c] Parameters passed in the constructor. -- [v] Reflection: The actual implementation does not handle reflection feature, -- [vi] Global variables references: There are edges cases not mapped about how to handle the "definition" of global variables, -- [vii] Path for conditional: The current logic always evaluate two paths for a conditional, regardless the condition being always set as True or Fals, -- [viii] Sanitizer method: The actual implementation does not handle intermediary method that Sanitizer uses. \ No newline at end of file +- [v] Reflection: The current implementation does not address the reflection feature, +- [vi] Global variables references: There are unaddressed edge cases regarding the handling of the definition of global variables., +- [vii] Path for conditional: The current logic always evaluates two paths for a conditional, regardless of whether the condition is set to True or False, +- [viii] Sanitizer method: The current implementation fails to deal with the intermediary method utilized by the sanitizer. \ No newline at end of file From 1122faf66e17940937273acdf04c0f21eb05a49e Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 16:08:18 -0300 Subject: [PATCH 42/46] typo --- .../jsvfa/jsvfa-metrics-v0.6.1.md | 83 ++++++++----------- 1 file changed, 34 insertions(+), 49 deletions(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 24ba0e3d..8db5a198 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -111,17 +111,17 @@ |:-------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Collections1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Collections2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Collections3 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Collections3 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | Collections4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Collections5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] -| Collections6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] -| Collections7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] -| Collections8 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] -| Collections9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] -| Collections10 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [iv] +| Collections5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections8 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections10 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [iv] | Collections11 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Collections12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] -| Collections13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Collections12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | Collections14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | @@ -133,8 +133,8 @@ | Datastructures1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Datastructures2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Datastructures3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Datastructures4 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [iv] -| Datastructures5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Datastructures4 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Datastructures5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | Datastructures6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | @@ -145,7 +145,7 @@ |:----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Factories1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Factories2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Factories3 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [iv] +| Factories3 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [iv] | TOTAL | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | @@ -156,15 +156,15 @@ | Inter1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter2 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [i] -| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] -| Inter6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] +| Inter4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [i] +| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Inter6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] | Inter7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | Inter10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter11 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | *flaky -| Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iv] +| Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | Inter13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 12 | 18 | 8/14 | 9 | 0 | 6 | 1.00 | 0.60 | 0.75 | @@ -174,9 +174,9 @@ | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| -| Session1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iii] -| Session2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iii] -| Session3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iii] +| Session1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] +| Session2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] +| Session3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] | TOTAL | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | @@ -186,9 +186,9 @@ |:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | StrongUpdates1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | | StrongUpdates2 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | -| StrongUpdates3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vi] +| StrongUpdates3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vi] | StrongUpdates4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| StrongUpdates5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vi] +| StrongUpdates5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vi] | TOTAL | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | @@ -202,11 +202,11 @@ These tests are not executed by Flowdroid |:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Pred1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | | Pred2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Pred3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vii] +| Pred3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vii] | Pred4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Pred5 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Pred6 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vii] -| Pred7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | [vii] +| Pred6 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vii] +| Pred7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vii] | Pred8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Pred9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | TOTAL | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | @@ -216,10 +216,10 @@ These tests are not executed by Flowdroid | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| -| Refl1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] -| Refl2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] -| Refl3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] -| Refl4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [v] +| Refl1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] +| Refl2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] +| Refl3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] +| Refl4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] | TOTAL | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | @@ -228,24 +228,9 @@ These tests are not executed by Flowdroid | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | |:-----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| | Sanitizers1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Sanitizers2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [viii] +| Sanitizers2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [viii] | Sanitizers3 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | -| Sanitizers4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [viii] -| Sanitizers5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [iii] -| Sanitizers6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | [viii] -| TOTAL | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | - - -### Common issues - -- [i] Wrong counting: Some tests from the Securibench benchmark are incorrectly labeled, leading to wrong expected values. -- [ii] Array Indexes: The actual implementation is unable to recognize tainted in specific indexes from an array. Currently, it marks all the array as tainted. -- [iii] Support Class Missing: Some tests use methods from securibench that are not mocked. -- [iv] Missing Context: The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs such as: - [a] Nested structures as HashMap, LinkedList, and others, - [b] Loop statement as "for" or "while", - [c] Parameters passed in the constructor. -- [v] Reflection: The current implementation does not address the reflection feature, -- [vi] Global variables references: There are unaddressed edge cases regarding the handling of the definition of global variables., -- [vii] Path for conditional: The current logic always evaluates two paths for a conditional, regardless of whether the condition is set to True or False, -- [viii] Sanitizer method: The current implementation fails to deal with the intermediary method utilized by the sanitizer. \ No newline at end of file +| Sanitizers4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [viii] +| Sanitizers5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] +| Sanitizers6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [viii] +| TOTAL | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | \ No newline at end of file From 4f6adf8d99fccd922224791e266f01e05cb2812f Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 17:17:07 -0300 Subject: [PATCH 43/46] add issue ix --- .../securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 8db5a198..6cb6c713 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -163,7 +163,7 @@ | Inter8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | Inter10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | -| Inter11 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | *flaky +| Inter11 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [ix] | Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] | Inter13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | From ebf5edd2b4cc6a00e9dff6147a3e9cfcf99bb5e2 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 24 Nov 2025 17:17:22 -0300 Subject: [PATCH 44/46] Add section about common issues --- README.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/README.md b/README.md index 16275c5a..c638d7ea 100644 --- a/README.md +++ b/README.md @@ -261,6 +261,87 @@ To have detailed information about each test category run, [see here.](modules/s To have detailed information about each test category run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md) (*computed in November 2025.*) +##### Common issues +From the 47 tests, we have categorized nine (9) issues. + +[i] **Wrong counting**: Some tests from the Securibench benchmark are incorrectly labeled, leading to wrong expected values. +We have mapped three cases: `(6.38%)` +- Aliasing2 +- Aliasing4 +- Inter4 + +[ii] Array Indexes: The actual implementation is unable to recognize tainted in specific indexes from an array. Currently, it marks all the array as tainted. +We have mapped six cases: `(12.77%)` +- Aliasing3 +- Arrays2 +- Arrays5 +- Arrays8 +- Arrays9 +- Arrays10 + +[iii] Support Class Missing: Some tests use methods from securibench that are not mocked. +We have mapped seven cases: `(14.89%)` +- Basic31 +- Basic36 +- Basic38 +- Session1 +- Session2 +- Session3 +- Sanitizers5 + +[iv] Missing Context: The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs such as: + [a] Nested structures as HashMap, LinkedList, and others, + [b] Loop statement as "for" or "while", + [c] Parameters passed in the constructor. +We have mapped 17 cases: `(36.17%)` +- Aliasing5 +- Basic42 +- Collections3 +- Collections5 +- Collections6 +- Collections7 +- Collections8 +- Collections9 +- Collections10 +- Collections12 +- Collections13 +- Datastructures4 +- Datastructures5 +- Factories3 +- Inter5 +- Inter9 +- Inter12 + +[v] Reflection: The current implementation does not address the reflection feature, +We have mapped 5 cases: `(10.64%)` +- Inter6 +- Refl1 +- Refl2 +- Refl3 +- Refl4 + +[vi] Global variables references: There are unaddressed edge cases regarding the handling of the definition of global variables., +We have mapped two cases: `(4.26%)` +- StrongUpdates3 +- StrongUpdates5 + + +[vii] Path for conditional: The current logic always evaluates two paths for a conditional, regardless of whether the condition is set to True or False, +We have mapped three cases: `(6.38%)` +- Pred3 +- Pred6 +- Pred7 + +[viii] Sanitizer method: The current implementation fails to deal with the intermediary method utilized by the sanitizer. +We have mapped three cases: `(6.38%)` +- Sanitizers2 +- Sanitizers4 +- Sanitizers6 + +[ix] Flaky +We have mapped one cases: `(2.13%)` +- Inter11 + #### FLOWDROID - failed: 36, passed: 67 of 103 tests. `(65.05%)` From ae0a4b013e0b07a1e49487b30a8e6d09bcacb364 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Wed, 26 Nov 2025 12:52:11 -0300 Subject: [PATCH 45/46] update error classification --- README.md | 6 +++--- .../src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c638d7ea..555be2f0 100644 --- a/README.md +++ b/README.md @@ -265,10 +265,11 @@ To have detailed information about each test category run, [see here.](modules/s From the 47 tests, we have categorized nine (9) issues. [i] **Wrong counting**: Some tests from the Securibench benchmark are incorrectly labeled, leading to wrong expected values. -We have mapped three cases: `(6.38%)` +We have mapped four cases: `(8.51%)` - Aliasing2 - Aliasing4 - Inter4 +- Inter5 [ii] Array Indexes: The actual implementation is unable to recognize tainted in specific indexes from an array. Currently, it marks all the array as tainted. We have mapped six cases: `(12.77%)` @@ -293,7 +294,7 @@ We have mapped seven cases: `(14.89%)` [a] Nested structures as HashMap, LinkedList, and others, [b] Loop statement as "for" or "while", [c] Parameters passed in the constructor. -We have mapped 17 cases: `(36.17%)` +We have mapped 16 cases: `(34.04%)` - Aliasing5 - Basic42 - Collections3 @@ -308,7 +309,6 @@ We have mapped 17 cases: `(36.17%)` - Datastructures4 - Datastructures5 - Factories3 -- Inter5 - Inter9 - Inter12 diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md index 6cb6c713..1c0daa9a 100644 --- a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md @@ -157,7 +157,7 @@ | Inter2 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [i] -| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [i] | Inter6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] | Inter7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | | Inter8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | From 0c970ee433088db0cebf15fc7c04d0c6a4994db9 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Mon, 1 Dec 2025 21:17:47 -0300 Subject: [PATCH 46/46] v0.6.1 updates --- release_notes.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/release_notes.txt b/release_notes.txt index f8473165..9f2ff551 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -20,4 +20,8 @@ v0.6.0 - Organize metrics results, - Create module for scripts, - Set up github workflow to not use tokens, -- Create logic to generate reports. \ No newline at end of file +- Create logic to generate reports. + +v0.6.1 +- Update readme information, +- Compute new test categories: Preds, Reflections, and Sanitizers. \ No newline at end of file