diff --git a/README.md b/README.md index 27b5b01b..ebd020d4 100644 --- a/README.md +++ b/README.md @@ -239,15 +239,15 @@ The result are presented in a table that contains the following information. To have detailed information about each test category run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.3.0.md) (*computed in June 2023.*) -#### New metrics (v0.6.1) +#### New metrics (v0.6.2) -> failed: 47, passed: 75 of 122 tests - (61.48%) +> failed: 46, passed: 76 of 122 tests - (62.3%) | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | |:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| | Aliasing | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | 33.33% | | Arrays | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | 50% | -| Basic | 57 | 60 | 38/42 | 55 | 1 | 4 | 0.98 | 0.93 | 0.95 | 90.48% | +| Basic | 60 | 60 | 38/42 | 55 | 2 | 2 | 0.96 | 0.96 | 0.96 | 90.48% | | Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | 35.71% | | Datastructures | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | 66.67% | | Factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | 66.67% | @@ -257,9 +257,9 @@ To have detailed information about each test category run, [see here.](modules/s | Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | | Reflection | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | 0% | | Sanitizers | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | 33.33% | -| TOTAL | 120 | 141 | 75/122 | 95 | 14 | 35 | 0.87 | 0.73 | 0.79 | 61.48% | +| TOTAL | 124 | 141 | 76/122 | 96 | 15 | 32 | 0.86 | 0.75 | 0.80 | 62.3% | -To have detailed information about each test category run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md) (*computed in November 2025.*) +To have detailed information about each test category run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.2.md) (*computed in December 2025.*) #### Running Securibench Tests @@ -279,9 +279,10 @@ sbt "testOnly br.unb.cic.securibench.deprecated.SecuribenchTestSuite" From the 47 tests, we have categorized nine (9) issues. [i] **Wrong counting**: Some tests from the Securibench benchmark are incorrectly labeled, leading to wrong expected values. -We have mapped four cases: `(8.51%)` +We have mapped four cases: `(10.64%)` - Aliasing2 - Aliasing4 +- Basic31 - Inter4 - Inter5 @@ -295,21 +296,19 @@ We have mapped six cases: `(12.77%)` - Arrays10 [iii] Support Class Missing: Some tests use methods from securibench that are not mocked. -We have mapped seven cases: `(14.89%)` -- Basic31 -- Basic36 -- Basic38 +We have mapped seven cases: `(6.38%)` - Session1 - Session2 - Session3 -- Sanitizers5 [iv] Missing Context: The logic for handling context is not entirely flawless, resulting in certain edge cases that lead to bugs such as: [a] Nested structures as HashMap, LinkedList, and others, [b] Loop statement as "for" or "while", [c] Parameters passed in the constructor. -We have mapped 16 cases: `(34.04%)` +We have mapped 16 cases: `(38.3%)` - Aliasing5 +- Basic36 +- Basic38 - Basic42 - Collections3 - Collections5 @@ -347,9 +346,10 @@ We have mapped three cases: `(6.38%)` - Pred7 [viii] Sanitizer method: The current implementation fails to deal with the intermediary method utilized by the sanitizer. -We have mapped three cases: `(6.38%)` +We have mapped three cases: `(8.51%)` - Sanitizers2 - Sanitizers4 +- Sanitizers5 - Sanitizers6 [ix] Flaky @@ -405,7 +405,7 @@ To have detailed information about each group of tests run, [see here.](modules/ | Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | |:------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|----------:| | JSVFA v0.3.0 | 102 | 139 | 63/122 | 80 | 11 | 50 | 0.88 | 0.62 | 0.72 | 51.64% | -| JSVFA v0.6.1 | 120 | 141 | 75/122 | 95 | 14 | 35 | 0.87 | 0.73 | 0.79 | 61.48% | +| JSVFA v0.6.2 | 124 | 141 | 76/122 | 96 | 15 | 32 | 0.86 | 0.75 | 0.80 | 62.3% | | Flowdroid | 98 | 126 | 67/103 | 77 | 9 | 37 | 0.90 | 0.68 | 0.77 | 65.05% | | Joana | 123 | 138 | 85/122 | 86 | 19 | 34 | 0.82 | 0.72 | 0.77 | 69.67% | diff --git a/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/JSVFA.scala b/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/JSVFA.scala index 0bf01409..f146028c 100644 --- a/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/JSVFA.scala +++ b/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/JSVFA.scala @@ -40,14 +40,14 @@ abstract class JSVFA val methodRules = languageParser.evaluate(code()) /* - * Create an edge from the definition of the local argument + * Create an edge from the definition of the local argument * to the definitions of the base object of a method call. In * more details, we should use this rule to address a situation * like: * * - virtualinvoke r3.(r1); * - * Where we wanto create an edge from the definitions of r1 to + * Where we want to create an edge from the definitions of r1 to * the definitions of r3. */ trait CopyFromMethodArgumentToBaseObject extends RuleAction { @@ -130,10 +130,14 @@ abstract class JSVFA * In more details, we should use this rule to address * a situation like: * - * - $r6 = virtualinvoke r3.(); + * [i] $r6 = virtualinvoke r3.(); + * [ii] virtualinvoke r3.(); * * Where we want to create an edge from the definitions of r3 to - * this statement. + * the current statement. + * + * CONDITIONS: + * - For [i] case, the left operation ($r6) must be a local variable */ trait CopyFromMethodCallToLocal extends RuleAction { def apply( @@ -142,10 +146,18 @@ abstract class JSVFA localDefs: SimpleLocalDefs ) = { val expr = invokeStmt.getInvokeExpr - if (hasBaseObject(expr) && invokeStmt.isInstanceOf[jimple.AssignStmt]) { - val base = getBaseObject(expr) + var isLocalLeftOpFromAssignStmt = true + + if (invokeStmt.isInstanceOf[jimple.AssignStmt]) { val local = invokeStmt.asInstanceOf[jimple.AssignStmt].getLeftOp - if (base.isInstanceOf[Local] && local.isInstanceOf[Local]) { + if (! local.isInstanceOf[Local]) { + isLocalLeftOpFromAssignStmt = false + } + } + + if (hasBaseObject(expr) && isLocalLeftOpFromAssignStmt) { + val base = getBaseObject(expr) + if (base.isInstanceOf[Local]) { val localBase = base.asInstanceOf[Local] localDefs .getDefsOfAt(localBase, invokeStmt) @@ -162,7 +174,11 @@ abstract class JSVFA /* Create an edge from the definitions of a local argument * to the assignment statement. In more details, we should use this rule to address * a situation like: - * $r12 = virtualinvoke $r11.(r6); + * + * [i] $r12 = virtualinvoke $r11.(r6); + * [ii] virtualinvoke $r11.(r6); + * + * Where we want to create an edge from the definitions of r6 to the current statement. */ trait CopyFromMethodArgumentToLocal extends RuleAction { def from: Int @@ -173,9 +189,9 @@ abstract class JSVFA localDefs: SimpleLocalDefs ) = { val srcArg = invokeStmt.getInvokeExpr.getArg(from) - if (invokeStmt.isInstanceOf[JAssignStmt] && srcArg.isInstanceOf[Local]) { + if (srcArg.isInstanceOf[Local]) { val local = srcArg.asInstanceOf[Local] - val targetStmt = invokeStmt.asInstanceOf[jimple.AssignStmt] + val targetStmt = invokeStmt // current statement localDefs .getDefsOfAt(local, targetStmt) .forEach(sourceStmt => { diff --git a/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/dsl/DSL.scala b/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/dsl/DSL.scala index 5ddaec0f..872da94d 100644 --- a/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/dsl/DSL.scala +++ b/modules/core/src/main/scala/br/unb/cic/soot/svfa/jimple/dsl/DSL.scala @@ -62,6 +62,31 @@ trait DSL { if NamedMethodRule(className: "java.lang.StringBuffer", methodName: "toString") then CopyFromMethodCallToLocal() + rule cookieMethods = + if NamedMethodRule(className: "javax.servlet.http.Cookie", methodName: "getName") + then CopyFromMethodCallToLocal() + + rule cookieMethods = + if NamedMethodRule(className: "javax.servlet.http.Cookie", methodName: "getValue") + then CopyFromMethodCallToLocal() + + rule cookieMethods = + if NamedMethodRule(className: "javax.servlet.http.Cookie", methodName: "getComment") + then CopyFromMethodCallToLocal() + + rule sessionMethods = + if NamedMethodRule(className: "javax.servlet.http.HttpSession", methodName: "setAttribute") + then [ + CopyFromMethodArgumentToLocal(from: 1), + CopyFromMethodCallToLocal() + ] + + rule sessionMethods = + if NamedMethodRule(className: "javax.servlet.http.HttpSession", methodName: "getAttribute") + then [ + CopyFromMethodCallToLocal() + ] + rule skipNativeMethods = if NativeRule() then DoNothing() rule skipMethodsWithoutActiveBody = diff --git a/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.2.md b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.2.md new file mode 100644 index 00000000..5092bb18 --- /dev/null +++ b/modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.2.md @@ -0,0 +1,236 @@ + +> SUMMARY (*computed in December 2025.*) + +- **securibench.micro** - failed: 46, passed: 76 of 122 tests - (62.3%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | Pass Rate | +|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|:---------:| +| Aliasing | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | 33.33% | +| Arrays | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | 50% | +| Basic | 60 | 60 | 38/42 | 55 | 2 | 2 | 0.96 | 0.96 | 0.96 | 90.48% | +| Collections | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | 35.71% | +| Datastructures | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | 66.67% | +| Factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | 66.67% | +| Inter | 12 | 18 | 8/14 | 9 | 0 | 6 | 1.00 | 0.60 | 0.75 | 57.14% | +| Session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | 0% | +| StrongUpdates | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | 60% | +| Pred | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | 66.67% | +| Reflection | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | 0% | +| Sanitizers | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | 33.33% | +| TOTAL | 124 | 141 | 76/122 | 96 | 15 | 32 | 0.86 | 0.75 | 0.80 | 62.3% | + + +> Details + +[//]: # () + +[//]: # ) + +- **securibench.micro.aliasing** - failed: 4, passed: 2 of 6 tests - (33.33%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:---------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [i] +| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [ii] +| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [i] +| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Aliasing6 | 7 | 7 | ✅ | 7 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | + + +- **securibench.micro.arrays** - failed: 5, passed: 5 of 10 tests - (50.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Arrays1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays2 | 3 | 1 | ❌ | 0 | 2 | 0 | 0.00 | 0.00 | 0.00 | * issue [ii] +| Arrays3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [ii] +| Arrays6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Arrays8 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [ii] +| Arrays9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [ii] +| Arrays10 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [ii] +| TOTAL | 11 | 9 | 5/10 | 5 | 4 | 2 | 0.56 | 0.71 | 0.63 | + + +- **securibench.micro.basic** - failed: 4, passed: 38 of 42 tests - (90.48%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Basic0 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic5 | 3 | 3 | ✅ | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic11 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic12 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic15 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic16 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic17 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic18 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic19 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic20 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic21 | 4 | 4 | ✅ | 4 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic22 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic23 | 3 | 3 | ✅ | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic24 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic25 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic26 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic27 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic28 | 2 | 2 | ✅ | 0 | 0 | 2 | 1.00 | 1.00 | 1.00 | +| Basic29 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic30 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic31 | 3 | 2 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [i] +| Basic32 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic33 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic34 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic35 | 6 | 6 | ✅ | 6 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic36 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Basic37 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic38 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Basic39 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic41 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic42 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| TOTAL | 60 | 60 | 38/42 | 55 | 2 | 2 | 0.96 | 0.96 | 0.96 | + + +- **securibench.micro.collections** - failed: 9, passed: 5 of 14 tests - (35.71%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Collections1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections3 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections8 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections10 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections11 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Collections14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 8 | 15 | 5/14 | 5 | 1 | 8 | 0.83 | 0.38 | 0.52 | + + +- **securibench.micro.datastructures** - failed: 2, passed: 4 of 6 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:---------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Datastructures1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures4 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Datastructures5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Datastructures6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 5 | 5 | 4/6 | 4 | 1 | 1 | 0.80 | 0.80 | 0.80 | + + +- **securibench.micro.factories** - failed: 1, passed: 2 of 3 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Factories1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Factories2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Factories3 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [iv] +| TOTAL | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | + + +- **securibench.micro.inter** - failed: 6, passed: 8 of 14 tests - (57.14%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Inter1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter2 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [i] +| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [i] +| Inter6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] +| Inter7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Inter10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter11 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [ix] +| Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iv] +| Inter13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 12 | 18 | 8/14 | 9 | 0 | 6 | 1.00 | 0.60 | 0.75 | + + +- **securibench.micro.session** - failed: 3, passed: 0 of 3 tests - (0.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Session1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] +| Session2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] +| Session3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] +| TOTAL | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | + + +- **securibench.micro.strong_updates** - failed: 2, passed: 3 of 5 tests - (60.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| StrongUpdates1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates2 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vi] +| StrongUpdates4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| StrongUpdates5 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vi] +| TOTAL | 3 | 1 | 3/5 | 1 | 2 | 0 | 0.33 | 1.00 | 0.50 | + + +> Extra Tests + +These tests are not executed by Flowdroid + +- **securibench.micro.pred** - failed: 3, passed: 6 of 9 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Pred1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| Pred2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vii] +| Pred4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred5 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred6 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vii] +| Pred7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | * issue [vii] +| Pred8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | + + +- **securibench.micro.reflection** - failed: 4, passed: 0 of 4 tests - (0.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Refl1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] +| Refl2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] +| Refl3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] +| Refl4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [v] +| TOTAL | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | + + +- **securibench.micro.sanitizers** - failed: 4, passed: 2 of 6 tests - (33.33%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Sanitizers1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Sanitizers2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [viii] +| Sanitizers3 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| Sanitizers4 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [viii] +| Sanitizers5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [iii] +| Sanitizers6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | * issue [viii] +| TOTAL | 2 | 6 | 2/6 | 1 | 0 | 4 | 1.00 | 0.20 | 0.33 | \ No newline at end of file diff --git a/modules/securibench/src/test/java/javax/http/mock/HttpServletRequest.java b/modules/securibench/src/test/java/javax/http/mock/HttpServletRequest.java index 6c5c4691..f51a721f 100644 --- a/modules/securibench/src/test/java/javax/http/mock/HttpServletRequest.java +++ b/modules/securibench/src/test/java/javax/http/mock/HttpServletRequest.java @@ -1,5 +1,14 @@ package javax.servlet.http.mock; +import javax.servlet.http.Cookie; + + +/** + * IMPORTANT: + * + * Although this class was created to mock some methods + * only one test (basic16) is using it. + */ public class HttpServletRequest { public String getParameter(String s) { return "secret"; diff --git a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala index e2f7c806..0c930ff3 100644 --- a/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala +++ b/modules/securibench/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchTestSuite.scala @@ -541,7 +541,7 @@ class SecuribenchTestSuite extends FunSuite { assert(svfa.reportConflictsSVG().size == expectedConflicts) } - ignore( + test( "in the class Basic28 we should detect 2 conflicts in a complicated control flow test case" ) { val testName = "Basic28" diff --git a/release_notes.txt b/release_notes.txt index 9f2ff551..a3d0f24b 100644 --- a/release_notes.txt +++ b/release_notes.txt @@ -24,4 +24,8 @@ v0.6.0 v0.6.1 - Update readme information, -- Compute new test categories: Preds, Reflections, and Sanitizers. \ No newline at end of file +- Compute new test categories: Preds, Reflections, and Sanitizers. + +v0.6.2 +- Add DSL rules for Cookie methods and Session methods, +- Improve traits rules to handle statements types. \ No newline at end of file