diff --git a/README.md b/README.md index 74563c96..335c2cf2 100644 --- a/README.md +++ b/README.md @@ -60,41 +60,39 @@ This project use some of the [FlowDroid](https://github.com/secure-software-engi ### Flowdroid -~~TTests failed: 34, passed: 64, ignored: 6 of 104 test~~T - -Tests failed: 40, passed: 64, ignored: 0 of 104 test -Tests failed: 33, passed: 71, ignored: 0 of 104 test (original) - -Tests failed: +17.5%, passed: +9.86, ignored: 0 of 104 test (original) - -#### AliasingTest -Tests failed: 0, passed: 5, ignored: 1 of 6 test -#### ArraysTest -Tests failed: 9, passed: 1, ignored: 0 of 10 test -#### BasicTest -Tests failed: 0, passed: 37, ignored: 5 of 42 test - -Fails: -17 -36 (same) -38 -42 - -#### CollectionTest -Tests failed: 14, passed: 1, ignored: 0 of 15 test -#### DataStructureTest ☑ -Tests failed: 1, passed: 5, ignored: 0 of 6 test -#### FactoryTest ☑ -Tests failed: 1, passed: 2, ignored: 0 of 3 test -#### InterTest -Tests failed: 7, passed: 7, ignored: 0 of 14 test -~~#### PredTest~~ -~~Tests failed: 3, passed: 6, ignored: 0 of 9 test~~ -~~#### ReflectionTest~~ -~~Tests failed: 4, passed: 0, ignored: 0 of 4 test~~ -~~#### SanitizerTest~~ -~~Tests failed: 2, passed: 4, ignored: 0 of 6 test~~ -#### SessionTest ☑ -Tests failed: 3, passed: 0, ignored: 0 of 3 test -#### StrongUpdateTest ☑ -Tests failed: 1, passed: 4, ignored: 0 of 5 test \ No newline at end of file + +## TEST METRICS + +> failed: 0, passed: 61, ignored: 42 of 103 tests. + +| Test | Σ | TP | FP | +|:---------------:|:-------:|:------:|:--:| +| Aliasing | 5/6 | 10/11 | 0 | +| Array | 1/10 | 0/9 | 0 | +| Basic | 35/42 | 56/61 | 2 | +| Collection | 2/14 | 2/14 | 1 | +| DataStructure | 4/6 | 5/5 | 2 | +| Factory | 2/3 | 3/3 | 1 | +| Inter | 8/14 | 10/16 | 0 | +| ~~Pred~~ | ~~0/9~~ | - | - | +| ~~Reflection~~ | ~~0/4~~ | - | - | +| ~~Sanitizers~~ | ~~0/6~~ | - | - | +| Session | 0/3 | 0/3 | 0 | +| StrongUpdate | 4/5 | 0/1 | 0 | +| **TOTAL** | 61/103 | 86/123 | 6 | + +- **Precision:** 0.93 +- **Recall:** 0.70 +- **F-score:** 0.80 +- **Pass Rate:** 59.22% + +To have detailed information about each group of tests run, [see here.](old-metrics) + +**OBSERVATIONS** +- Flowdroid is not taking in count the TP expected in StrongUpdate4; +- Test Basic40 is commented in the test suite so the amount of TP differs from the original run by Flowdroid; +- There are two flaky tests: Basic6 and Inter11. + + +## DISCLAIMER +- The last code changes for this Release were added in March, 2023. \ No newline at end of file diff --git a/build.sbt b/build.sbt index 4c181852..424eba1b 100644 --- a/build.sbt +++ b/build.sbt @@ -3,7 +3,7 @@ scalaVersion := "2.12.8" name := "svfa-scala" organization := "br.unb.cic" -version := "0.2.1-SNAPSHOT" +version := "0.2.9" githubOwner := "rbonifacio" githubRepository := "svfa-scala" diff --git a/old-metrics.md b/old-metrics.md new file mode 100644 index 00000000..dc2b06f2 --- /dev/null +++ b/old-metrics.md @@ -0,0 +1,222 @@ +#### JSVFA old metrics + +> Metrics +- **securibench.micro** - failed: 46, passed: 57 of 103 tests - (55.34%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| aliasing | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | +| arrays | 0 | 9 | 1/10 | 0 | 0 | 9 | 0.00 | 0.00 | 0.00 | +| basic | 60 | 60 | 36/42 | 52 | 3 | 3 | 0.95 | 0.95 | 0.95 | +| collections | 3 | 15 | 1/14 | 1 | 1 | 13 | 0.50 | 0.07 | 0.12 | +| datastructures | 7 | 5 | 4/6 | 4 | 2 | 0 | 0.67 | 1.00 | 0.80 | +| factories | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | +| inter | 10 | 18 | 7/14 | 8 | 0 | 8 | 1.00 | 0.50 | 0.67 | +| session | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | +| strong_updates | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 94 | 126 | 57/103 | 75 | 8 | 40 | 0.90 | 0.65 | 0.75 | + + +> Details + +- **securibench.micro.aliasing** - failed: 4, passed: 2 of 6 tests - (33.33%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:---------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Aliasing1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Aliasing2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing4 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Aliasing5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Aliasing6 | 7 | 7 | ✅ | 7 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 10 | 12 | 2/6 | 8 | 1 | 3 | 0.89 | 0.73 | 0.80 | + +- **securibench.micro.arrays** - failed: 9, passed: 1 of 10 tests - (10.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Arrays1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays5 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| Arrays6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays8 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Arrays10 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 0 | 9 | 1/10 | 0 | 0 | 9 | 0.00 | 0.00 | 0.00 | + + +- **securibench.micro.basic** - failed: 6, passed: 36 of 42 tests - (85.71%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Basic0 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic5 | 3 | 3 | ✅ | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic7 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic11 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic12 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic15 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic16 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic17 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Basic18 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic19 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic20 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic21 | 4 | 4 | ✅ | 4 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic22 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic23 | 3 | 3 | ✅ | 3 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic24 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic25 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic26 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic27 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic28 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic29 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic30 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic31 | 3 | 2 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Basic32 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic33 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic34 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic35 | 6 | 6 | ✅ | 6 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic36 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Basic37 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic38 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Basic39 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic41 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Basic42 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 60 | 60 | 36/42 | 52 | 3 | 3 | 0.95 | 0.95 | 0.95 | + + +- **securibench.micro.collections** - failed: 13, passed: 1 of 14 tests - (7.14%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Collections1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Collections2 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Collections3 | 0 | 2 | ❌ | 0 | 0 | 2 | 0.00 | 0.00 | 0.00 | +| Collections4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections5 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections8 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections9 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections10 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections11 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections13 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Collections14 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 3 | 15 | 1/14 | 1 | 1 | 13 | 0.50 | 0.07 | 0.12 | + + +- **securibench.micro.datastructures** - failed: 2, passed: 4 of 6 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:---------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Datastructures1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures2 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Datastructures3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures4 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Datastructures5 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Datastructures6 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 7 | 5 | 4/6 | 4 | 2 | 0 | 0.67 | 1.00 | 0.80 | + + +- **securibench.micro.factories** - failed: 1, passed: 2 of 3 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:----------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Factories1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Factories2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Factories3 | 2 | 1 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| TOTAL | 4 | 3 | 2/3 | 2 | 1 | 0 | 0.67 | 1.00 | 0.80 | + + +- **securibench.micro.inter** - failed: 7, passed: 7 of 14 tests - (50.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Inter1 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter2 | 2 | 2 | ✅ | 2 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter3 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter4 | 0 | 2 | ❌ | 0 | 0 | 2 | 0.00 | 0.00 | 0.00 | +| Inter5 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter6 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter7 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter9 | 1 | 2 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter10 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter11 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter12 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Inter13 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Inter14 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 10 | 18 | 7/14 | 8 | 0 | 8 | 1.00 | 0.50 | 0.67 | + + +- **securibench.micro.session** - failed: 3, passed: 0 of 3 tests - (0.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Session1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Session2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Session3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 0 | 3 | 0/3 | 0 | 0 | 3 | 0.00 | 0.00 | 0.00 | + + +- **securibench.micro.strong_updates** - failed: 1, passed: 4 of 5 tests - (80.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| StrongUpdates1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates2 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates3 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| StrongUpdates4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| StrongUpdates5 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| TOTAL | 0 | 1 | 4/5 | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | + +**OBSERVATIONS** +- Flowdroid is not taking in count the TP expected in StrongUpdate4; +- Test Basic40 is commented in the test suite so the amount of TP differs from the original run by Flowdroid; +- There are two flaky tests: Basic6 and Inter11. + +> Extra Test +These tests are not executed by Flowdroid + +- **securibench.micro.pred** - failed: 3, passed: 6 of 9 tests - (66.67%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Pred1 | 0 | 0 | ✅ | 0 | 0 | 0 | 0.00 | 0.00 | 0.00 | +| Pred2 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred3 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred4 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred5 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred6 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred7 | 1 | 0 | ❌ | 0 | 1 | 0 | 0.00 | 0.00 | 0.00 | +| Pred8 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| Pred9 | 1 | 1 | ✅ | 1 | 0 | 0 | 1.00 | 1.00 | 1.00 | +| TOTAL | 8 | 5 | 6/9 | 5 | 3 | 0 | 0.63 | 1.00 | 0.77 | + + +- **securibench.micro.reflection** - failed: 4, passed: 0 of 4 tests - (0.0%) + +| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score | +|:-----:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:| +| Refl1 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Refl2 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Refl3 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| Refl4 | 0 | 1 | ❌ | 0 | 0 | 1 | 0.00 | 0.00 | 0.00 | +| TOTAL | 0 | 4 | 0/4 | 0 | 0 | 4 | 0.00 | 0.00 | 0.00 | + + +- **securibench.micro.sanitizers** - failed: 6, passed: 0 of 6 tests - (0.0%) +An exception is thrown when the tests run so it was not possible to compute metrics. \ No newline at end of file diff --git a/src/test/java/securibench/micro/aliasing/Aliasing7.java b/src/test/java/securibench/micro/aliasing/Aliasing7.java deleted file mode 100644 index 4e6281a9..00000000 --- a/src/test/java/securibench/micro/aliasing/Aliasing7.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - @author Benjamin Livshits - - $Id: Aliasing6.java,v 1.1 2006/04/21 17:14:27 livshits Exp $ - */ -package securibench.micro.aliasing; - -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import securibench.micro.BasicTestCase; -import securibench.micro.MicroTestCase; - -/** - * @servlet description="false positive of aliasing with copy propagation" - * @servlet vuln_count = "0" - * */ -public class Aliasing7 extends BasicTestCase implements MicroTestCase { - private static final String FIELD_NAME = "name"; - - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String[] names = new String[2]; - names[0] = req.getParameter(FIELD_NAME); - names[1] = "Foo"; - Object - o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15, o16, o17, o18, o19, o20, - o21, o22, o23, o24, o25, o26, o27, o28, o29, o30, o31, o32, o33, o34, o35, o36, o37, o38, o39, o40; - o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = o10 = o11 = o12 = o13 = o14 = o15 = o16 = o17 = o18 = o19 = o20 = - o21 = o22 = o23 = o24 = o25 = o26 = o27 = o28 = o29 = o30 = o31 = o32 = o33 = o34 = o35 = o36 = o37 = o38 = o39 = o40 = - names[1]; - - PrintWriter writer = resp.getWriter(); - writer.println(o1); /* OK */ - writer.println(o2); /* OK */ - writer.println(o3); /* OK */ - writer.println(o4); /* OK */ - writer.println(o32); /* OK */ - writer.println(o37); /* OK */ - writer.println(o40); /* OK */ - } - - public String getDescription() { - return "false positive of aliasing with copy propagation"; - } - - public int getVulnerabilityCount() { - return 0; - } -} \ No newline at end of file diff --git a/src/test/java/securibench/micro/aliasing/Aliasing8.java b/src/test/java/securibench/micro/aliasing/Aliasing8.java deleted file mode 100644 index eedee156..00000000 --- a/src/test/java/securibench/micro/aliasing/Aliasing8.java +++ /dev/null @@ -1,58 +0,0 @@ -/** - @author Benjamin Livshits - - $Id: Aliasing6.java,v 1.1 2006/04/21 17:14:27 livshits Exp $ - */ -package securibench.micro.aliasing; - -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import securibench.micro.BasicTestCase; -import securibench.micro.MicroTestCase; - -/** - * @servlet description="aliasing in an array index" - * @servlet vuln_count = "1" - * */ -public class Aliasing8 extends BasicTestCase implements MicroTestCase { - private static final String FIELD_NAME = "name"; - - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String[] names = new String[2]; - names[0] = req.getParameter(FIELD_NAME); - names[1] = "Foo"; - Object - o1, o2, o3, o4, o5, o6, o7, o8, o9, o10, o11, o12, o13, o14, o15, o16, o17, o18, o19, o20, - o21, o22, o23, o24, o25, o26, o27, o28, o29, o30, o31, o32, o33, o34, o35, o36, o37, o38, o39, o40; - o1 = o2 = o3 = o4 = o5 = o6 = o7 = o8 = o9 = o10 = o11 = o12 = o13 = o14 = o15 = o16 = o17 = o18 = o19 = o20 = - o21 = o22 = o23 = o24 = o25 = o26 = o27 = o28 = o29 = o30 = o31 = o32 = o33 = o34 = o35 = o36 = o37 = o38 = o39 = o40 = - names[1]; - - o2 = names[0]; - - PrintWriter writer = resp.getWriter(); - writer.println(o1); /* BAD */ - writer.println(o2); /* BAD */ - writer.println(o3); /* BAD */ - writer.println(o4); /* BAD */ - writer.println(o32); /* BAD */ - writer.println(o37); /* BAD */ - writer.println(o40); /* BAD */ - - sink(names); /* This should be consider a leak too, because one array element is tainted */ - } - - public String getDescription() { - return "aliasing in an array index"; - } - - public int getVulnerabilityCount() { - return 7; - } - - public void sink(String[] data) { } -} \ No newline at end of file diff --git a/src/test/java/securibench/micro/aliasing/Aliasing9.java b/src/test/java/securibench/micro/aliasing/Aliasing9.java deleted file mode 100644 index 73635096..00000000 --- a/src/test/java/securibench/micro/aliasing/Aliasing9.java +++ /dev/null @@ -1,51 +0,0 @@ -/** - @author Benjamin Livshits - - $Id: Aliasing6.java,v 1.1 2006/04/21 17:14:27 livshits Exp $ - */ -package securibench.micro.aliasing; - -import java.io.IOException; -import java.io.PrintWriter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -import securibench.micro.BasicTestCase; -import securibench.micro.MicroTestCase; - -/** - * @servlet description="interprocedural aliasing in an array index" - * @servlet vuln_count = "1" - * */ -public class Aliasing9 extends BasicTestCase implements MicroTestCase { - private static final String FIELD_NAME = "name"; - - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws IOException { - String[] names = source(req); - Object o1, o2; - o1 = o2 = names[1]; - - o2 = names[0]; - - PrintWriter writer = resp.getWriter(); - writer.println(o1); /* OK */ - writer.println(o2); /* BAD */ - } - - public String[] source(HttpServletRequest req) { - String[] names = new String[2]; - names[0] = req.getParameter(FIELD_NAME); - names[1] = "Foo"; - - return names; - } - - public String getDescription() { - return "aliasing with copy propagation"; - } - - public int getVulnerabilityCount() { - return 7; - } -} \ No newline at end of file diff --git a/src/test/java/securibench/micro/datastructures/Datastructures4.java b/src/test/java/securibench/micro/datastructures/Datastructures4.java index b3794774..ff08a281 100644 --- a/src/test/java/securibench/micro/datastructures/Datastructures4.java +++ b/src/test/java/securibench/micro/datastructures/Datastructures4.java @@ -16,7 +16,7 @@ /** * @servlet description="simple nexted data (false positive)" - * @servlet vuln_count = "1" + * @servlet vuln_count = "0" * */ public class Datastructures4 extends BasicTestCase implements MicroTestCase { public class C { @@ -50,6 +50,6 @@ public String getDescription() { } public int getVulnerabilityCount() { - return 1; + return 0; } } \ No newline at end of file diff --git a/src/test/java/securibench/micro/inter/Inter9.java b/src/test/java/securibench/micro/inter/Inter9.java index 02749c2e..6dee00b5 100644 --- a/src/test/java/securibench/micro/inter/Inter9.java +++ b/src/test/java/securibench/micro/inter/Inter9.java @@ -51,6 +51,6 @@ public String getDescription() { } public int getVulnerabilityCount() { - return 1; + return 2; } } \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/flowdroid/AliasingTest.scala b/src/test/scala/br/unb/cic/flowdroid/AliasingTest.scala deleted file mode 100644 index bc1a1ffa..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/AliasingTest.scala +++ /dev/null @@ -1,92 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph.NodeType -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class AliasingTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - if (unit.isInstanceOf[InvokeStmt]) { - val invokeStmt = unit.asInstanceOf[InvokeStmt] - return analyzeInvokeExpr(invokeStmt.getInvokeExpr) - } - if (unit.isInstanceOf[soot.jimple.AssignStmt]) { - val assignStmt = unit.asInstanceOf[AssignStmt] - if (assignStmt.getRightOp.isInstanceOf[InvokeExpr]) { - val invokeExpr = assignStmt.getRightOp.asInstanceOf[InvokeExpr] - return analyzeInvokeExpr(invokeExpr) - } - } - SimpleNode - } - - def analyzeInvokeExpr(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class AliasingTestSuite extends FunSuite { - test("in the class Aliasing1 we should detect 1 conflict of a simple aliasing test case") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Aliasing2 we should not detect any conflict in this false positive test case") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().isEmpty) - } - - test("in the class Aliasing3 we should not detect any conflict, but in Flowdroid this test case was not conclusive") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().isEmpty) - } - - test("in the class Aliasing4 we should detect 2 conflict") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - ignore("in the class Aliasing5 we should detect 1 conflict") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Aliasing6 we should detect 7 conflicts") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 7) - } - - ignore("in the class Aliasing7 we should detect 7 conflicts") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing7", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 7) - } - - ignore("in the class Aliasing8 we should detect 8 conflicts") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing8", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 8) - } - - ignore("in the class Aliasing9 we should detect 1 conflicts") { - val svfa = new AliasingTest("securibench.micro.aliasing.Aliasing9", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/ArrayTest.scala b/src/test/scala/br/unb/cic/flowdroid/ArrayTest.scala deleted file mode 100644 index f510c0c4..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/ArrayTest.scala +++ /dev/null @@ -1,99 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class ArrayTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class ArrayTestSuite extends FunSuite { - - test("description: Array1") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Array2") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Array3") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Array4") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Array5") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().isEmpty) - } - - test("description: Array6") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Array7") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays7", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Array8") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays8", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Array9") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays9", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Array10") { - val svfa = new ArrayTest("securibench.micro.arrays.Arrays10", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/BasicTest.scala b/src/test/scala/br/unb/cic/flowdroid/BasicTest.scala deleted file mode 100644 index 7512806c..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/BasicTest.scala +++ /dev/null @@ -1,294 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph.{NodeType, _} -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class BasicTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def runInFullSparsenessMode() = false - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class BasicTestSuite extends FunSuite { - test("in the class Basic2 we should detect 1 conflict of a simple XSS test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic0", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic1 we should detect 1 conflict of a simple XSS test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic2 we should detect 1 conflict of a XSS combined with a simple conditional test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic3 we should detect 1 conflict of a simple derived string test, very similar to Basic0") { - val svfa = new BasicTest("securibench.micro.basic.Basic3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic4 we should detect 1 conflict of a sensitive path test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic5 we should detect 3 conflicts of a moderately complex derived string test") { - val svfa = new BasicTest("securibench.micro.basic.Basic5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 3) - } - - //TODO: it looks like a flaky test. - ignore("in the class Basic6 we should detect 1 conflict of a complex derived string test") { - val svfa = new BasicTest("securibench.micro.basic.Basic6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic7 we should detect 1 conflict of a complex derived string with buffers test") { - val svfa = new BasicTest("securibench.micro.basic.Basic7", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic8 we should detect 1 conflict of a complex conditional test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic8", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic9 we should detect 1 conflict of a chain of assignments test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic9", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic10 we should detect 1 conflict of a chain of assignments and buffers test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic10", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic11 we should detect 2 conflicts of a simple derived string test with a false positive") { - val svfa = new BasicTest("securibench.micro.basic.Basic11", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("in the class Basic12 we should detect 2 conflicts of a simple conditional test case where both sides have sinks") { - val svfa = new BasicTest("securibench.micro.basic.Basic12", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("in the class Basic13 we should detect 1 conflict of a simple test case, the source method was modified to getInitParameterInstead") { - val svfa = new BasicTest("securibench.micro.basic.Basic13", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic14 we should detect 1 conflict of a servlet context and casts test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic14", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic15 we should detect 1 conflict of a casts more exhaustively test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic15", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic16 we should detect 1 conflict of a store statement in heap-allocated data structures test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic16", "doGet") - svfa.buildSparseValueFlowGraph() - // println(svfa.svgToDotModel()) - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore("in the class Basic17 we should detect 1 conflict of a store statement in heap-allocated data structures and a false positive test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic17", "doGet") - svfa.buildSparseValueFlowGraph() - // println(svfa.svgToDotModel()) - assert(svfa.reportConflictsSVG().size == 1) // the search should be context sensitive - } - - test("in the class Basic18 we should detect 1 conflict of a simple loop unrolling test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic18", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic19 we should detect 1 conflict of a simple SQL injection with prepared statements test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic19", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic20 we should detect 1 conflict of a simple SQL injection test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic20", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic21 we should detect 4 conflicts in a SQL injection with less commonly used methods test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic21", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 4) - } - - test("in the class Basic22 we should detect 1 conflict in a basic path traversal test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic22", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic23 we should detect 3 conflicts in a path traversal test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic23", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 3) - } - - test("in the class Basic24 we should detect 1 conflict in a unsafe redirect test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic24", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic25 we should detect 1 conflict in a test getParameterValues test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic25", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic26 we should detect 1 conflict in a getParameterMap test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic26", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic27 we should detect 1 conflict in a getParameterMap test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic27", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic28 we should detect 2 conflicts in a complicated control flow test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic28", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic29 we should detect 2 conflicts in a recursive data structures test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic29", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("in the class Basic30 we should detect 1 conflict in a field sensitivity test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic30", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic31 we should detect 3 conflicts in a values obtained from cookies test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic31", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 3) - } - - test("in the class Basic32 we should detect 1 conflict in a values obtained from headers test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic32", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic33 we should detect 1 conflict in a values obtained from headers test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic33", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic34 we should detect 2 conflicts in a values obtained from headers test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic34", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("in the class Basic35 we should detect 6 conflicts in a values obtained from HttpServletRequest test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic35", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 6) - } - - ignore("in the class Basic36 we should detect 1 conflict in a values obtained from HttpServletRequest input stream test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic36", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic37 we should detect 1 conflict in a StringTokenizer test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic37", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore("in the class Basic38 we should detect 1 conflict in a StringTokenizer with a false positive test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic38", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic39 we should detect 1 conflict in a StringTokenizer test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic39", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("in the class Basic41 we should detect 1 conflict in a use getInitParameter instead test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic41", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - ignore("in the class Basic42 we should detect 1 conflict in a use getInitParameterNames test case") { - val svfa = new BasicTest("securibench.micro.basic.Basic42", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/CollectionTest.scala b/src/test/scala/br/unb/cic/flowdroid/CollectionTest.scala deleted file mode 100644 index afd4889b..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/CollectionTest.scala +++ /dev/null @@ -1,129 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class CollectionTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class CollectionSuite extends FunSuite { - - test("description: Collection1") { - val svfa = new CollectionTest("securibench.micro.collections.Collections1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection2") { - val svfa = new CollectionTest("securibench.micro.collections.Collections2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection3") { - val svfa = new CollectionTest("securibench.micro.collections.Collections3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("description: Collection4") { - val svfa = new CollectionTest("securibench.micro.collections.Collections4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection5") { - val svfa = new CollectionTest("securibench.micro.collections.Collections5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection6") { - val svfa = new CollectionTest("securibench.micro.collections.Collections6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection7") { - val svfa = new CollectionTest("securibench.micro.collections.Collections7", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection8") { - val svfa = new CollectionTest("securibench.micro.collections.Collections8", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection9") { - val svfa = new CollectionTest("securibench.micro.collections.Collections9", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection10") { - val svfa = new CollectionTest("securibench.micro.collections.Collections10", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection11") { - val svfa = new CollectionTest("securibench.micro.collections.Collections11", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection11b") { - val svfa = new CollectionTest("securibench.micro.collections.Collections11b", "foo") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection12") { - val svfa = new CollectionTest("securibench.micro.collections.Collections12", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection13") { - val svfa = new CollectionTest("securibench.micro.collections.Collections13", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Collection14") { - val svfa = new CollectionTest("securibench.micro.collections.Collections14", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/DataStructureTest.scala b/src/test/scala/br/unb/cic/flowdroid/DataStructureTest.scala deleted file mode 100644 index 29897028..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/DataStructureTest.scala +++ /dev/null @@ -1,75 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class DataStructureTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class DataStructureTestSuite extends FunSuite { - - test("description: DataStructure1") { - val svfa = new DataStructureTest("securibench.micro.datastructures.Datastructures1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: DataStructure2") { - val svfa = new DataStructureTest("securibench.micro.datastructures.Datastructures2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: DataStructure3") { - val svfa = new DataStructureTest("securibench.micro.datastructures.Datastructures3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: DataStructure4") { - val svfa = new DataStructureTest("securibench.micro.datastructures.Datastructures4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: DataStructure5") { - val svfa = new DataStructureTest("securibench.micro.datastructures.Datastructures5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: DataStructure6") { - val svfa = new DataStructureTest("securibench.micro.datastructures.Datastructures6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/FactoryTest.scala b/src/test/scala/br/unb/cic/flowdroid/FactoryTest.scala deleted file mode 100644 index 40a95423..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/FactoryTest.scala +++ /dev/null @@ -1,57 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class FactoryTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class FactoryTestSuite extends FunSuite { - - test("description: Factory1") { - val svfa = new FactoryTest("securibench.micro.factories.Factories1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Factory2") { - val svfa = new FactoryTest("securibench.micro.factories.Factories2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Factory3") { - val svfa = new FactoryTest("securibench.micro.factories.Factories3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/InterTest.scala b/src/test/scala/br/unb/cic/flowdroid/InterTest.scala deleted file mode 100644 index fdf464f4..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/InterTest.scala +++ /dev/null @@ -1,125 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class InterTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class InterTestSuite extends FunSuite { - - test("description: Inter1") { - val svfa = new InterTest("securibench.micro.inter.Inter1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Inter2") { - val svfa = new InterTest("securibench.micro.inter.Inter2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("description: Inter3") { - val svfa = new InterTest("securibench.micro.inter.Inter3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Inter4") { - val svfa = new InterTest("securibench.micro.inter.Inter4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("description: Inter5") { - val svfa = new InterTest("securibench.micro.inter.Inter5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("description: Inter6") { - val svfa = new InterTest("securibench.micro.inter.Inter6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Inter7") { - val svfa = new InterTest("securibench.micro.inter.Inter7", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Inter8") { - val svfa = new InterTest("securibench.micro.inter.Inter8", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Inter9") { - val svfa = new InterTest("securibench.micro.inter.Inter9", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Inter10") { - val svfa = new InterTest("securibench.micro.inter.Inter10", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Inter11") { - val svfa = new InterTest("securibench.micro.inter.Inter11", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Inter12") { - val svfa = new InterTest("securibench.micro.inter.Inter12", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - - test("description: Inter13") { - val svfa = new InterTest("securibench.micro.inter.Inter13", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - - test("description: Inter14") { - val svfa = new InterTest("securibench.micro.inter.Inter14", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/PredTest.scala b/src/test/scala/br/unb/cic/flowdroid/PredTest.scala deleted file mode 100644 index 7cefec8a..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/PredTest.scala +++ /dev/null @@ -1,93 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class PredTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class PredTestSuite extends FunSuite { - - test("description: Pred1") { - val svfa = new PredTest("securibench.micro.pred.Pred1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test("description: Pred2") { - val svfa = new PredTest("securibench.micro.pred.Pred2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Pred3") { - val svfa = new PredTest("securibench.micro.pred.Pred3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test("description: Pred4") { - val svfa = new PredTest("securibench.micro.pred.Pred4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Pred5") { - val svfa = new PredTest("securibench.micro.pred.Pred5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Pred6") { - val svfa = new PredTest("securibench.micro.pred.Pred6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test("description: Pred7") { - val svfa = new PredTest("securibench.micro.pred.Pred7", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test("description: Pred8") { - val svfa = new PredTest("securibench.micro.pred.Pred8", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Pred9") { - val svfa = new PredTest("securibench.micro.pred.Pred9", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/ReflectionTest.scala b/src/test/scala/br/unb/cic/flowdroid/ReflectionTest.scala deleted file mode 100644 index 44814b10..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/ReflectionTest.scala +++ /dev/null @@ -1,63 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class ReflectionTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class ReflectionTestSuite extends FunSuite { - - test("description: Reflection1") { - val svfa = new ReflectionTest("securibench.micro.reflection.Refl1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Reflection2") { - val svfa = new ReflectionTest("securibench.micro.reflection.Refl2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Reflection3") { - val svfa = new ReflectionTest("securibench.micro.reflection.Refl3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Reflection4") { - val svfa = new ReflectionTest("securibench.micro.reflection.Refl4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/SanitizerTest.scala b/src/test/scala/br/unb/cic/flowdroid/SanitizerTest.scala deleted file mode 100644 index 08c3bc22..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/SanitizerTest.scala +++ /dev/null @@ -1,75 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class SanitizerTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class SanitizerTestSuite extends FunSuite { - - test("description: Sanitizer1") { - val svfa = new SanitizerTest("securibench.micro.sanitizers.Sanitizers1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Sanitizer2") { - val svfa = new SanitizerTest("securibench.micro.sanitizers.Sanitizers2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Sanitizer3") { - val svfa = new SanitizerTest("securibench.micro.sanitizers.Sanitizers3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test("description: Sanitizer4") { - val svfa = new SanitizerTest("securibench.micro.sanitizers.Sanitizers4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 2) - } - - test("description: Sanitizer5") { - val svfa = new SanitizerTest("securibench.micro.sanitizers.Sanitizers5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Sanitizer6") { - val svfa = new SanitizerTest("securibench.micro.sanitizers.Sanitizers6", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/SessionTest.scala b/src/test/scala/br/unb/cic/flowdroid/SessionTest.scala deleted file mode 100644 index ab9967cb..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/SessionTest.scala +++ /dev/null @@ -1,57 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class SessionTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class SessionTestSuite extends FunSuite { - - test("description: Session1") { - val svfa = new SessionTest("securibench.micro.session.Session1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Session2") { - val svfa = new SessionTest("securibench.micro.session.Session2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: Session3") { - val svfa = new SessionTest("securibench.micro.session.Session3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } -} diff --git a/src/test/scala/br/unb/cic/flowdroid/StrongUpdateTest.scala b/src/test/scala/br/unb/cic/flowdroid/StrongUpdateTest.scala deleted file mode 100644 index 40b8ed01..00000000 --- a/src/test/scala/br/unb/cic/flowdroid/StrongUpdateTest.scala +++ /dev/null @@ -1,69 +0,0 @@ -package br.unb.cic.flowdroid - -import br.unb.cic.soot.graph._ -import org.scalatest.FunSuite -import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} - -class StrongUpdateTest(var className: String = "", var mainMethod: String = "") extends FlowdroidSpec { - - override def getClassName(): String = className - - override def getMainMethod(): String = mainMethod - - override def analyze(unit: soot.Unit): NodeType = { - unit match { - case invokeStmt: InvokeStmt => - analyzeInvokeStmt(invokeStmt.getInvokeExpr) - case assignStmt: AssignStmt => - assignStmt.getRightOp match { - case invokeStmt: InvokeExpr => - analyzeInvokeStmt(invokeStmt) - case _ => - SimpleNode - } - case _ => SimpleNode - } - } - - def analyzeInvokeStmt(exp: InvokeExpr): NodeType = { - if (sourceList.contains(exp.getMethod.getSignature)) { - return SourceNode - } else if (sinkList.contains(exp.getMethod.getSignature)) { - return SinkNode - } - SimpleNode - } -} - -class StrongUpdateTestSuite extends FunSuite { - - test("description: StrongUpdate1") { - val svfa = new StrongUpdateTest("securibench.micro.strong_updates.StrongUpdates1", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test("description: StrongUpdate2") { - val svfa = new StrongUpdateTest("securibench.micro.strong_updates.StrongUpdates2", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test("description: StrongUpdate3") { - val svfa = new StrongUpdateTest("securibench.micro.strong_updates.StrongUpdates3", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } - - test("description: StrongUpdate4") { - val svfa = new StrongUpdateTest("securibench.micro.strong_updates.StrongUpdates4", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 1) - } - - test("description: StrongUpdate5") { - val svfa = new StrongUpdateTest("securibench.micro.strong_updates.StrongUpdates5", "doGet") - svfa.buildSparseValueFlowGraph() - assert(svfa.reportConflictsSVG().size == 0) - } -} diff --git a/src/test/scala/br/unb/cic/metrics/CustomMetrics.md b/src/test/scala/br/unb/cic/metrics/CustomMetrics.md new file mode 100644 index 00000000..82680861 --- /dev/null +++ b/src/test/scala/br/unb/cic/metrics/CustomMetrics.md @@ -0,0 +1,131 @@ +# CustomMetrics Trait Documentation + +## Overview + +The `CustomMetrics` trait provides a comprehensive framework for collecting, storing, and reporting test-related metrics. It is especially useful for evaluating the effectiveness of tools or algorithms in terms of detection (e.g., vulnerabilities, warnings, or test results). + +--- + +## Internal Data Structure + +- **metricsByTest**: + A mutable map (`mutable.Map[String, Metrics]`) that associates each test (by its name) with a `Metrics` object. This object holds all the relevant counters for that test. + +- **getOrCreateMetrics(testName: String): Metrics**: + Helper method to fetch the `Metrics` object for a test, or create a new one if it doesn't exist. + +--- + +## Metric Reporting Methods + +These methods incrementally update the metrics for a given test: + +- **reportTruePositives(testName, truePositives)**: + Adds to the count of true positives for the test. + +- **reportFalsePositives(testName, falsePositives)**: + Adds to the count of false positives. + +- **reportFalseNegatives(testName, falseNegatives)**: + Adds to the count of false negatives. + +- **reportTrueNegatives(testName)**: + Increments the count of true negatives by 1. + +- **reportPassedTest(testName)**: + Increments the count of passed tests by 1. + +- **reportFailedTest(testName)**: + Increments the count of failed tests by 1. + +- **reportExpected(testName, expected)**: + Adds to the count of expected findings (e.g., expected vulnerabilities). + +- **reportFound(testName, found)**: + Adds to the count of found findings (e.g., detected vulnerabilities). + +--- + +## Automated Metric Computation + +- **compute(expected, found, testName)**: + Given the expected and found counts for a test, this method: + - Updates the expected and found counts. + - Determines the test outcome: + - If both are zero: test passed, increment true negatives. + - If both are equal and nonzero: test passed, increment true positives. + - If found > expected: test failed, increment false positives. + - If expected > found: test failed, increment false negatives. + +--- + +## Metric Calculation Methods + +These methods compute standard evaluation metrics, either for a specific test or aggregated across all tests: + +- **precision(testName: String = null): Double** + \( \text{Precision} = \frac{TP}{TP + FP} \) + Returns the precision for a test or overall (rounded to 2 decimal places). + +- **recall(testName: String = null): Double** + \( \text{Recall} = \frac{TP}{TP + FN} \) + Returns the recall for a test or overall (rounded to 2 decimal places). + +- **f1Score(testName: String = null): Double** + \( F1 = 2 \times \frac{\text{Precision} \times \text{Recall}}{\text{Precision} + \text{Recall}} \) + Returns the F1 score for a test or overall (rounded to 2 decimal places). + +- **passRate(testName: String = null): Double** + \( \text{Pass Rate} = \frac{\text{Passed}}{\text{Passed} + \text{Failed}} \times 100 \) + Returns the pass rate as a percentage for a test or overall. + +- **vulnerabilities(testName: String = null): Int** + Returns the expected vulnerabilities for a test or the sum for all tests. + +- **vulnerabilitiesFound(testName: String = null): Int** + Returns the found vulnerabilities for a test or the sum for all tests. + +--- + +## Access and Reporting + +- **metricsFor(testName: String): Metrics** + Returns the `Metrics` object for a given test. + +- **report(testName: String): Unit** + Prints a detailed report for a specific test, including: + - Number of failed/passed tests + - Pass rate + - Expected vs. found warnings + - TP, FP, FN, TN counts + - Precision, recall, F1 score + +- **reportAll(): Unit** + Prints a report for every test in the map. + +- **reportSummary(reportName: String): Unit** + Prints a Markdown-style summary table for all tests, including: + - Test name, found/expected counts, status (pass/fail), TP, FP, FN, precision, recall, F1 score + - Totals and overall metrics at the end + +--- + +## Design and Usage + +- The trait is designed to be mixed into test suites or analysis tools that need to track and report on detection metrics. +- It supports both per-test and aggregate reporting. +- All metrics are rounded to two decimal places for readability. +- The reporting methods are designed for console output, with Markdown-style tables for easy copy-pasting into reports. + +--- + +## Extensibility + +- The trait relies on a `Metrics` class (not shown here) to store the actual counters. +- The design is modular, so you can extend or override methods for custom reporting or additional metrics. + +--- + +## Summary + +`CustomMetrics` is a reusable trait for tracking, computing, and reporting detailed test metrics (TP, FP, FN, TN, precision, recall, F1, pass rate, etc.) on a per-test and aggregate basis, with built-in support for both programmatic access and human-readable reporting. \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/metrics/CustomMetrics.scala b/src/test/scala/br/unb/cic/metrics/CustomMetrics.scala new file mode 100644 index 00000000..5f1dfe61 --- /dev/null +++ b/src/test/scala/br/unb/cic/metrics/CustomMetrics.scala @@ -0,0 +1,185 @@ +package br.unb.cic.metrics +import scala.collection.mutable + +trait CustomMetrics { + + private val metricsByTest = mutable.Map[String, Metrics]() + + private def getOrCreateMetrics(testName: String): Metrics = + metricsByTest.getOrElseUpdate(testName, Metrics()) + + def reportTruePositives(testName: String, truePositives: Int): Unit = { + getOrCreateMetrics(testName).truePositives += truePositives + } + + def reportFalsePositives(testName: String, falsePositives: Int): Unit = { + getOrCreateMetrics(testName).falsePositives += falsePositives + } + + def reportFalseNegatives(testName: String, falseNegatives: Int): Unit = { + getOrCreateMetrics(testName).falseNegatives += falseNegatives + } + + def reportTrueNegatives(testName: String): Unit = { + getOrCreateMetrics(testName).trueNegatives += 1 + } + + def reportPassedTest(testName: String): Unit = { + getOrCreateMetrics(testName).passedTests += 1 + } + + def reportFailedTest(testName: String): Unit = { + getOrCreateMetrics(testName).failedTests += 1 + } + + def reportExpected(testName: String, expected: Int): Unit = { + getOrCreateMetrics(testName).expected += expected + } + + def reportFound(testName: String, found: Int): Unit = { + getOrCreateMetrics(testName).found += found + } + + def compute(expected: Int, found: Int, testName: String): Unit = { + reportExpected(testName, expected) + reportFound(testName, found) + (expected, found) match { + case (e, f) if e == f && e == 0 => + reportPassedTest(testName) + reportTrueNegatives(testName) + case (e, f) if e == f => + reportPassedTest(testName) + reportTruePositives(testName, e) + case (e, f) if f > e => + reportFailedTest(testName) + reportFalsePositives(testName, f - e) + case (e, f) if e > f => + reportFailedTest(testName) + reportFalseNegatives(testName, e - f) + } + } + + def precision(testName: String = null): Double = { + val (tp, fp) = Option(testName) match { + case Some(name) => + val m = getOrCreateMetrics(name) + (m.truePositives, m.falsePositives) + case None => + metricsByTest.values.foldLeft((0, 0)) { case ((accTP, accFP), m) => (accTP + m.truePositives, accFP + m.falsePositives) } + } + val denom = tp + fp + val value = denom match { + case 0 => 0.0 + case d => (tp * 1.0) / d + } + BigDecimal(value).setScale(2, BigDecimal.RoundingMode.HALF_UP).toDouble + } + + def recall(testName: String = null): Double = { + val (tp, fn) = Option(testName) match { + case Some(name) => + val m = getOrCreateMetrics(name) + (m.truePositives, m.falseNegatives) + case None => + metricsByTest.values.foldLeft((0, 0)) { case ((accTP, accFN), m) => (accTP + m.truePositives, accFN + m.falseNegatives) } + } + val denom = tp + fn + val value = denom match { + case 0 => 0.0 + case d => (tp * 1.0) / d + } + BigDecimal(value).setScale(2, BigDecimal.RoundingMode.HALF_UP).toDouble + } + + def f1Score(testName: String = null): Double = { + val p = precision(testName) + val r = recall(testName) + val value = (p + r) match { + case 0.0 => 0.0 + case s => 2 * (p * r) / s + } + BigDecimal(value).setScale(2, BigDecimal.RoundingMode.HALF_UP).toDouble + } + + def passRate(testName: String = null): Double = { + val (passed, failed) = Option(testName) match { + case Some(name) => + val m = getOrCreateMetrics(name) + (m.passedTests, m.failedTests) + case None => + metricsByTest.values.foldLeft((0, 0)) { case ((accPassed, accFailed), m) => (accPassed + m.passedTests, accFailed + m.failedTests) } + } + val denom = passed + failed + val value = denom match { + case 0 => 0.0 + case d => (passed * 1.0) / d * 100 + } + BigDecimal(value).setScale(2, BigDecimal.RoundingMode.HALF_UP).toDouble + } + + def vulnerabilities(testName: String = null): Int = Option(testName) match { + case Some(name) => getOrCreateMetrics(name).expected + case None => metricsByTest.values.map(_.expected).sum + } + + def vulnerabilitiesFound(testName: String = null): Int = Option(testName) match { + case Some(name) => getOrCreateMetrics(name).found + case None => metricsByTest.values.map(_.found).sum + } + + def metricsFor(testName: String): Metrics = getOrCreateMetrics(testName) + + def report(testName: String): Unit = { + val m = getOrCreateMetrics(testName) + println("----------------------------------------------------------------------------------------------------------------") + println(s"Metrics for test: $testName") + println(s"failed = ${m.failedTests}, passed = ${m.passedTests} of = ${m.passedTests + m.failedTests} tests.") + println(s"Pass Rate = ${passRate(testName)}%") + println(s"Expecting ${vulnerabilities(testName)} of ${vulnerabilitiesFound(testName)} warnings.") + println(s"TP = ${m.truePositives} FP = ${m.falsePositives} FN = ${m.falseNegatives} TN = ${m.trueNegatives}") + println(s"Precision = ${precision(testName)}% Recall = ${recall(testName)}% F-score = ${f1Score(testName)}%") + } + + def reportAll(): Unit = { + metricsByTest.keys.foreach(report) + } + + def reportSummary(reportName: String): Unit = { + + println(s"- **$reportName** - failed: ${metricsByTest.values.count(_.failedTests > 0)}, passed: ${metricsByTest.values.count(_.passedTests > 0)} of ${metricsByTest.values.size} tests - (${passRate()}%)") + + val header = "| Test | Found | Expected | Status | TP | FP | FN | Precision | Recall | F-score |" + val sep = "|:--------------:|:-----:|:--------:|:------:|:--:|:--:|:---|:---------:|:------:|:-------:|" + println(header) + println(sep) + var totalFound = 0 + var totalExpected = 0 + var totalPassed = 0 + var totalTests = 0 + var totalTP = 0 + var totalFP = 0 + var totalFN = 0 + + metricsByTest.toSeq.sortBy(_._1).foreach { case (testName, m) => + val status = if (m.found == m.expected) "✅" else "❌" + val prec = precision(testName) + val rec = recall(testName) + val f1 = f1Score(testName) + val shortTestName = testName.split('.').last.padTo(14, ' ') + println(f"| $shortTestName| ${m.found}%5d | ${m.expected}%8d | ${status}%6s | ${m.truePositives}%2d | ${m.falsePositives}%2d | ${m.falseNegatives}%3d | ${prec}%9.2f | ${rec}%6.2f | ${f1}%7.2f |") + totalFound += m.found + totalExpected += m.expected + totalPassed += m.passedTests + totalTests += m.passedTests + m.failedTests + totalTP += m.truePositives + totalFP += m.falsePositives + totalFN += m.falseNegatives + } + + val totalPrec = precision() + val totalRec = recall() + val totalF1 = f1Score() + val totalStatus = s"${totalPassed}/${totalTests}" + println(f"| TOTAL | ${totalFound}%5d | ${totalExpected}%8d | ${totalStatus}%6s | ${totalTP}%2d | ${totalFP}%2d | ${totalFN}%3d | ${totalPrec}%9.2f | ${totalRec}%6.2f | ${totalF1}%7.2f |") + } +} diff --git a/src/test/scala/br/unb/cic/metrics/Metrics.scala b/src/test/scala/br/unb/cic/metrics/Metrics.scala new file mode 100644 index 00000000..9735aa45 --- /dev/null +++ b/src/test/scala/br/unb/cic/metrics/Metrics.scala @@ -0,0 +1,12 @@ +package br.unb.cic.metrics + +case class Metrics( + var truePositives: Int = 0, + var falsePositives: Int = 0, + var falseNegatives: Int = 0, + var trueNegatives: Int = 0, + var passedTests: Int = 0, + var failedTests: Int = 0, + var expected: Int = 0, + var found: Int = 0 +) diff --git a/src/test/scala/br/unb/cic/metrics/MetricsTest.scala b/src/test/scala/br/unb/cic/metrics/MetricsTest.scala new file mode 100644 index 00000000..859a5dc5 --- /dev/null +++ b/src/test/scala/br/unb/cic/metrics/MetricsTest.scala @@ -0,0 +1,83 @@ +package br.unb.cic.metrics + +import org.scalatest.FunSuite + + +class MetricsTest extends FunSuite with CustomMetrics { + + test("precision returns 0.0 when denominator is zero") { + val testName = "precisionZeroDenom" + compute(0, 0, testName) + assert(precision(testName) == 0.0) + } + + test("recall returns 0.0 when denominator is zero") { + val testName = "recallZeroDenom" + compute(0, 0, testName) + assert(recall(testName) == 0.0) + } + + test("f1Score returns 0.0 when both precision and recall are zero") { + val testName = "f1Zero" + compute(0, 0, testName) + assert(f1Score(testName) == 0.0) + } + + test("passRate returns 0.0 when denominator is zero") { + val testName = "passRateZeroDenom" + // No compute call, so passed/failed remain zero + assert(passRate(testName) == 0.0) + } + + test("precision, recall, f1Score, and passRate normal cases") { + val testName = "normalCase" + // Simulate 3 passed, 1 failed, 8 TP, 2 FP, 2 FN + for (_ <- 1 to 3) reportPassedTest(testName) + reportFailedTest(testName) + reportTruePositives(testName, 8) + reportFalsePositives(testName, 2) + reportFalseNegatives(testName, 2) + assert(precision(testName) == 0.8) + assert(recall(testName) == 0.8) + assert(f1Score(testName) == 0.8) + assert(passRate(testName) == 75.0) + } + + test("compute method updates metrics for true positive case") { + val testName = "tpCase" + compute(5, 5, testName) + val m = metricsFor(testName) + assert(m.truePositives == 5) + assert(m.passedTests == 1) + assert(m.failedTests == 0) + assert(m.trueNegatives == 0) + } + + test("compute method updates metrics for true negative case") { + val testName = "tnCase" + compute(0, 0, testName) + val m = metricsFor(testName) + assert(m.trueNegatives == 1) + assert(m.passedTests == 1) + assert(m.failedTests == 0) + assert(m.truePositives == 0) + } + + test("compute method updates metrics for false positive case") { + val testName = "fpCase" + compute(2, 5, testName) + val m = metricsFor(testName) + assert(m.falsePositives == 3) + assert(m.failedTests == 1) + assert(m.passedTests == 0) + } + + test("compute method updates metrics for false negative case") { + val testName = "fnCase" + compute(5, 2, testName) + val m = metricsFor(testName) + assert(m.falseNegatives == 3) + assert(m.failedTests == 1) + assert(m.passedTests == 0) + } +} diff --git a/src/test/scala/br/unb/cic/securibench/SecuribenchBaseTest.scala b/src/test/scala/br/unb/cic/securibench/SecuribenchBaseTest.scala new file mode 100644 index 00000000..31d75ca6 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/SecuribenchBaseTest.scala @@ -0,0 +1,36 @@ +package br.unb.cic.securibench + +import br.unb.cic.soot.graph._ +import org.scalatest.FunSuite +import securibench.micro.MicroTestCase +import soot.jimple.{AssignStmt, InvokeExpr, InvokeStmt} + +class SecuribenchBaseTest(var className: String = "", var mainMethod: String = "") extends SecuribenchSpec { + override def getClassName(): String = className + + override def getMainMethod(): String = mainMethod + + override def analyze(unit: soot.Unit): NodeType = { + if (unit.isInstanceOf[InvokeStmt]) { + val invokeStmt = unit.asInstanceOf[InvokeStmt] + return analyzeInvokeExpr(invokeStmt.getInvokeExpr) + } + if (unit.isInstanceOf[soot.jimple.AssignStmt]) { + val assignStmt = unit.asInstanceOf[AssignStmt] + if (assignStmt.getRightOp.isInstanceOf[InvokeExpr]) { + val invokeExpr = assignStmt.getRightOp.asInstanceOf[InvokeExpr] + return analyzeInvokeExpr(invokeExpr) + } + } + SimpleNode + } + + def analyzeInvokeExpr(exp: InvokeExpr): NodeType = { + if (sourceList.contains(exp.getMethod.getSignature)) { + return SourceNode + } else if (sinkList.contains(exp.getMethod.getSignature)) { + return SinkNode + } + SimpleNode + } +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/flowdroid/FlowdroidSpec.scala b/src/test/scala/br/unb/cic/securibench/SecuribenchSpec.scala similarity index 97% rename from src/test/scala/br/unb/cic/flowdroid/FlowdroidSpec.scala rename to src/test/scala/br/unb/cic/securibench/SecuribenchSpec.scala index c471e69b..747cd523 100644 --- a/src/test/scala/br/unb/cic/flowdroid/FlowdroidSpec.scala +++ b/src/test/scala/br/unb/cic/securibench/SecuribenchSpec.scala @@ -1,8 +1,8 @@ -package br.unb.cic.flowdroid +package br.unb.cic.securibench import br.unb.cic.soot.JSVFATest -abstract class FlowdroidSpec extends JSVFATest { +abstract class SecuribenchSpec extends JSVFATest { val sinkList: Seq[String] = List( "", "", diff --git a/src/test/scala/br/unb/cic/securibench/SecuribenchTest.scala b/src/test/scala/br/unb/cic/securibench/SecuribenchTest.scala new file mode 100644 index 00000000..8b39ddf6 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/SecuribenchTest.scala @@ -0,0 +1,90 @@ +package br.unb.cic.securibench + +import java.io.File +import java.nio.file.{Files, Paths} +import br.unb.cic.metrics.CustomMetrics +import org.scalatest.FunSuite +import securibench.micro.MicroTestCase + +abstract class SecuribenchTest extends FunSuite with CustomMetrics { + + def basePackage(): String + + def entryPointMethod(): String + + def getJavaFilesFromPackage(packageName: String): List[AnyRef] = { + val classPath = System.getProperty("java.class.path") + val paths = classPath.split(File.pathSeparator) + + paths.flatMap { path => + val packagePath = packageName.replace('.', '/') + val fullPath = Paths.get(path, packagePath) + if (Files.exists(fullPath) && Files.isDirectory(fullPath)) { + + val filesBySubdir: List[AnyRef] = Files.walk(fullPath) + .filter(Files.isDirectory(_)) + .map[List[AnyRef]](d => getJavaFilesFromPackage(s"$packageName.${d.getFileName.toString}")) + .filter(_.nonEmpty) + .toArray + .toList + + val filesByDir = Files.walk(fullPath) + .filter { + case f if f.toString.endsWith(".class") => { + try { + val className = f.getFileName.toString.split("/").last.replace(".class", "") + val fullClassName = s"${packageName}.$className" + val clazz = Class.forName(fullClassName) + classOf[MicroTestCase].isAssignableFrom(clazz) && + ! clazz.isInterface && + ! java.lang.reflect.Modifier.isAbstract(clazz.getModifiers) + } catch { + case _ => false + } + } + case _ => false + } + .toArray + .toList + filesByDir ++ filesBySubdir + } else { + List.empty[String] + } + }.toList + } + + def generateDynamicTests(packageName: String): Unit = { + val files = getJavaFilesFromPackage(packageName) + this.generateDynamicTests(files, packageName) + this.reportSummary(packageName) + } + + def generateDynamicTests(files: List[AnyRef], packageName: String): Unit = { + files.foreach { + case list: List[AnyRef] => this.generateDynamicTests(list, packageName) + case list : java.nio.file.Path => generateDynamicTests(list, packageName) + case _ => + } + } + + def generateDynamicTests(file: AnyRef, packageName: String): Unit = { + var fileName = file.toString.replace(".class", "").replace("/",".") + fileName = fileName.split(packageName).last; + val className = s"$packageName$fileName" + val clazz = Class.forName(className) + + val svfa = new SecuribenchBaseTest(className, entryPointMethod()) + svfa.buildSparseValueFlowGraph() + val conflicts = svfa.reportConflictsSVG() + + val expected = clazz.getMethod("getVulnerabilityCount").invoke(clazz.getDeclaredConstructor().newInstance()).asInstanceOf[Int] + val found = conflicts.size + + this.compute(expected, found, className) + } + + test(s"running testsuite from ${basePackage()}") { + generateDynamicTests(basePackage()) + assert(this.vulnerabilities() == this.vulnerabilitiesFound()) + } +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedExtraTest.scala b/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedExtraTest.scala new file mode 100644 index 00000000..bffd830c --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedExtraTest.scala @@ -0,0 +1,131 @@ +package br.unb.cic.securibench.deprecated + +import br.unb.cic.securibench.SecuribenchBaseTest +import org.scalatest.FunSuite + +class SecuribenchDeprecatedExtraTest extends FunSuite { + + /** + * PRED TESTs + */ + + test("in the class Pred1 we should detect 0 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test("in the class Pred2 we should detect 1 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Pred3 we should detect 0 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test("in the class Pred4 we should detect 1 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Pred5 we should detect 1 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Pred6 we should detect 0 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + ignore("in the class Pred7 we should detect 0 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred7", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test("in the class Pred8 we should detect 1 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred8", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Pred9 we should detect 1 conflict(s) of a Pred test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.pred.Pred9", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * REFLECTION TESTs + */ + ignore("in the class Refl1 we should detect 1 conflict(s) of a Reflection test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.reflection.Refl1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Refl2 we should detect 1 conflict(s) of a Reflection test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.reflection.Refl2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Refl3 we should detect 1 conflict(s) of a Reflection test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.reflection.Refl3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Refl4 we should detect 1 conflict(s) of a Reflection test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.reflection.Refl4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * SANITIZERS TESTs + */ + ignore("in the class Sanitizers1 we should detect 1 conflict(s) of a Sanitizers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.sanitizers.Sanitizers1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Sanitizers2 we should detect 1 conflict(s) of a Sanitizers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.sanitizers.Sanitizers2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Sanitizers3 we should detect 0 conflict(s) of a Sanitizers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.sanitizers.Sanitizers3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + ignore("in the class Sanitizers4 we should detect 2 conflict(s) of a Sanitizers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.sanitizers.Sanitizers4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + ignore("in the class Sanitizers5 we should detect 1 conflict(s) of a Sanitizers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.sanitizers.Sanitizers5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Sanitizers6 we should detect 1 conflict(s) of a Sanitizers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.sanitizers.Sanitizers6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedTestSuite.scala b/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedTestSuite.scala new file mode 100644 index 00000000..46e7c84b --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/deprecated/SecuribenchDeprecatedTestSuite.scala @@ -0,0 +1,668 @@ +package br.unb.cic.securibench.deprecated + +import br.unb.cic.securibench.SecuribenchBaseTest +import org.scalatest.FunSuite + +class SecuribenchDeprecatedTestSuite extends FunSuite { + + /** + * ALIASING TESTs + */ + + test("in the class Aliasing1 we should detect 1 conflict of a simple aliasing test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.aliasing.Aliasing1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Aliasing2 we should not detect any conflict in this false positive test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.aliasing.Aliasing2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test("in the class Aliasing3 we should not detect any conflict, but in Flowdroid this test case was not conclusive") { + val svfa = new SecuribenchBaseTest("securibench.micro.aliasing.Aliasing3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test("in the class Aliasing4 we should detect 2 conflict") { + val svfa = new SecuribenchBaseTest("securibench.micro.aliasing.Aliasing4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + ignore("in the class Aliasing5 we should detect 1 conflict") { + val svfa = new SecuribenchBaseTest("securibench.micro.aliasing.Aliasing5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Aliasing6 we should detect 7 conflicts") { + val svfa = new SecuribenchBaseTest("securibench.micro.aliasing.Aliasing6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 7) + } + + /** + * ARRAY TESTs + */ + + ignore("description: Array1") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Array2") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Array3") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Array4") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: Array5") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + ignore("description: Array6") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Array7") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays7", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Array8") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays8", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Array9") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays9", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Array10") { + val svfa = new SecuribenchBaseTest("securibench.micro.arrays.Arrays10", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * BASIC TESTs + */ + + test("in the class Basic1 we should detect 1 conflict of a simple XSS test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic2 we should detect 1 conflict of a XSS combined with a simple conditional test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic3 we should detect 1 conflict of a simple derived string test, very similar to Basic0") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic4 we should detect 1 conflict of a sensitive path test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic5 we should detect 3 conflicts of a moderately complex derived string test") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 3) + } + + //FLAKY + ignore("in the class Basic6 we should detect 1 conflict of a complex derived string test") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic7 we should detect 1 conflict of a complex derived string with buffers test") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic7", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic8 we should detect 1 conflict of a complex conditional test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic8", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic9 we should detect 1 conflict of a chain of assignments test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic9", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic10 we should detect 1 conflict of a chain of assignments and buffers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic10", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic11 we should detect 2 conflicts of a simple derived string test with a false positive") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic11", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + test("in the class Basic12 we should detect 2 conflicts of a simple conditional test case where both sides have sinks") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic12", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + test("in the class Basic13 we should detect 1 conflict of a simple test case, the source method was modified to getInitParameterInstead") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic13", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic14 we should detect 1 conflict of a servlet context and casts test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic14", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic15 we should detect 1 conflict of a casts more exhaustively test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic15", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic16 we should detect 1 conflict of a store statement in heap-allocated data structures test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic16", "doGet") + svfa.buildSparseValueFlowGraph() + // println(svfa.svgToDotModel()) + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Basic17 we should detect 1 conflict of a store statement in heap-allocated data structures and a false positive test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic17", "doGet") + svfa.buildSparseValueFlowGraph() + // println(svfa.svgToDotModel()) + assert(svfa.reportConflictsSVG().size == 1) // the search should be context sensitive + } + + test("in the class Basic18 we should detect 1 conflict of a simple loop unrolling test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic18", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic19 we should detect 1 conflict of a simple SQL injection with prepared statements test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic19", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic20 we should detect 1 conflict of a simple SQL injection test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic20", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic21 we should detect 4 conflicts in a SQL injection with less commonly used methods test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic21", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 4) + } + + test("in the class Basic22 we should detect 1 conflict in a basic path traversal test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic22", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic23 we should detect 3 conflicts in a path traversal test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic23", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 3) + } + + test("in the class Basic24 we should detect 1 conflict in a unsafe redirect test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic24", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic25 we should detect 1 conflict in a test getParameterValues test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic25", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic26 we should detect 1 conflict in a getParameterMap test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic26", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic27 we should detect 1 conflict in a getParameterMap test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic27", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Basic28 we should detect 2 conflicts in a complicated control flow test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic28", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + test("in the class Basic29 we should detect 2 conflicts in a recursive data structures test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic29", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + test("in the class Basic30 we should detect 1 conflict in a field sensitivity test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic30", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic31 we should detect 3 conflicts in a values obtained from cookies test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic31", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 3) + } + + test("in the class Basic32 we should detect 1 conflict in a values obtained from headers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic32", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic33 we should detect 1 conflict in a values obtained from headers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic33", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic34 we should detect 2 conflicts in a values obtained from headers test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic34", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + test("in the class Basic35 we should detect 6 conflicts in a values obtained from HttpServletRequest test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic35", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 6) + } + + ignore("in the class Basic36 we should detect 1 conflict in a values obtained from HttpServletRequest input stream test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic36", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic37 we should detect 1 conflict in a StringTokenizer test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic37", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Basic38 we should detect 1 conflict in a StringTokenizer with a false positive test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic38", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic39 we should detect 1 conflict in a StringTokenizer test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic39", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Basic40 we should detect 1 conflict in a use getInitParameter instead test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic40", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("in the class Basic41 we should detect 1 conflict in a use getInitParameter instead test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic41", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("in the class Basic42 we should detect 1 conflict in a use getInitParameterNames test case") { + val svfa = new SecuribenchBaseTest("securibench.micro.basic.Basic42", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * COLLECTION TESTs + */ + + test("description: Collection1") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection2") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection3") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + ignore("description: Collection4") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection5") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection6") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection7") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections7", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection8") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections8", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: Collection9") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections9", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + ignore("description: Collection10") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections10", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection11") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections11", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection12") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections12", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection13") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections13", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Collection14") { + val svfa = new SecuribenchBaseTest("securibench.micro.collections.Collections14", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * DATASTRUCTURE TESTs + */ + + test("description: DataStructure1") { + val svfa = new SecuribenchBaseTest("securibench.micro.datastructures.Datastructures1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: DataStructure2") { + val svfa = new SecuribenchBaseTest("securibench.micro.datastructures.Datastructures2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: DataStructure3") { + val svfa = new SecuribenchBaseTest("securibench.micro.datastructures.Datastructures3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: DataStructure4") { + val svfa = new SecuribenchBaseTest("securibench.micro.datastructures.Datastructures4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test("description: DataStructure5") { + val svfa = new SecuribenchBaseTest("securibench.micro.datastructures.Datastructures5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: DataStructure6") { + val svfa = new SecuribenchBaseTest("securibench.micro.datastructures.Datastructures6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * FACTORY TESTs + */ + + test("description: Factory1") { + val svfa = new SecuribenchBaseTest("securibench.micro.factories.Factories1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: Factory2") { + val svfa = new SecuribenchBaseTest("securibench.micro.factories.Factories2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Factory3") { + val svfa = new SecuribenchBaseTest("securibench.micro.factories.Factories3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * INTER TESTs + */ + + test("description: Inter1") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: Inter2") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + test("description: Inter3") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Inter4") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter4", "doGet") + svfa.buildSparseValueFlowGraph() +// println(svfa.svgToDotModel()) + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: Inter5") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Inter6") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter6", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Inter7") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter7", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: Inter8") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter8", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Inter9") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter9", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 2) + } + + test("description: Inter10") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter10", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + +// FLAKY + ignore("description: Inter11") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter11", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Inter12") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter12", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + + test("description: Inter13") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter13", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + + test("description: Inter14") { + val svfa = new SecuribenchBaseTest("securibench.micro.inter.Inter14", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * SESSION TESTs + */ + + ignore("description: Session1") { + val svfa = new SecuribenchBaseTest("securibench.micro.session.Session1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Session2") { + val svfa = new SecuribenchBaseTest("securibench.micro.session.Session2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + ignore("description: Session3") { + val svfa = new SecuribenchBaseTest("securibench.micro.session.Session3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + /** + * STRONG UPDATE TESTs + */ + + test("description: StrongUpdate1") { + val svfa = new SecuribenchBaseTest("securibench.micro.strong_updates.StrongUpdates1", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test("description: StrongUpdate2") { + val svfa = new SecuribenchBaseTest("securibench.micro.strong_updates.StrongUpdates2", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + test("description: StrongUpdate3") { + val svfa = new SecuribenchBaseTest("securibench.micro.strong_updates.StrongUpdates3", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } + + ignore("description: StrongUpdate4") { + val svfa = new SecuribenchBaseTest("securibench.micro.strong_updates.StrongUpdates4", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 1) + } + + test("description: StrongUpdate5") { + val svfa = new SecuribenchBaseTest("securibench.micro.strong_updates.StrongUpdates5", "doGet") + svfa.buildSparseValueFlowGraph() + assert(svfa.reportConflictsSVG().size == 0) + } +} diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchAliasingTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchAliasingTest.scala new file mode 100644 index 00000000..1677ecbe --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchAliasingTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchAliasingTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.aliasing" + + def entryPointMethod(): String = "doGet" +} diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchAllTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchAllTest.scala new file mode 100644 index 00000000..11ffe8cd --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchAllTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchAllTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchArraysTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchArraysTest.scala new file mode 100644 index 00000000..1df57925 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchArraysTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchArraysTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.arrays" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchBasicTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchBasicTest.scala new file mode 100644 index 00000000..c495c606 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchBasicTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchBasicTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.basic" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchCollectionsTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchCollectionsTest.scala new file mode 100644 index 00000000..dbd8f031 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchCollectionsTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchCollectionsTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.collections" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchDatastructuresTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchDatastructuresTest.scala new file mode 100644 index 00000000..a33b6210 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchDatastructuresTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchDatastructuresTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.datastructures" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchFactoriesTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchFactoriesTest.scala new file mode 100644 index 00000000..999b5a06 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchFactoriesTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchFactoriesTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.factories" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchInterTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchInterTest.scala new file mode 100644 index 00000000..bd644886 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchInterTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchInterTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.inter" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchPredTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchPredTest.scala new file mode 100644 index 00000000..124aa72f --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchPredTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchPredTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.pred" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchReflectionTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchReflectionTest.scala new file mode 100644 index 00000000..a5354fa5 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchReflectionTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchReflectionTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.reflection" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchSanitizersTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchSanitizersTest.scala new file mode 100644 index 00000000..d8d10cab --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchSanitizersTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchSanitizersTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.sanitizers" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchSessionTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchSessionTest.scala new file mode 100644 index 00000000..3190ec4a --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchSessionTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchSessionTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.session" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/securibench/tests/SecuribenchStrongUpdatesTest.scala b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchStrongUpdatesTest.scala new file mode 100644 index 00000000..5400d007 --- /dev/null +++ b/src/test/scala/br/unb/cic/securibench/tests/SecuribenchStrongUpdatesTest.scala @@ -0,0 +1,9 @@ +package br.unb.cic.securibench.tests + +import br.unb.cic.securibench.SecuribenchTest + +class SecuribenchStrongUpdatesTest extends SecuribenchTest { + def basePackage(): String = "securibench.micro.strong_updates" + + def entryPointMethod(): String = "doGet" +} \ No newline at end of file diff --git a/src/test/scala/br/unb/cic/soot/JSVFATest.scala b/src/test/scala/br/unb/cic/soot/JSVFATest.scala index 9057007d..6f6eaabc 100644 --- a/src/test/scala/br/unb/cic/soot/JSVFATest.scala +++ b/src/test/scala/br/unb/cic/soot/JSVFATest.scala @@ -9,7 +9,8 @@ abstract class JSVFATest extends JSVFA with Interprocedural with FieldSensitive override def sootClassPath(): String = "" - override def applicationClassPath(): List[String] = List("target/scala-2.12/test-classes", System.getProperty("user.home")+"/.m2/repository/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar") + //TO-DO: It must be done dinamically + override def applicationClassPath(): List[String] = List("target/scala-2.12/test-classes", System.getProperty("user.home")+"/.m2/repository/javax/servlet/javax.servlet-api/3.0.1/javax.servlet-api-3.0.1.jar") override def getEntryPoints(): List[SootMethod] = { val sootClass = Scene.v().getSootClass(getClassName())