From 9fb1313331dfb44d8daf1f1341c0d440e8e5d130 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 2 Dec 2025 16:38:45 -0500 Subject: [PATCH 1/7] use run-taintbench.sh --- .github/workflows/test_suite.yml | 2 +- README.md | 30 ++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index 121dec98..822bfcc3 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -82,4 +82,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sbt "testOnly br.unb.cic.securibench.deprecated.SecuribenchTestSuite" - ./run-tests.sh --android-sdk $RUNNER_TEMP/android-sdk --taint-bench $RUNNER_TEMP/TaintBench AndroidTaintBenchSuiteExperiment1 + ./scripts/run-taintbench.sh --android-sdk $RUNNER_TEMP/android-sdk --taint-bench $RUNNER_TEMP/TaintBench AndroidTaintBenchSuiteExperiment1 diff --git a/README.md b/README.md index 83c1bb3c..e74a8223 100644 --- a/README.md +++ b/README.md @@ -261,7 +261,29 @@ To have detailed information about each test category run, [see here.](modules/s To have detailed information about each test category run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md) (*computed in November 2025.*) -##### Common issues +#### Running Android Tests + +You can run Android tests in several ways: + +**1. Using the convenience shell script (Recommended):** +```bash +./scripts/run-taintbench.sh --android-sdk /path/to/android/sdk --taint-bench /path/to/taintbench roidsec +``` + +**2. Using environment variables:** +```bash +sbt test +# Or run specific tests: +sbt testRoidsec +sbt testAndroid +``` + +**5. Using SBT testOnly command:** +```bash +sbt "testOnly br.unb.cic.securibench.suite.SecuribenchSuiteTest" +``` + +#### Common issues From the 47 tests, we have categorized nine (9) issues. [i] **Wrong counting**: Some tests from the Securibench benchmark are incorrectly labeled, leading to wrong expected values. @@ -422,9 +444,9 @@ You can run Android tests in several ways: **1. Using the convenience shell script (Recommended):** ```bash -./run-tests.sh --android-sdk /path/to/android/sdk --taint-bench /path/to/taintbench roidsec -./run-tests.sh --android-sdk /path/to/android/sdk --taint-bench /path/to/taintbench android -./run-tests.sh --android-sdk /path/to/android/sdk --taint-bench /path/to/taintbench all +./scripts/run-taintbench.sh --android-sdk /path/to/android/sdk --taint-bench /path/to/taintbench roidsec +./scripts/run-taintbench.sh --android-sdk /path/to/android/sdk --taint-bench /path/to/taintbench android +./scripts/run-taintbench.sh --android-sdk /path/to/android/sdk --taint-bench /path/to/taintbench all ``` **2. Using environment variables:** From bbf69dcde5a72b059aee6389bd952e5c922255c9 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 2 Dec 2025 16:38:52 -0500 Subject: [PATCH 2/7] remove unused script --- run-tests.sh | 156 --------------------------------------------------- 1 file changed, 156 deletions(-) delete mode 100755 run-tests.sh diff --git a/run-tests.sh b/run-tests.sh deleted file mode 100755 index 466828c7..00000000 --- a/run-tests.sh +++ /dev/null @@ -1,156 +0,0 @@ -#!/bin/bash - -# SVFA Test Runner -# This script provides convenient ways to run tests with environment variables - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Function to print usage -usage() { - echo -e "${BLUE}SVFA Test Runner${NC}" - echo "" - echo "Usage: $0 [OPTIONS] [TEST_NAME]" - echo "" - echo "Options:" - echo " --android-sdk PATH Path to Android SDK (required)" - echo " --taint-bench PATH Path to TaintBench dataset (required)" - echo " --help Show this help message" - echo "" - echo "Test Names:" - echo " roidsec Run RoidsecTest" - echo " android Run all Android tests" - echo " all Run all tests" - echo "" - echo "Examples:" - echo " $0 --android-sdk /opt/android-sdk --taint-bench /opt/taintbench roidsec" - echo " $0 --android-sdk \$ANDROID_HOME --taint-bench \$HOME/taintbench android" - echo "" - echo "Environment Variables (alternative to command line options):" - echo " ANDROID_SDK Path to Android SDK" - echo " TAINT_BENCH Path to TaintBench dataset" -} - -# Default values -ANDROID_SDK="" -TAINT_BENCH="" -TEST_NAME="" - -# Parse command line arguments -while [[ $# -gt 0 ]]; do - case $1 in - --android-sdk) - ANDROID_SDK="$2" - shift 2 - ;; - --taint-bench) - TAINT_BENCH="$2" - shift 2 - ;; - --help) - usage - exit 0 - ;; - roidsec|AndroidTaintBenchSuiteExperiment1|AndroidTaintBenchSuiteExperiment2|android|all) - TEST_NAME="$1" - shift - ;; - *) - echo -e "${RED}Unknown option: $1${NC}" - usage - exit 1 - ;; - esac -done - -# Use environment variables as fallback -if [[ -z "$ANDROID_SDK" && -n "$ANDROID_SDK_ENV" ]]; then - ANDROID_SDK="$ANDROID_SDK_ENV" -fi - -if [[ -z "$TAINT_BENCH" && -n "$TAINT_BENCH_ENV" ]]; then - TAINT_BENCH="$TAINT_BENCH_ENV" -fi - -# Check if required parameters are provided -if [[ -z "$ANDROID_SDK" ]]; then - echo -e "${RED}Error: Android SDK path is required${NC}" - echo "Use --android-sdk /path/to/sdk or set ANDROID_SDK environment variable" - exit 1 -fi - -if [[ -z "$TAINT_BENCH" ]]; then - echo -e "${RED}Error: TaintBench path is required${NC}" - echo "Use --taint-bench /path/to/taintbench or set TAINT_BENCH environment variable" - exit 1 -fi - -if [[ -z "$TEST_NAME" ]]; then - echo -e "${RED}Error: Test name is required${NC}" - usage - exit 1 -fi - -# Validate paths -if [[ ! -d "$ANDROID_SDK" ]]; then - echo -e "${RED}Error: Android SDK path does not exist: $ANDROID_SDK${NC}" - exit 1 -fi - -if [[ ! -d "$TAINT_BENCH" ]]; then - echo -e "${RED}Error: TaintBench path does not exist: $TAINT_BENCH${NC}" - exit 1 -fi - -# Export environment variables -export ANDROID_SDK="$ANDROID_SDK" -export TAINT_BENCH="$TAINT_BENCH" - -echo -e "${GREEN}Running SVFA tests with:${NC}" -echo -e " ${BLUE}Android SDK:${NC} $ANDROID_SDK" -echo -e " ${BLUE}TaintBench:${NC} $TAINT_BENCH" -echo -e " ${BLUE}Test:${NC} $TEST_NAME" -echo "" - -# Run the appropriate test -case $TEST_NAME in - roidsec) - echo -e "${YELLOW}Running RoidsecTest...${NC}" - sbt "taintbench/testOnly br.unb.cic.android.RoidsecTest" - ;; - AndroidTaintBenchSuiteExperiment1) - echo -e "${GREEN}Running AndroidTaintBenchSuiteExperiment1Test specifically...${NC}" - sbt AndroidTaintBenchSuiteExperiment1Test - ;; - AndroidTaintBenchSuiteExperiment2) - echo -e "${GREEN}Running AndroidTaintBenchSuiteExperiment2Test specifically...${NC}" - sbt AndroidTaintBenchSuiteExperiment2Test - ;; - android) - echo -e "${YELLOW}Running all Android tests...${NC}" - sbt "taintbench/testOnly br.unb.cic.android.*" - ;; - all) - echo -e "${YELLOW}Running all tests...${NC}" - sbt taintbench/test - ;; - *) - echo -e "${RED}Unknown test name: $TEST_NAME${NC}" - usage - exit 1 - ;; -esac - -echo -e "${GREEN}Tests completed successfully!${NC}" - - - - - - From b61b31ff2173216e681ae1b7a038eefe804d3f55 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 2 Dec 2025 17:11:04 -0500 Subject: [PATCH 3/7] add info Running Securibench Tests --- README.md | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index e74a8223..27b5b01b 100644 --- a/README.md +++ b/README.md @@ -261,26 +261,18 @@ To have detailed information about each test category run, [see here.](modules/s To have detailed information about each test category run, [see here.](modules/securibench/src/docs-metrics/jsvfa/jsvfa-metrics-v0.6.1.md) (*computed in November 2025.*) -#### Running Android Tests +#### Running Securibench Tests -You can run Android tests in several ways: +You can run Securibench tests in two ways: **1. Using the convenience shell script (Recommended):** ```bash -./scripts/run-taintbench.sh --android-sdk /path/to/android/sdk --taint-bench /path/to/taintbench roidsec -``` - -**2. Using environment variables:** -```bash -sbt test -# Or run specific tests: -sbt testRoidsec -sbt testAndroid +./scripts/run-securibench.sh ``` -**5. Using SBT testOnly command:** +**2. Using SBT testOnly command:** ```bash -sbt "testOnly br.unb.cic.securibench.suite.SecuribenchSuiteTest" +sbt "testOnly br.unb.cic.securibench.deprecated.SecuribenchTestSuite" ``` #### Common issues From ae6855093108802f1042a692588a4bd4240cd9de Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 2 Dec 2025 17:28:03 -0500 Subject: [PATCH 4/7] rename var environment --- .github/workflows/test_suite.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index 822bfcc3..74d4d774 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -54,19 +54,19 @@ jobs: - name: Set up Android SDK env: - ANDROID_SDK_ROOT: ${{ runner.temp }}/android-sdk + ANDROID_SDK: ${{ runner.temp }}/android-sdk run: | - mkdir -p "$ANDROID_SDK_ROOT/cmdline-tools" + mkdir -p "$ANDROID_SDK/cmdline-tools" # Downgraded to commandlinetools-linux-7583922_latest.zip for Java 8 compatibility # Use cache for Android command line tools if [ ! -f "$RUNNER_TEMP/cmdline-tools.zip" ]; then curl -fo "$RUNNER_TEMP/cmdline-tools.zip" https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip fi - unzip -q -o "$RUNNER_TEMP/cmdline-tools.zip" -d "$ANDROID_SDK_ROOT/cmdline-tools" - mv "$ANDROID_SDK_ROOT/cmdline-tools/cmdline-tools" "$ANDROID_SDK_ROOT/cmdline-tools/latest" - yes | "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager" --licenses - "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager" "platform-tools" "platforms;android-30" "build-tools;30.0.3" + unzip -q -o "$RUNNER_TEMP/cmdline-tools.zip" -d "$ANDROID_SDK/cmdline-tools" + mv "$ANDROID_SDK/cmdline-tools/cmdline-tools" "$ANDROID_SDK/cmdline-tools/latest" + yes | "$ANDROID_SDK/cmdline-tools/latest/bin/sdkmanager" --licenses + "$ANDROID_SDK/cmdline-tools/latest/bin/sdkmanager" "platform-tools" "platforms;android-30" "build-tools;30.0.3" - name: Download TaintBench suite run: | From ade673f7a5235f64839c72b1882f2dfc31f0d107 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 2 Dec 2025 17:41:24 -0500 Subject: [PATCH 5/7] comment test from taintbench in pipeline --- .github/workflows/test_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index 74d4d774..17418604 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -82,4 +82,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | sbt "testOnly br.unb.cic.securibench.deprecated.SecuribenchTestSuite" - ./scripts/run-taintbench.sh --android-sdk $RUNNER_TEMP/android-sdk --taint-bench $RUNNER_TEMP/TaintBench AndroidTaintBenchSuiteExperiment1 +# ./scripts/run-taintbench.sh --android-sdk $RUNNER_TEMP/android-sdk --taint-bench $RUNNER_TEMP/TaintBench AndroidTaintBenchSuiteExperiment1 From 41815e64f3f3eaa6ac4e78423e8d239bc4cf45cd Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 2 Dec 2025 17:46:50 -0500 Subject: [PATCH 6/7] run ./scripts/run-securibench.sh --- .github/workflows/test_suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index 17418604..eeae4863 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -81,5 +81,5 @@ jobs: # Only set GITHUB_TOKEN if it's available in secrets (for CI) GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - sbt "testOnly br.unb.cic.securibench.deprecated.SecuribenchTestSuite" + ./scripts/run-securibench.sh # ./scripts/run-taintbench.sh --android-sdk $RUNNER_TEMP/android-sdk --taint-bench $RUNNER_TEMP/TaintBench AndroidTaintBenchSuiteExperiment1 From 273421767bba53a823002af17390a14a01b40b56 Mon Sep 17 00:00:00 2001 From: jose clavo tafur Date: Tue, 2 Dec 2025 17:59:33 -0500 Subject: [PATCH 7/7] reverse changes --- .github/workflows/test_suite.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_suite.yml b/.github/workflows/test_suite.yml index eeae4863..a0ba41f3 100644 --- a/.github/workflows/test_suite.yml +++ b/.github/workflows/test_suite.yml @@ -54,19 +54,19 @@ jobs: - name: Set up Android SDK env: - ANDROID_SDK: ${{ runner.temp }}/android-sdk + ANDROID_SDK_ROOT: ${{ runner.temp }}/android-sdk run: | - mkdir -p "$ANDROID_SDK/cmdline-tools" + mkdir -p "$ANDROID_SDK_ROOT/cmdline-tools" # Downgraded to commandlinetools-linux-7583922_latest.zip for Java 8 compatibility # Use cache for Android command line tools if [ ! -f "$RUNNER_TEMP/cmdline-tools.zip" ]; then curl -fo "$RUNNER_TEMP/cmdline-tools.zip" https://dl.google.com/android/repository/commandlinetools-linux-7583922_latest.zip fi - unzip -q -o "$RUNNER_TEMP/cmdline-tools.zip" -d "$ANDROID_SDK/cmdline-tools" - mv "$ANDROID_SDK/cmdline-tools/cmdline-tools" "$ANDROID_SDK/cmdline-tools/latest" - yes | "$ANDROID_SDK/cmdline-tools/latest/bin/sdkmanager" --licenses - "$ANDROID_SDK/cmdline-tools/latest/bin/sdkmanager" "platform-tools" "platforms;android-30" "build-tools;30.0.3" + unzip -q -o "$RUNNER_TEMP/cmdline-tools.zip" -d "$ANDROID_SDK_ROOT/cmdline-tools" + mv "$ANDROID_SDK_ROOT/cmdline-tools/cmdline-tools" "$ANDROID_SDK_ROOT/cmdline-tools/latest" + yes | "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager" --licenses + "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager" "platform-tools" "platforms;android-30" "build-tools;30.0.3" - name: Download TaintBench suite run: |