diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 6d033257..d770f47f 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -3,11 +3,11 @@ name: Python Tests on: push: paths: - - 'src/python/**' + - 'futag-package/**' - '.github/workflows/python-tests.yml' pull_request: paths: - - 'src/python/**' + - 'futag-package/**' jobs: test: @@ -22,9 +22,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install package with test dependencies run: | - cd src/python/futag-package + cd futag-package pip install -e ".[test]" - name: Run tests run: | - cd src/python/futag-package + cd futag-package python -m pytest tests/ -v --tb=short diff --git a/.github/workflows/syntax-check.yml b/.github/workflows/syntax-check.yml index 8beea1f3..801577d9 100644 --- a/.github/workflows/syntax-check.yml +++ b/.github/workflows/syntax-check.yml @@ -15,7 +15,7 @@ jobs: python -c " import ast, sys, pathlib errors = [] - for f in pathlib.Path('src/python/futag-package/src/futag').glob('*.py'): + for f in pathlib.Path('futag-package/src/futag').glob('*.py'): try: ast.parse(f.read_text()) except SyntaxError as e: diff --git a/.gitignore b/.gitignore index 188a4c46..471278b0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,28 +7,28 @@ futag-llvm futag-install futag-build futag-build/** -custom-llvm/INFO -custom-llvm/*.tar.xz -custom-llvm/futag-llvm/** -custom-llvm/AFLplusplus-4.02c -custom-llvm/binutils -custom-llvm/llvm-project-14.0.6.src.tar.xz -custom-llvm/llvm-project -custom-llvm/fuzz-introspector -custom-llvm/*.tar.gz -custom-llvm/*.tar.xz +build-llvm/INFO +build-llvm/*.tar.xz +build-llvm/futag-llvm/** +build-llvm/AFLplusplus-4.02c +build-llvm/binutils +build-llvm/llvm-project-14.0.6.src.tar.xz +build-llvm/llvm-project +build-llvm/fuzz-introspector +build-llvm/*.tar.gz +build-llvm/*.tar.xz **/futag-llvm.*.latest.tar.xz **/futag-llvm.*.tar.xz TODO -product-tests/*.tar.xz -src/Checkers/lib/CMakeLists.txt.test -src/Checkers/include/Checkers.td.test -src/Checkers/lib/FutagTest.cpp +integration-tests/*.tar.xz +analyzers/checkers/lib/CMakeLists.txt.test +analyzers/checkers/include/Checkers.td.test +analyzers/checkers/lib/FutagTest.cpp futag-llvm* Algorithms -src/python/futag-package/dist/** -src/python/futag-package/build/** -src/python/futag-package/src/*.egg-info/ +futag-package/dist/** +futag-package/build/** +futag-package/src/*.egg-info/ # Python __pycache__/ diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 00000000..388c4596 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,125 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## Project Overview + +FUTAG (Fuzz target Automated Generator) is a tool from ISP RAS for automated generation of fuzzing wrappers (fuzz targets) for software libraries. It analyzes library source code via custom Clang/LLVM static analysis checkers and generates fuzz targets in LibFuzzer or AFLplusplus format. + +## Build Commands + +### Building the custom LLVM/Clang toolchain + +```bash +# 1. Download LLVM sources (interactive - select LLVM version) +cd build-llvm && ./prepare.sh + +# 2. Build LLVM with Futag checkers integrated +cd ../build && ./build.sh + +# Output: futag-llvm/ directory with compiled toolchain +``` + +### Building AFLplusplus support (optional) + +```bash +cd futag-llvm && ./buildAFLplusplus.sh +``` + +### Installing the Python package + +```bash +cd futag-package && pip install . +``` + +### Running integration tests (Docker) + +Tests are Dockerized per platform in `integration-tests/`: +- `integration-tests/build-test/` — build validation on Ubuntu 20.04/22.04/24.04, Alt 11/12 +- `integration-tests/libraries-test/` — end-to-end tests against real libraries (json-c, php, FreeImage, etc.) +- `integration-tests/package-test/` — Python package tests + +## Architecture + +### Three-layer design + +1. **C++ Clang Checkers** (`analyzers/checkers/`, `analyzers/clang-patches/`) — Static analysis plugins that run inside Clang's StaticAnalyzer to extract function signatures, types, and usage patterns from library code. Version-specific implementations exist for LLVM 14 and 18 (files suffixed `14`/`18`). + +2. **Python Orchestration** (`futag-package/src/futag/`) — User-facing API that drives the full pipeline: + - `preprocessor.py` — `Builder` builds and analyzes target libraries; `ConsumerBuilder` handles library+consumer pairs + - `generator.py` — `Generator` produces fuzz targets from analysis JSON; `ContextGenerator` uses consumer usage contexts + - `fuzzer.py` — `Fuzzer` executes generated targets with configurable timeouts, memory limits, and sanitizers + - `sysmsg.py` — Constants and error messages (LIBFUZZER, AFLPLUSPLUS engine identifiers, paths) + +3. **Build Infrastructure** (`build-llvm/`) — Shell scripts that download LLVM sources, patch in Futag's checkers and clang modifications, and build the complete toolchain via CMake. + +### Data flow + +`Builder.auto_build()` → `Builder.analyze()` (runs Clang checkers, produces JSON) → `Generator.gen_targets()` (reads JSON, writes C fuzz targets) → `Generator.compile_targets()` (compiles with instrumentation) → `Fuzzer.fuzz()` (runs fuzzing) + +### LLVM version handling + +The project maintains version-specific copies of several files (e.g., `FutagAnalyzer14.cpp` / `FutagAnalyzer18.cpp`, `ASTMatchFinder14.cpp` / `ASTMatchFinder18.cpp`). The build script (`build.sh`) selects the correct version-specific `CMakeLists.txt` and source files during LLVM compilation. + +### Key third-party code + +- `vendors/json/` — nlohmann/json for C++ JSON serialization in checkers +- `.clang-format` — LLVM style formatting for C++ code + +### Recent refactoring (2024) + +The Python package underwent major refactoring: +- **BaseGenerator ABC** (`base_generator.py`) — shared infrastructure for all 5 generator subclasses +- **GeneratorState** (`generator_state.py`) — dataclass replacing 13 mutable instance variables +- **BaseFuzzer** (`fuzzer.py`) — shared fuzzer logic; Fuzzer and NatchFuzzer are thin subclasses +- **Custom exceptions** (`exceptions.py`) — FutagError hierarchy replacing sys.exit() +- Generator classes use single-underscore protected methods for proper inheritance + +### Running Python tests + +```bash +cd futag-package +pip install -e ".[test]" +python -m pytest tests/ -v +``` + +### Python module structure + +``` +src/futag/ + base_generator.py # BaseGenerator ABC (~2500 lines) + generator.py # Generator + re-exports (~235 lines) + fdp_generator.py # FuzzDataProviderGenerator (~220 lines) + blob_stamper_generator.py # BlobStamperGenerator (~40 lines) + context_generator.py # ContextGenerator (~820 lines) + natch_generator.py # NatchGenerator (~1240 lines) + generator_state.py # GeneratorState dataclass (~85 lines) + preprocessor.py # Builder, ConsumerBuilder (~960 lines) + fuzzer.py # BaseFuzzer, Fuzzer, NatchFuzzer (~900 lines) + toolchain.py # ToolchainConfig dataclass (~140 lines) + exceptions.py # FutagError hierarchy + sysmsg.py # Constants and messages +``` + +## System Requirements + +- CMake >= 3.13.4, GCC >= 7.1.0, Python >= 3.8, pip >= 22.1.1 +- On Ubuntu: `python-is-python3`, `gcc-multilib`, `binutils-gold`, `binutils-dev` + +## Typical Python API usage + +```python +from futag.preprocessor import * +from futag.generator import * +from futag.toolchain import ToolchainConfig + +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) +builder = Builder(library_root, clean=True, toolchain=tc) +builder.auto_build() +builder.analyze() +generator = Generator(library_root, toolchain=tc) +generator.gen_targets(anonymous=False, max_wrappers=10) +generator.compile_targets(4) # parallel workers +``` + +See `scripts/template-script.py` for a complete workflow example and `workshop/` for library-specific tutorials. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 130d8ffd..5ad23b3c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,21 +11,21 @@ ### Building the LLVM toolchain ```bash -cd custom-llvm && ./prepare.sh +cd build-llvm && ./prepare.sh cd ../build && ./build.sh ``` ### Installing the Python package (development mode) ```bash -cd src/python/futag-package +cd futag-package pip install -e ".[test]" ``` ### Running tests ```bash -cd src/python/futag-package +cd futag-package python -m pytest tests/ -v ``` diff --git a/README.en.md b/README.en.md index 0be1686b..2f2e71a9 100644 --- a/README.en.md +++ b/README.en.md @@ -42,7 +42,7 @@ graph TD end subgraph "Layer 1: Build Infrastructure" - D["custom-llvm / build.sh — Download and patch LLVM 14/18/19"] + D["build-llvm / build.sh — Download and patch LLVM 14/18/19"] end D -->|"futag-llvm toolchain"| E @@ -106,7 +106,7 @@ Thank you for acknowledging the authors' work when you use FUTAG or report bugs # 3. Installation ## 3.1. Using the Docker container -You can try building FUTAG using the provided Dockerfiles for Ubuntu: https://github.com/ispras/Futag/tree/main/product-tests/build-test +You can try building FUTAG using the provided Dockerfiles for Ubuntu: https://github.com/ispras/Futag/tree/main/integration-tests/build-test ## 3.2. Using a prepackaged release - Download the latest release (for example, futag-llvm.3.0.1.tar.xz) from https://github.com/ispras/Futag/releases/tag/v3.0.1 and extract it. The tool will be installed to the futag-llvm directory. @@ -145,12 +145,12 @@ On Ubuntu you may also need to install: ```bash ~$ git clone https://github.com/ispras/Futag ``` -- Prepare the "custom-llvm" directory by running the script: +- Prepare the "build-llvm" directory by running the script: ```bash - ~/Futag/custom-llvm$ ./prepare.sh + ~/Futag/build-llvm$ ./prepare.sh ``` -This script creates the Futag/build directory and copies Futag/custom-llvm/build.sh into it. +This script creates the Futag/build directory and copies Futag/build-llvm/build.sh into it. - Run the copied build script in "Futag/build": @@ -176,45 +176,41 @@ This script creates the Futag/build directory and copies Futag/custom-llvm/build ``` ### 4.1. Automatic generation of fuzzing wrappers when usage contexts are absent -- Run the build, check and analysis when no usage contexts exist: ```python from futag.preprocessor import * +from futag.generator import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "/home/futag/Futag-tests/futag-llvm/" lib_path = "path/to/library/source/code" + +# --- Create toolchain --- +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) + +# --- Build and analyze the library --- build_test = Builder( - FUTAG_PATH, lib_path, clean=True, # remove all folders generated by FUTAG before building # intercept=True, # enable compilation with the "intercept" tool to analyze compile_command.json # processes=4, # number of build jobs # build_ex_params="--with-openssl --with-mhash" # extra params for library build + toolchain=tc, ) build_test.auto_build() build_test.analyze() -``` - -- Generate and compile drivers: - -```python -from futag.generator import * - -FUTAG_PATH = "/home/futag/Futag-tests/futag-llvm/" -lib_path = "path/to/library/source/code" +# --- Generate and compile drivers --- generator = Generator( - FUTAG_PATH, # path to the "futag-llvm" directory lib_path, # path to the directory containing the target software source code - # target_type=AFLPLUSPLUS, + # target_type=AFLPLUSPLUS, + toolchain=tc, ) -# Generate fuzzing wrappers generator.gen_targets( anonymous=False, # option to generate wrappers for non-public functions - max_wrappers=10 # limit the number of generated wrappers per function + max_wrappers=10, # limit the number of generated wrappers per function ) -# Compile fuzz drivers generator.compile_targets( 4, # number of build jobs # keep_failed=True, # keep failed targets @@ -230,25 +226,27 @@ By default, successfully compiled fuzzing wrappers for target functions are plac ```python from futag.preprocessor import * -from futag.generator import * -from futag.fuzzer import * +from futag.generator import * +from futag.fuzzer import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "/home/futag/Futag/futag-llvm" library_root = "json-c-json-c-0.16-20220414" consumer_root = "libstorj-1.0.3" consumber_builder = ConsumerBuilder( - FUTAG_PATH, # path to the "futag-llvm" directory library_root, # path to the directory with the tested library's source code consumer_root, # path to the directory with the consumer application's source code + toolchain=tc, # clean=True, # processes=16, ) consumber_builder.auto_build() consumber_builder.analyze() +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) context_generator = ContextGenerator( - FUTAG_PATH, - library_root, + library_root, + toolchain=tc, ) context_generator.gen_context() # generate fuzzing wrappers for contexts @@ -258,13 +256,13 @@ context_generator.compile_targets( # compile generated fuzzing wrappers ``` If multiple fuzzing wrappers are generated for a function, the target function's subdirectory will contain numbered subdirectories (appended to the function name). -Python package documentation is available at: https://github.com/ispras/Futag/tree/main/src/python/futag-package +Python package documentation is available at: https://github.com/ispras/Futag/tree/main/futag-package -More information about using FUTAG is available at: https://github.com/ispras/Futag/blob/main/How-to-work-with-Futag.md +More information about using FUTAG is available at: https://github.com/ispras/Futag/blob/main/docs/how-to-work-with-futag.md -A template for run scripts can be found here: https://github.com/ispras/Futag/blob/main/src/python/template-script.py +A template for run scripts can be found here: https://github.com/ispras/Futag/blob/main/scripts/template-script.py -A test repository was created at https://github.com/thientc/Futag-tests to test FUTAG on various libraries (json-c, php, FreeImage, etc.). You can try testing using the Docker container at https://github.com/ispras/Futag/tree/main/product-tests/libraries-test. +A test repository was created at https://github.com/thientc/Futag-tests to test FUTAG on various libraries (json-c, php, FreeImage, etc.). You can try testing using the Docker container at https://github.com/ispras/Futag/tree/main/integration-tests/libraries-test. ## Documentation diff --git a/README.md b/README.md index d4f5933e..27db8256 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ graph TD end subgraph "Уровень 1: Инфраструктура сборки" - D["custom-llvm / build.sh — Загрузка и патч LLVM 14/18/19"] + D["build-llvm / build.sh — Загрузка и патч LLVM 14/18/19"] end D -->|"инструментарий futag-llvm"| E @@ -92,7 +92,7 @@ graph TD # 2. Установка ## 2.1. Использование докер-контейнера -Вы можете попробовать собрать Futag с готовыми [Докер-файлами](https://github.com/ispras/Futag/tree/main/product-tests/build-test) для ОС Ubuntu. +Вы можете попробовать собрать Futag с готовыми [Докер-файлами](https://github.com/ispras/Futag/tree/main/integration-tests/build-test) для ОС Ubuntu. ## 2.2. Использование предварительно упакованного пакета - Загрузите последнюю версию [futag-llvm.3.0.1.tar.xz](https://github.com/ispras/Futag/releases/tag/v3.0.1) и разархивируйте. В результате инструмент будет установлен в директорию futag-llvm. @@ -131,11 +131,11 @@ graph TD ```bash ~$ git clone https://github.com/ispras/Futag ``` -- Подготовьте директорию "custom-llvm", запустив скрипт: +- Подготовьте директорию "build-llvm", запустив скрипт: ```bash - ~/Futag/custom-llvm$ ./prepare.sh + ~/Futag/build-llvm$ ./prepare.sh ``` -Этот скрипт создает директорию Futag/build и копирует в неё скрипт Futag/custom-llvm/build.sh +Этот скрипт создает директорию Futag/build и копирует в неё скрипт Futag/build-llvm/build.sh - Запустите скопированный скрипт в "Futag/build": @@ -162,45 +162,41 @@ graph TD ``` ## 3.1. Автоматическая генерация фаззинг-оберток в условии отсутствия контекстов использования -- Запуск сборки, проверки и анализа в условии отсутствия контекстов использования ```python from futag.preprocessor import * +from futag.generator import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "/home/futag/Futag-tests/futag-llvm/" lib_path = "path/to/library/source/code" + +# --- Создание toolchain --- +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) + +# --- Сборка и анализ библиотеки --- build_test = Builder( - FUTAG_PATH, lib_path, clean=True, # удалить все папки сгенерированные Futag-ом перед сборкой # intercept=True, # запуск компиляции с инструментом "intercept" для анализа compile_command.json # processes=4, # количество задач при сборке # build_ex_params="--with-openssl --with-mhash" # дополнительные параметры при сборке библиотеки + toolchain=tc, ) build_test.auto_build() build_test.analyze() -``` - -- Генерация и компиляция драйверов - -```python -from futag.generator import * - -FUTAG_PATH = "/home/futag/Futag-tests/futag-llvm/" -lib_path = "path/to/library/source/code" +# --- Генерация и компиляция драйверов --- generator = Generator( - FUTAG_PATH, # путь к директории "futag-llvm" lib_path, # путь к директории содержащей исходные кода исследуемого ПО - # target_type=AFLPLUSPLUS, + # target_type=AFLPLUSPLUS, + toolchain=tc, ) -# Генерация фаззинг-оберток generator.gen_targets( - anonymous=False # опция для генерации фаззинг-обертки для функций, которые не имеют публичный доступ - max_wrappers= 10 # опция для органичения количества сгенерированных фаззинг-оберток для одной функции + anonymous=False, # опция для генерации фаззинг-обертки для функций, которые не имеют публичный доступ + max_wrappers=10, # опция для органичения количества сгенерированных фаззинг-оберток для одной функции ) -# Compile fuzz drivers generator.compile_targets( 4, # количество задач при сборке # keep_failed=True, # сохранить не скомпилированные цели @@ -215,25 +211,27 @@ generator.compile_targets( ```python from futag.preprocessor import * -from futag.generator import * -from futag.fuzzer import * +from futag.generator import * +from futag.fuzzer import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "/home/futag/Futag/futag-llvm" library_root = "json-c-json-c-0.16-20220414" consumer_root = "libstorj-1.0.3" consumber_builder = ConsumerBuilder( - FUTAG_PATH, # путь к директории "futag-llvm" library_root, # путь к директории содержащей исходные кода тестируемой библиотеки consumer_root, # путь к директории содержащей исходные кода потребительской программы + toolchain=tc, # clean=True, # processes=16, ) consumber_builder.auto_build() consumber_builder.analyze() +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) context_generator = ContextGenerator( - FUTAG_PATH, - library_root, + library_root, + toolchain=tc, ) context_generator.gen_context() # генерация фаззинг-оберток для контекстов @@ -243,13 +241,13 @@ context_generator.compile_targets( #компиляция сгенерирова ``` Если для функции сгенерировалось несколько фаззинг-оберток, в подкаталоге целевой функции создаются соответствующие директории, где к имени целевой функции добавляется порядковый номер. -Документация Python-пакета находится [по ссылке](https://github.com/ispras/Futag/tree/main/src/python/futag-package) +Документация Python-пакета находится [по ссылке](https://github.com/ispras/Futag/tree/main/futag-package) -Подобную информацию о работе Futag можно прочитать [по ссылке](https://github.com/ispras/Futag/blob/main/How-to-work-with-Futag.md) +Подобную информацию о работе Futag можно прочитать [по ссылке](https://github.com/ispras/Futag/blob/main/docs/how-to-work-with-futag.md) -Шаблон скриптов запуска можно посмотреть [здесь](https://github.com/ispras/Futag/blob/main/src/python/template-script.py) +Шаблон скриптов запуска можно посмотреть [здесь](https://github.com/ispras/Futag/blob/main/scripts/template-script.py) -Был создан [репозиторий](https://github.com/thientc/Futag-tests) для тестирования Futag над библиотеками (json-c, php, FreeImage, и т.д.), можете протестировать с [Докер-контейнером](https://github.com/ispras/Futag/tree/main/product-tests/libraries-test). +Был создан [репозиторий](https://github.com/thientc/Futag-tests) для тестирования Futag над библиотеками (json-c, php, FreeImage, и т.д.), можете протестировать с [Докер-контейнером](https://github.com/ispras/Futag/tree/main/integration-tests/libraries-test). ## Документация diff --git a/src/Checkers/include/Checkers.td b/analyzers/checkers/include/Checkers.td similarity index 100% rename from src/Checkers/include/Checkers.td rename to analyzers/checkers/include/Checkers.td diff --git a/src/Checkers/include/Checkers14.td b/analyzers/checkers/include/Checkers14.td similarity index 100% rename from src/Checkers/include/Checkers14.td rename to analyzers/checkers/include/Checkers14.td diff --git a/src/Checkers/include/Checkers18.td b/analyzers/checkers/include/Checkers18.td similarity index 100% rename from src/Checkers/include/Checkers18.td rename to analyzers/checkers/include/Checkers18.td diff --git a/src/Checkers/lib/CMakeLists.txt b/analyzers/checkers/lib/CMakeLists.txt similarity index 100% rename from src/Checkers/lib/CMakeLists.txt rename to analyzers/checkers/lib/CMakeLists.txt diff --git a/src/Checkers/lib/CMakeLists14.txt b/analyzers/checkers/lib/CMakeLists14.txt similarity index 100% rename from src/Checkers/lib/CMakeLists14.txt rename to analyzers/checkers/lib/CMakeLists14.txt diff --git a/src/Checkers/lib/CMakeLists18.txt b/analyzers/checkers/lib/CMakeLists18.txt similarity index 100% rename from src/Checkers/lib/CMakeLists18.txt rename to analyzers/checkers/lib/CMakeLists18.txt diff --git a/src/Checkers/lib/FutagAnalyzer.cpp b/analyzers/checkers/lib/FutagAnalyzer.cpp similarity index 100% rename from src/Checkers/lib/FutagAnalyzer.cpp rename to analyzers/checkers/lib/FutagAnalyzer.cpp diff --git a/src/Checkers/lib/FutagAnalyzer14.cpp b/analyzers/checkers/lib/FutagAnalyzer14.cpp similarity index 100% rename from src/Checkers/lib/FutagAnalyzer14.cpp rename to analyzers/checkers/lib/FutagAnalyzer14.cpp diff --git a/src/Checkers/lib/FutagAnalyzer18.cpp b/analyzers/checkers/lib/FutagAnalyzer18.cpp similarity index 100% rename from src/Checkers/lib/FutagAnalyzer18.cpp rename to analyzers/checkers/lib/FutagAnalyzer18.cpp diff --git a/src/Checkers/lib/FutagCatchInfo.cpp b/analyzers/checkers/lib/FutagCatchInfo.cpp similarity index 100% rename from src/Checkers/lib/FutagCatchInfo.cpp rename to analyzers/checkers/lib/FutagCatchInfo.cpp diff --git a/src/Checkers/lib/FutagConsumerAnalyzer.cpp b/analyzers/checkers/lib/FutagConsumerAnalyzer.cpp similarity index 100% rename from src/Checkers/lib/FutagConsumerAnalyzer.cpp rename to analyzers/checkers/lib/FutagConsumerAnalyzer.cpp diff --git a/src/Checkers/lib/FutagConsumerAnalyzer14.cpp b/analyzers/checkers/lib/FutagConsumerAnalyzer14.cpp similarity index 100% rename from src/Checkers/lib/FutagConsumerAnalyzer14.cpp rename to analyzers/checkers/lib/FutagConsumerAnalyzer14.cpp diff --git a/src/Checkers/lib/FutagConsumerAnalyzer18.cpp b/analyzers/checkers/lib/FutagConsumerAnalyzer18.cpp similarity index 100% rename from src/Checkers/lib/FutagConsumerAnalyzer18.cpp rename to analyzers/checkers/lib/FutagConsumerAnalyzer18.cpp diff --git a/src/Checkers/lib/FutagSimpleChecker.cpp b/analyzers/checkers/lib/FutagSimpleChecker.cpp similarity index 100% rename from src/Checkers/lib/FutagSimpleChecker.cpp rename to analyzers/checkers/lib/FutagSimpleChecker.cpp diff --git a/src/clang/include/Futag/ArgumentsUsage.h b/analyzers/clang-patches/include/Futag/ArgumentsUsage.h similarity index 100% rename from src/clang/include/Futag/ArgumentsUsage.h rename to analyzers/clang-patches/include/Futag/ArgumentsUsage.h diff --git a/src/clang/include/Futag/Basic.h b/analyzers/clang-patches/include/Futag/Basic.h similarity index 100% rename from src/clang/include/Futag/Basic.h rename to analyzers/clang-patches/include/Futag/Basic.h diff --git a/src/clang/include/Futag/ConsumerFinder.h b/analyzers/clang-patches/include/Futag/ConsumerFinder.h similarity index 100% rename from src/clang/include/Futag/ConsumerFinder.h rename to analyzers/clang-patches/include/Futag/ConsumerFinder.h diff --git a/src/clang/include/Futag/MatchFinder.h b/analyzers/clang-patches/include/Futag/MatchFinder.h similarity index 100% rename from src/clang/include/Futag/MatchFinder.h rename to analyzers/clang-patches/include/Futag/MatchFinder.h diff --git a/src/clang/include/Futag/Utils.h b/analyzers/clang-patches/include/Futag/Utils.h similarity index 100% rename from src/clang/include/Futag/Utils.h rename to analyzers/clang-patches/include/Futag/Utils.h diff --git a/src/clang/include/clang/ASTMatchFinder.h b/analyzers/clang-patches/include/clang/ASTMatchFinder.h similarity index 100% rename from src/clang/include/clang/ASTMatchFinder.h rename to analyzers/clang-patches/include/clang/ASTMatchFinder.h diff --git a/src/clang/include/clang/ASTMatchFinder14.h b/analyzers/clang-patches/include/clang/ASTMatchFinder14.h similarity index 100% rename from src/clang/include/clang/ASTMatchFinder14.h rename to analyzers/clang-patches/include/clang/ASTMatchFinder14.h diff --git a/src/clang/include/clang/ASTMatchFinder18.h b/analyzers/clang-patches/include/clang/ASTMatchFinder18.h similarity index 100% rename from src/clang/include/clang/ASTMatchFinder18.h rename to analyzers/clang-patches/include/clang/ASTMatchFinder18.h diff --git a/src/clang/lib/CMakeLists.txt b/analyzers/clang-patches/lib/CMakeLists.txt similarity index 100% rename from src/clang/lib/CMakeLists.txt rename to analyzers/clang-patches/lib/CMakeLists.txt diff --git a/src/clang/lib/CMakeLists14.txt b/analyzers/clang-patches/lib/CMakeLists14.txt similarity index 100% rename from src/clang/lib/CMakeLists14.txt rename to analyzers/clang-patches/lib/CMakeLists14.txt diff --git a/src/clang/lib/CMakeLists18.txt b/analyzers/clang-patches/lib/CMakeLists18.txt similarity index 100% rename from src/clang/lib/CMakeLists18.txt rename to analyzers/clang-patches/lib/CMakeLists18.txt diff --git a/src/clang/lib/Futag/Basic.cpp b/analyzers/clang-patches/lib/Futag/Basic.cpp similarity index 100% rename from src/clang/lib/Futag/Basic.cpp rename to analyzers/clang-patches/lib/Futag/Basic.cpp diff --git a/src/clang/lib/Futag/CMakeLists.txt b/analyzers/clang-patches/lib/Futag/CMakeLists.txt similarity index 100% rename from src/clang/lib/Futag/CMakeLists.txt rename to analyzers/clang-patches/lib/Futag/CMakeLists.txt diff --git a/src/clang/lib/Futag/ConsumerFinder.cpp b/analyzers/clang-patches/lib/Futag/ConsumerFinder.cpp similarity index 100% rename from src/clang/lib/Futag/ConsumerFinder.cpp rename to analyzers/clang-patches/lib/Futag/ConsumerFinder.cpp diff --git a/src/clang/lib/Futag/MatchFinder.cpp b/analyzers/clang-patches/lib/Futag/MatchFinder.cpp similarity index 100% rename from src/clang/lib/Futag/MatchFinder.cpp rename to analyzers/clang-patches/lib/Futag/MatchFinder.cpp diff --git a/src/clang/lib/clang/ASTMatchFinder.cpp b/analyzers/clang-patches/lib/clang/ASTMatchFinder.cpp similarity index 100% rename from src/clang/lib/clang/ASTMatchFinder.cpp rename to analyzers/clang-patches/lib/clang/ASTMatchFinder.cpp diff --git a/src/clang/lib/clang/ASTMatchFinder14.cpp b/analyzers/clang-patches/lib/clang/ASTMatchFinder14.cpp similarity index 100% rename from src/clang/lib/clang/ASTMatchFinder14.cpp rename to analyzers/clang-patches/lib/clang/ASTMatchFinder14.cpp diff --git a/src/clang/lib/clang/ASTMatchFinder18.cpp b/analyzers/clang-patches/lib/clang/ASTMatchFinder18.cpp similarity index 100% rename from src/clang/lib/clang/ASTMatchFinder18.cpp rename to analyzers/clang-patches/lib/clang/ASTMatchFinder18.cpp diff --git a/custom-llvm/build-debug.sh b/build-llvm/build-debug.sh similarity index 69% rename from custom-llvm/build-debug.sh rename to build-llvm/build-debug.sh index cb25aeab..54b12ce7 100755 --- a/custom-llvm/build-debug.sh +++ b/build-llvm/build-debug.sh @@ -18,11 +18,11 @@ echo "* a tool of ISP RAS *" echo "************************************************" echo "" -futag_src="$(pwd)/../src" +futag_src="$(pwd)/../analyzers" futag_install_folder="$(pwd)/../futag-llvm" vendors="$(pwd)/../vendors" -custom_llvm="$(pwd)/../custom-llvm/llvm-project" -custom_prepare="$(pwd)/../custom-llvm" +custom_llvm="$(pwd)/../build-llvm/llvm-project" +custom_prepare="$(pwd)/../build-llvm" #copy source code to llvm-project cp -r $vendors/json-3.10.5/single_include/nlohmann $custom_llvm/clang/include/ @@ -35,17 +35,17 @@ ASTMatchFindercpp="ASTMatchFinder.cpp" Checkerstd="Checkers.td" CheckerCMakeLists="CMakeLists.txt" -cp -r $futag_src/clang/include/clang/$ASTMatchFinderh $custom_llvm/clang/include/clang/ASTMatchers/ASTMatchFinder.h -cp -r $futag_src/clang/lib/clang/$ASTMatchFindercpp $custom_llvm/clang/lib/ASTMatchers/ASTMatchFinder.cpp +cp -r $futag_src/clang-patches/include/clang/$ASTMatchFinderh $custom_llvm/clang/include/clang/ASTMatchers/ASTMatchFinder.h +cp -r $futag_src/clang-patches/lib/clang/$ASTMatchFindercpp $custom_llvm/clang/lib/ASTMatchers/ASTMatchFinder.cpp -cp -r $futag_src/clang/include/Futag $custom_llvm/clang/include/ -cp $futag_src/clang/lib/$clanglibCMakeLists $custom_llvm/clang/lib/CMakeLists.txt -cp -r $futag_src/clang/lib/Futag $custom_llvm/clang/lib/ +cp -r $futag_src/clang-patches/include/Futag $custom_llvm/clang/include/ +cp $futag_src/clang-patches/lib/$clanglibCMakeLists $custom_llvm/clang/lib/CMakeLists.txt +cp -r $futag_src/clang-patches/lib/Futag $custom_llvm/clang/lib/ # copy clang Checker -cp $futag_src/Checkers/include/$Checkerstd $custom_llvm/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td -cp $futag_src/Checkers/lib/*.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ -cp -r $futag_src/Checkers/lib/$CheckerCMakeLists $custom_llvm/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt +cp $futag_src/checkers/include/$Checkerstd $custom_llvm/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +cp $futag_src/checkers/lib/*.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ +cp -r $futag_src/checkers/lib/$CheckerCMakeLists $custom_llvm/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt cmake -G "Unix Makefiles" -DLLVM_BUILD_TESTS=OFF -DLLVM_ENABLE_ZLIB=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$futag_install_folder -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCLANG_INCLUDE_DOCS="OFF" -DLLVM_BUILD_LLVM_DYLIB="ON" -DLLVM_ENABLE_BINDINGS="OFF" -DLLVM_ENABLE_PROJECTS='clang;' -DLLVM_ENABLE_WARNINGS="OFF" -DLLVM_INCLUDE_BENCHMARKS="OFF" -DLLVM_INCLUDE_DOCS="OFF" -DLLVM_INCLUDE_EXAMPLES="OFF" -DLLVM_INCLUDE_TESTS="OFF" -DLLVM_LINK_LLVM_DYLIB="ON" -DLLVM_TARGETS_TO_BUILD="host" -DLLVM_ENABLE_RUNTIMES="compiler-rt;lld" $custom_llvm/llvm @@ -56,16 +56,16 @@ then rm -rf $futag_install_folder/python-package fi mkdir $futag_install_folder/python-package -cp -r $futag_src/python/futag-package/dist/*.tar.gz $futag_install_folder/python-package -cp -r $futag_src/python/futag-package/requirements.txt $futag_install_folder/python-package -cp -r $futag_src/python/*.py $futag_install_folder/python-package -cp -r $futag_src/svres-tmpl $futag_install_folder/ +cp -r $(pwd)/../futag-package/dist/*.tar.gz $futag_install_folder/python-package +cp -r $(pwd)/../futag-package/requirements.txt $futag_install_folder/python-package +cp -r $(pwd)/../scripts/*.py $futag_install_folder/python-package +cp -r $(pwd)/../src/svres-tmpl $futag_install_folder/ cp -r ../*.md $futag_install_folder/ cp -r ../LICENSE $futag_install_folder/ cp $custom_prepare/INFO $futag_install_folder/ git rev-parse HEAD >> $futag_install_folder/INFO -cd ../product-tests +cd ../integration-tests XZ_OPT='-T12 -9' tar cJf futag-llvm$version.latest.tar.xz ../futag-llvm echo "" diff --git a/custom-llvm/build.sh b/build-llvm/build.sh similarity index 78% rename from custom-llvm/build.sh rename to build-llvm/build.sh index af86e40e..4bf2305f 100755 --- a/custom-llvm/build.sh +++ b/build-llvm/build.sh @@ -18,11 +18,11 @@ echo "* a tool of ISP RAS *" echo "************************************************" echo "" -futag_src="$(pwd)/../src" +futag_src="$(pwd)/../analyzers" futag_install_folder="$(pwd)/../futag-llvm" vendors="$(pwd)/../vendors" -custom_llvm="$(pwd)/../custom-llvm/llvm-project" -custom_prepare="$(pwd)/../custom-llvm" +custom_llvm="$(pwd)/../build-llvm/llvm-project" +custom_prepare="$(pwd)/../build-llvm" #copy source code to llvm-project cp -r $vendors/json-3.10.5/single_include/nlohmann $custom_llvm/clang/include/ @@ -48,18 +48,18 @@ CheckerCMakeLists="CMakeLists$version.txt" FutagAnalyzer="FutagAnalyzer$version.cpp" FutagConsumerAnalyzer="FutagConsumerAnalyzer$version.cpp" -cp -r $futag_src/clang/include/clang/$ASTMatchFinderh $custom_llvm/clang/include/clang/ASTMatchers/ASTMatchFinder.h -cp -r $futag_src/clang/lib/clang/$ASTMatchFindercpp $custom_llvm/clang/lib/ASTMatchers/ASTMatchFinder.cpp +cp -r $futag_src/clang-patches/include/clang/$ASTMatchFinderh $custom_llvm/clang/include/clang/ASTMatchers/ASTMatchFinder.h +cp -r $futag_src/clang-patches/lib/clang/$ASTMatchFindercpp $custom_llvm/clang/lib/ASTMatchers/ASTMatchFinder.cpp -cp -r $futag_src/clang/include/Futag $custom_llvm/clang/include/ -cp $futag_src/clang/lib/$clanglibCMakeLists $custom_llvm/clang/lib/CMakeLists.txt -cp -r $futag_src/clang/lib/Futag $custom_llvm/clang/lib/ +cp -r $futag_src/clang-patches/include/Futag $custom_llvm/clang/include/ +cp $futag_src/clang-patches/lib/$clanglibCMakeLists $custom_llvm/clang/lib/CMakeLists.txt +cp -r $futag_src/clang-patches/lib/Futag $custom_llvm/clang/lib/ # copy clang Checker -cp $futag_src/Checkers/include/$Checkerstd $custom_llvm/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td -cp $futag_src/Checkers/lib/$FutagConsumerAnalyzer $custom_llvm/clang/lib/StaticAnalyzer/Checkers/FutagConsumerAnalyzer.cpp -cp $futag_src/Checkers/lib/$FutagAnalyzer $custom_llvm/clang/lib/StaticAnalyzer/Checkers/FutagAnalyzer.cpp -cp -r $futag_src/Checkers/lib/$CheckerCMakeLists $custom_llvm/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt +cp $futag_src/checkers/include/$Checkerstd $custom_llvm/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +cp $futag_src/checkers/lib/$FutagConsumerAnalyzer $custom_llvm/clang/lib/StaticAnalyzer/Checkers/FutagConsumerAnalyzer.cpp +cp $futag_src/checkers/lib/$FutagAnalyzer $custom_llvm/clang/lib/StaticAnalyzer/Checkers/FutagAnalyzer.cpp +cp -r $futag_src/checkers/lib/$CheckerCMakeLists $custom_llvm/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt if [ $llvmVersion == "LLVM=19.1.7" ]; then cmake -G "Unix Makefiles" -DCMAKE_POLICY_WARNING_CMP=FALSE -DLLVM_BUILD_TESTS=OFF -DLLVM_ENABLE_ZLIB=ON -DCMAKE_BUILD_TYPE=Release -DLLVM_BINUTILS_INCDIR=/usr/include/ -DCMAKE_INSTALL_PREFIX=$futag_install_folder -DCMAKE_EXPORT_COMPILE_COMMANDS=1 -DCLANG_INCLUDE_DOCS="OFF" -DLLVM_BUILD_LLVM_DYLIB="ON" -DLLVM_ENABLE_BINDINGS="OFF" -DLLVM_ENABLE_PROJECTS='clang;lld;compiler-rt' -DLLVM_ENABLE_WARNINGS="OFF" -DLLVM_INCLUDE_BENCHMARKS="OFF" -DLLVM_INCLUDE_DOCS="OFF" -DLLVM_INCLUDE_EXAMPLES="OFF" -DLLVM_INCLUDE_TESTS="OFF" -DLLVM_LINK_LLVM_DYLIB="ON" -DLLVM_TARGETS_TO_BUILD="host" $custom_llvm/llvm @@ -82,16 +82,16 @@ then rm -rf $futag_install_folder/python-package fi mkdir $futag_install_folder/python-package -cp -r $futag_src/python/futag-package/dist/*.tar.gz $futag_install_folder/python-package -cp -r $futag_src/python/futag-package/requirements.txt $futag_install_folder/python-package -cp -r $futag_src/python/*.py $futag_install_folder/python-package -cp -r $futag_src/svres-tmpl $futag_install_folder/ +cp -r $(pwd)/../futag-package/dist/*.tar.gz $futag_install_folder/python-package +cp -r $(pwd)/../futag-package/requirements.txt $futag_install_folder/python-package +cp -r $(pwd)/../scripts/*.py $futag_install_folder/python-package +cp -r $(pwd)/../src/svres-tmpl $futag_install_folder/ cp -r ../*.md $futag_install_folder/ cp -r ../LICENSE $futag_install_folder/ cp $custom_prepare/buildAFLplusplus.sh $futag_install_folder/ -cd ../product-tests +cd ../integration-tests XZ_OPT='-T'$(($(nproc)/2))' -9' tar cJf futag-llvm$version.latest.tar.xz ../futag-llvm diff --git a/custom-llvm/buildAFLplusplus.sh b/build-llvm/buildAFLplusplus.sh similarity index 100% rename from custom-llvm/buildAFLplusplus.sh rename to build-llvm/buildAFLplusplus.sh diff --git a/custom-llvm/buildwAFLplusplusFuzzIntro.sh b/build-llvm/buildwAFLplusplusFuzzIntro.sh similarity index 82% rename from custom-llvm/buildwAFLplusplusFuzzIntro.sh rename to build-llvm/buildwAFLplusplusFuzzIntro.sh index ce759364..f8b2472b 100644 --- a/custom-llvm/buildwAFLplusplusFuzzIntro.sh +++ b/build-llvm/buildwAFLplusplusFuzzIntro.sh @@ -18,11 +18,11 @@ echo "* a tool of ISP RAS *" echo "************************************************" echo "" -futag_src="$(pwd)/../src" +futag_src="$(pwd)/../analyzers" futag_install_folder="$(pwd)/../futag-llvm" vendors="$(pwd)/../vendors" -custom_llvm="$(pwd)/../custom-llvm/llvm-project" -custom_prepare="$(pwd)/../custom-llvm" +custom_llvm="$(pwd)/../build-llvm/llvm-project" +custom_prepare="$(pwd)/../build-llvm" build_folder="$(pwd)" fuzz_introspector=$futag_install_folder/fuzz-introspector-1.0.0 @@ -54,18 +54,18 @@ ASTMatchFindercpp="ASTMatchFinder$version.cpp" Checkerstd="Checkers$version.td" CheckerCMakeLists="CMakeLists$version.txt" -cp -r $futag_src/clang/include/clang/$ASTMatchFinderh $custom_llvm/clang/include/clang/ASTMatchers/ASTMatchFinder.h -cp -r $futag_src/clang/lib/clang/$ASTMatchFindercpp $custom_llvm/clang/lib/ASTMatchers/ASTMatchFinder.cpp +cp -r $futag_src/clang-patches/include/clang/$ASTMatchFinderh $custom_llvm/clang/include/clang/ASTMatchers/ASTMatchFinder.h +cp -r $futag_src/clang-patches/lib/clang/$ASTMatchFindercpp $custom_llvm/clang/lib/ASTMatchers/ASTMatchFinder.cpp -cp -r $futag_src/clang/include/Futag $custom_llvm/clang/include/ -cp $futag_src/clang/lib/$clanglibCMakeLists $custom_llvm/clang/lib/CMakeLists.txt -cp -r $futag_src/clang/lib/Futag $custom_llvm/clang/lib/ +cp -r $futag_src/clang-patches/include/Futag $custom_llvm/clang/include/ +cp $futag_src/clang-patches/lib/$clanglibCMakeLists $custom_llvm/clang/lib/CMakeLists.txt +cp -r $futag_src/clang-patches/lib/Futag $custom_llvm/clang/lib/ # copy clang Checker -cp $futag_src/Checkers/include/$Checkerstd $custom_llvm/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td -cp $futag_src/Checkers/lib/FutagAnalyzer.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ -cp $futag_src/Checkers/lib/FutagContextConsumer.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ -cp -r $futag_src/Checkers/lib/$CheckerCMakeLists $custom_llvm/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt +cp $futag_src/checkers/include/$Checkerstd $custom_llvm/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +cp $futag_src/checkers/lib/FutagAnalyzer.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ +cp $futag_src/checkers/lib/FutagContextConsumer.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ +cp -r $futag_src/checkers/lib/$CheckerCMakeLists $custom_llvm/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt # # copy clang Plugin # cp -r $futag_src/Plugins/* $custom_llvm/clang/ @@ -112,14 +112,14 @@ then rm -rf $futag_install_folder/python-package fi mkdir $futag_install_folder/python-package -cp -r $futag_src/python/futag-package/dist/*.tar.gz $futag_install_folder/python-package -cp -r $futag_src/python/futag-package/requirements.txt $futag_install_folder/python-package -cp -r $futag_src/svres-tmpl $futag_install_folder/ +cp -r $(pwd)/../futag-package/dist/*.tar.gz $futag_install_folder/python-package +cp -r $(pwd)/../futag-package/requirements.txt $futag_install_folder/python-package +cp -r $(pwd)/../src/svres-tmpl $futag_install_folder/ cp -r ../*.md $futag_install_folder/ cp -r ../LICENSE $futag_install_folder/ cp $custom_prepare/INFO $futag_install_folder/ -cd ../product-tests +cd ../integration-tests XZ_OPT='-T8 -9' tar cJf futag-llvm$version.AFLplusplus.fuzz-introspector.latest.tar.xz ../futag-llvm echo "" diff --git a/custom-llvm/buildwFuzzIntro.sh b/build-llvm/buildwFuzzIntro.sh similarity index 82% rename from custom-llvm/buildwFuzzIntro.sh rename to build-llvm/buildwFuzzIntro.sh index 3593ab47..26cf588c 100644 --- a/custom-llvm/buildwFuzzIntro.sh +++ b/build-llvm/buildwFuzzIntro.sh @@ -18,11 +18,11 @@ echo "* a tool of ISP RAS *" echo "************************************************" echo "" -futag_src="$(pwd)/../src" +futag_src="$(pwd)/../analyzers" futag_install_folder="$(pwd)/../futag-llvm" vendors="$(pwd)/../vendors" -custom_llvm="$(pwd)/../custom-llvm/llvm-project" -custom_prepare="$(pwd)/../custom-llvm" +custom_llvm="$(pwd)/../build-llvm/llvm-project" +custom_prepare="$(pwd)/../build-llvm" build_folder="$(pwd)" fuzz_introspector=$futag_install_folder/fuzz-introspector-1.0.0 @@ -54,18 +54,18 @@ ASTMatchFindercpp="ASTMatchFinder$version.cpp" Checkerstd="Checkers$version.td" CheckerCMakeLists="CMakeLists$version.txt" -cp -r $futag_src/clang/include/clang/$ASTMatchFinderh $custom_llvm/clang/include/clang/ASTMatchers/ASTMatchFinder.h -cp -r $futag_src/clang/lib/clang/$ASTMatchFindercpp $custom_llvm/clang/lib/ASTMatchers/ASTMatchFinder.cpp +cp -r $futag_src/clang-patches/include/clang/$ASTMatchFinderh $custom_llvm/clang/include/clang/ASTMatchers/ASTMatchFinder.h +cp -r $futag_src/clang-patches/lib/clang/$ASTMatchFindercpp $custom_llvm/clang/lib/ASTMatchers/ASTMatchFinder.cpp -cp -r $futag_src/clang/include/Futag $custom_llvm/clang/include/ -cp $futag_src/clang/lib/$clanglibCMakeLists $custom_llvm/clang/lib/CMakeLists.txt -cp -r $futag_src/clang/lib/Futag $custom_llvm/clang/lib/ +cp -r $futag_src/clang-patches/include/Futag $custom_llvm/clang/include/ +cp $futag_src/clang-patches/lib/$clanglibCMakeLists $custom_llvm/clang/lib/CMakeLists.txt +cp -r $futag_src/clang-patches/lib/Futag $custom_llvm/clang/lib/ # copy clang Checker -cp $futag_src/Checkers/include/$Checkerstd $custom_llvm/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td -cp $futag_src/Checkers/lib/FutagAnalyzer.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ -cp $futag_src/Checkers/lib/FutagContextConsumer.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ -cp -r $futag_src/Checkers/lib/$CheckerCMakeLists $custom_llvm/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt +cp $futag_src/checkers/include/$Checkerstd $custom_llvm/clang/include/clang/StaticAnalyzer/Checkers/Checkers.td +cp $futag_src/checkers/lib/FutagAnalyzer.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ +cp $futag_src/checkers/lib/FutagContextConsumer.cpp $custom_llvm/clang/lib/StaticAnalyzer/Checkers/ +cp -r $futag_src/checkers/lib/$CheckerCMakeLists $custom_llvm/clang/lib/StaticAnalyzer/Checkers/CMakeLists.txt # # copy clang Plugin # cp -r $futag_src/Plugins/* $custom_llvm/clang/ @@ -101,14 +101,14 @@ then rm -rf $futag_install_folder/python-package fi mkdir $futag_install_folder/python-package -cp -r $futag_src/python/futag-package/dist/*.tar.gz $futag_install_folder/python-package -cp -r $futag_src/python/futag-package/requirements.txt $futag_install_folder/python-package -cp -r $futag_src/svres-tmpl $futag_install_folder/ +cp -r $(pwd)/../futag-package/dist/*.tar.gz $futag_install_folder/python-package +cp -r $(pwd)/../futag-package/requirements.txt $futag_install_folder/python-package +cp -r $(pwd)/../src/svres-tmpl $futag_install_folder/ cp -r ../*.md $futag_install_folder/ cp -r ../LICENSE $futag_install_folder/ cp $custom_prepare/INFO $futag_install_folder/ -cd ../product-tests +cd ../integration-tests XZ_OPT='-T8 -9' tar cJf futag-llvm$version.fuzz-introspector.latest.tar.xz ../futag-llvm echo "" echo "======== End of build script for FUTAG - a Fuzz target automated generator ========" diff --git a/custom-llvm/prepare.sh b/build-llvm/prepare.sh similarity index 100% rename from custom-llvm/prepare.sh rename to build-llvm/prepare.sh diff --git a/custom-llvm/sed_cmdsMacM.sh b/build-llvm/sed_cmdsMacM.sh similarity index 100% rename from custom-llvm/sed_cmdsMacM.sh rename to build-llvm/sed_cmdsMacM.sh diff --git a/docs/analysis-backend.md b/docs/analysis-backend.md new file mode 100644 index 00000000..aa4a72db --- /dev/null +++ b/docs/analysis-backend.md @@ -0,0 +1,174 @@ +# Alternative Analysis Backends + +This guide explains how to produce a `futag-analysis-result.json` file from tools other than Futag's built-in Clang checkers (e.g., CodeQL, Joern, or custom analyzers). + +## Overview + +Futag's Python generators consume a single JSON file — `futag-analysis-result.json` — that describes library functions, types, and compilation metadata. The generators do not care how this JSON was produced. Any tool that can output this format can serve as an analysis backend. + +See [analysis-schema.json](analysis-schema.json) for the formal JSON Schema specification. + +## Usage Modes + +| Mode | What you need | Example | +|------|---------------|---------| +| Full pipeline | futag-llvm toolchain | `Builder(lib_root, toolchain=tc)` → `Generator(lib_root, toolchain=tc)` | +| Pre-built JSON + system clang | analysis JSON + any clang | `Generator(library_root=lib_root, json_file="analysis.json", toolchain=ToolchainConfig.from_system())` | +| Generation only | analysis JSON only | `Generator(library_root=lib_root, json_file="analysis.json")` → produces `.c`/`.cpp` source files | + +## Required vs Optional Fields + +### Required for `gen_targets()` (source code generation) + +- `functions[]` — at least the `name`, `qname`, `func_type`, `access_type`, `params`, `fuzz_it`, and `location` fields +- `functions[].params[].gen_list` — the type decomposition chain (this is the critical data structure) +- `enums[]` — needed if any parameter has `gen_type: 4` (GEN_ENUM) +- `records[]` — needed if any parameter has `gen_type: 9/10/11` (struct/union/class) + +### Required for `compile_targets()` (compilation) + +- `compiled_files[]` — include paths and compiler flags per source file + +### Optional + +- `typedefs[]` — used for type resolution but not strictly required +- `functions[].contexts` — only used by `ContextGenerator` +- `functions[].gen_return_type` — used for some return type analysis + +## The `gen_list` Type Decomposition + +The most important data structure is `gen_list` — an array of `GenTypeInfo` objects that describes how to decompose a C/C++ type for code generation. + +### gen_type Values + +| Value | Name | C/C++ Example | What the generator does | +|-------|------|---------------|------------------------| +| 0 | GEN_BUILTIN | `int`, `float`, `double` | `memcpy(&x, buf, sizeof(int))` | +| 1 | GEN_CSTRING | `char *`, `const char *` | `malloc` + `memcpy` + null terminator | +| 2 | GEN_WSTRING | `wchar_t *` | Wide string allocation | +| 3 | GEN_CXXSTRING | `std::string` | `std::string` construction | +| 4 | GEN_ENUM | `enum Color` | Index into enum values array | +| 5 | GEN_ARRAY | `int[10]` | `malloc(sizeof(int) * 10)` | +| 6 | GEN_VOID | `void *` | `NULL` (cannot generate meaningful data) | +| 7 | GEN_QUALIFIER | `const int` | Reference to underlying variable | +| 8 | GEN_POINTER | `int *` | `&` address-of underlying variable | +| 9 | GEN_STRUCT | `struct Point` | Field-by-field initialization | +| 10 | GEN_UNION | `union Data` | First-field initialization | +| 11 | GEN_CLASS | `class MyClass` | Constructor call | +| 12 | GEN_INCOMPLETE | incomplete types | Skipped | +| 13 | GEN_FUNCTION | `void (*)(int)` | `NULL` (function pointer) | +| 14 | GEN_INPUT_FILE | file path (read) | Generate temp file path | +| 15 | GEN_OUTPUT_FILE | file path (write) | Generate temp file path | +| 18 | GEN_UNKNOWN | unknown types | Skipped | + +### Decomposition Examples + +**Simple: `int x`** +```json +"gen_list": [ + {"gen_type": 0, "type_name": "int", "base_type_name": "int", "local_qualifier": "", "length": 0} +] +``` + +**Pointer: `const char *s`** +```json +"gen_list": [ + {"gen_type": 1, "type_name": "const char *", "base_type_name": "char", "local_qualifier": "const", "length": 0} +] +``` + +**Array: `int arr[10]`** +```json +"gen_list": [ + {"gen_type": 5, "type_name": "int *", "base_type_name": "int", "length": 10} +] +``` + +## `param_usage` Classification + +The `param_usage` field helps the generator produce more realistic inputs: + +| Value | Meaning | Generator behavior | +|-------|---------|-------------------| +| `UNKNOWN` | No special semantics | Generate from type info only | +| `FILE_PATH_READ` | Read-only file path | Generate temp file with random content | +| `FILE_PATH_WRITE` | Write file path | Generate temp file path | +| `FILE_PATH_RW` | Read-write file path | Generate temp file with random content | +| `FILE_PATH` | Generic file path | Generate temp file path | +| `SIZE_FIELD` | Size of a preceding buffer | Link to buffer size variable | +| `C_STRING` | Used as a C string | Ensure null termination | + +### Heuristics for Classification + +When building a backend, use these heuristics: +- **FILE_PATH**: parameter name contains "path", "file", "filename", "dir" AND type is `char *` +- **SIZE_FIELD**: parameter follows a pointer/array parameter AND has integer type AND name contains "size", "len", "count", "num" +- **C_STRING**: type is `char *` or `const char *` AND used in string operations (`strcmp`, `strlen`, `strcpy`, etc.) +- **UNKNOWN**: default for all other parameters + +## Minimal Example + +A complete JSON for a library with one function `int add(int a, int b)`: + +```json +{ + "functions": [ + { + "name": "add", + "qname": "add", + "hash": "abc123", + "is_simple": true, + "func_type": 4, + "access_type": 3, + "storage_class": 0, + "parent_hash": "", + "return_type": {"type_name": "int"}, + "gen_return_type": [{"gen_type": 0, "type_name": "int", "base_type_name": "int", "local_qualifier": "", "length": 0}], + "params": [ + { + "param_name": "a", + "param_type": "int", + "param_usage": "UNKNOWN", + "gen_list": [{"gen_type": 0, "type_name": "int", "base_type_name": "int", "local_qualifier": "", "length": 0}] + }, + { + "param_name": "b", + "param_type": "int", + "param_usage": "UNKNOWN", + "gen_list": [{"gen_type": 0, "type_name": "int", "base_type_name": "int", "local_qualifier": "", "length": 0}] + } + ], + "fuzz_it": true, + "contexts": [], + "location": {"file": "math.c", "line": "5", "directory": "/src", "fullpath": "/src/math.c"} + } + ], + "enums": [], + "records": [], + "typedefs": [], + "compiled_files": [ + { + "filename": "/src/math.c", + "headers": ["\"math.h\""], + "include_paths": ["/src"], + "compiler_opts": "-I/src" + } + ] +} +``` + +## CodeQL Integration Notes + +For CodeQL users, these QL predicates map to the required JSON fields: + +| JSON field | CodeQL predicate | +|-----------|-----------------| +| `functions[].name` | `Function.getName()` | +| `functions[].qname` | `Function.getQualifiedName()` | +| `functions[].params` | `Function.getParameter(i)` | +| `params[].param_type` | `Parameter.getType().toString()` | +| `enums` | `EnumType` + `EnumConstant` | +| `records` | `Struct`, `Union`, `Class` | +| `compiled_files` | Compilation database (`compile_commands.json`) | + +The `gen_list` type decomposition requires recursive type analysis — start from the parameter type and unwrap qualifiers, pointers, and arrays. The `hash` field can be any unique identifier (CodeQL uses `getUniqueId()`). diff --git a/docs/analysis-schema.json b/docs/analysis-schema.json new file mode 100644 index 00000000..3a281e81 --- /dev/null +++ b/docs/analysis-schema.json @@ -0,0 +1,207 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Futag Analysis Result", + "description": "Schema for futag-analysis-result.json — the JSON interface between analysis backends (Clang checkers, CodeQL, etc.) and the Python generator layer.", + "type": "object", + "required": ["functions"], + "properties": { + "functions": { + "type": "array", + "description": "List of analyzed library functions.", + "items": { "$ref": "#/definitions/Function" } + }, + "enums": { + "type": "array", + "description": "Enum type definitions found in the library.", + "items": { "$ref": "#/definitions/Enum" } + }, + "records": { + "type": "array", + "description": "Struct, union, and class definitions.", + "items": { "$ref": "#/definitions/Record" } + }, + "typedefs": { + "type": "array", + "description": "Typedef definitions.", + "items": { "$ref": "#/definitions/Typedef" } + }, + "compiled_files": { + "type": "array", + "description": "Per-file compilation info (include paths, compiler flags). Required for compile_targets().", + "items": { "$ref": "#/definitions/CompiledFile" } + } + }, + "definitions": { + "Function": { + "type": "object", + "required": ["name", "qname", "hash", "is_simple", "func_type", "access_type", "params", "return_type", "fuzz_it", "location"], + "properties": { + "name": { "type": "string", "description": "Unqualified function name." }, + "qname": { "type": "string", "description": "Fully qualified name (e.g., 'namespace::class::func')." }, + "hash": { "type": "string", "description": "Unique identifier (ODR hash or equivalent)." }, + "is_simple": { "type": "boolean", "description": "True if all parameters are simple (builtin) types." }, + "func_type": { + "type": "integer", + "description": "Function kind: 0=CXXMETHOD, 1=CONSTRUCTOR, 2=DEFAULT_CONSTRUCTOR, 3=DESTRUCTOR, 4=GLOBAL, 5=STATIC", + "minimum": 0, "maximum": 5 + }, + "access_type": { + "type": "integer", + "description": "Access specifier: 0=PUBLIC, 1=PROTECTED, 2=PRIVATE, 3=NONE (C functions)", + "minimum": 0, "maximum": 3 + }, + "storage_class": { "type": "integer", "description": "Storage class specifier.", "default": 0 }, + "parent_hash": { "type": "string", "description": "Hash of parent class/struct (empty for free functions).", "default": "" }, + "return_type": { + "type": "object", + "required": ["type_name"], + "properties": { + "type_name": { "type": "string" } + } + }, + "gen_return_type": { + "type": "array", + "items": { "$ref": "#/definitions/GenTypeInfo" }, + "description": "Type decomposition chain for the return type." + }, + "params": { + "type": "array", + "items": { "$ref": "#/definitions/Parameter" } + }, + "fuzz_it": { "type": "boolean", "description": "Whether this function should be fuzz-targeted." }, + "contexts": { "type": "array", "description": "Internal call contexts (optional).", "default": [] }, + "location": { "$ref": "#/definitions/Location" } + } + }, + "Parameter": { + "type": "object", + "required": ["param_name", "param_type", "param_usage", "gen_list"], + "properties": { + "param_name": { "type": "string", "description": "Parameter name from source code." }, + "param_type": { "type": "string", "description": "C/C++ type string (e.g., 'const char *')." }, + "param_usage": { + "type": "string", + "description": "Semantic classification of how the parameter is used.", + "enum": ["UNKNOWN", "FILE_PATH_READ", "FILE_PATH_WRITE", "FILE_PATH_RW", "FILE_PATH", "FILE_DESCRIPTOR", "SIZE_FIELD", "C_STRING"] + }, + "gen_list": { + "type": "array", + "description": "Type decomposition chain — the key data structure for code generation.", + "items": { "$ref": "#/definitions/GenTypeInfo" } + } + } + }, + "GenTypeInfo": { + "type": "object", + "required": ["gen_type", "type_name", "base_type_name", "local_qualifier", "length"], + "properties": { + "gen_type": { + "type": "integer", + "description": "Type classification: 0=BUILTIN, 1=CSTRING, 2=WSTRING, 3=CXXSTRING, 4=ENUM, 5=ARRAY, 6=VOID, 7=QUALIFIER, 8=POINTER, 9=STRUCT, 10=UNION, 11=CLASS, 12=INCOMPLETE, 13=FUNCTION, 14=INPUT_FILE, 15=OUTPUT_FILE, 16=CXXFILE, 17=CFILE, 18=UNKNOWN, 19=REFSTRING", + "minimum": 0, "maximum": 19 + }, + "type_name": { "type": "string", "description": "Current qualified type (e.g., 'const char *')." }, + "base_type_name": { "type": "string", "description": "Unqualified/pointee type (e.g., 'char')." }, + "local_qualifier": { "type": "string", "description": "Qualifier: 'const', 'volatile', 'restrict', or empty." }, + "length": { "type": "integer", "description": "Array length (0 if not an array).", "default": 0 } + } + }, + "Location": { + "type": "object", + "required": ["file", "line", "directory", "fullpath"], + "properties": { + "file": { "type": "string", "description": "Source file name." }, + "line": { "type": "string", "description": "Line number (as string)." }, + "directory": { "type": "string", "description": "Directory containing the source file." }, + "fullpath": { "type": "string", "description": "Full path to the source file." } + } + }, + "Enum": { + "type": "object", + "required": ["name", "qname", "hash", "enum_values"], + "properties": { + "name": { "type": "string" }, + "qname": { "type": "string" }, + "hash": { "type": "string" }, + "access_type": { "type": "integer", "default": 3 }, + "enum_values": { + "type": "array", + "items": { + "type": "object", + "required": ["field_name", "value"], + "properties": { + "field_name": { "type": "string" }, + "value": { "type": "integer" } + } + } + } + } + }, + "Record": { + "type": "object", + "required": ["name", "qname", "hash", "record_type", "fields"], + "properties": { + "name": { "type": "string" }, + "qname": { "type": "string" }, + "hash": { "type": "string" }, + "is_simple": { "type": "boolean", "default": true }, + "record_type": { + "type": "integer", + "description": "0=CLASS, 1=UNION, 2=STRUCT" + }, + "access_type": { "type": "integer", "default": 3 }, + "parent_hash": { "type": "string", "default": "" }, + "fields": { + "type": "array", + "items": { + "type": "object", + "required": ["field_name", "field_type", "gen_list"], + "properties": { + "field_name": { "type": "string" }, + "field_type": { "type": "string" }, + "gen_list": { + "type": "array", + "items": { "$ref": "#/definitions/GenTypeInfo" } + } + } + } + } + } + }, + "Typedef": { + "type": "object", + "required": ["name", "qname", "type_name"], + "properties": { + "name": { "type": "string" }, + "qname": { "type": "string" }, + "type_name": { "type": "string", "description": "The underlying type." }, + "type_source_hash": { "type": "string", "default": "" }, + "is_builtin": { "type": "boolean", "default": false } + } + }, + "CompiledFile": { + "type": "object", + "required": ["filename"], + "properties": { + "filename": { "type": "string", "description": "Full path to the compiled source file." }, + "headers": { + "type": "array", + "items": { "type": "string" }, + "description": "Header files included by this file." + }, + "include_paths": { + "type": "array", + "items": { "type": "string" }, + "description": "Include search paths (-I directories)." + }, + "compiler_opts": { + "description": "Compiler options. Can be a string (space-separated) or array.", + "oneOf": [ + { "type": "string" }, + { "type": "array", "items": { "type": "string" } } + ] + } + } + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index e34a5930..f8e5505c 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,66 +9,44 @@ static analysis checkers and generates fuzz targets in LibFuzzer or AFLplusplus ``` +---------------------------------------------------------------+ -| Layer 3: Build Infrastructure | -| custom-llvm/prepare.sh --> build/build.sh | -| (download LLVM sources, patch Futag checkers, compile) | +| Layer 1: Build Infrastructure | +| build-llvm/prepare.sh --> build/build.sh | +| (download LLVM sources, patch Futag checkers, compile) | +---------------------------------------------------------------+ | futag-llvm/ toolchain | +---------------------------------------------------------------+ -| Layer 1: C++ Clang Checkers | -| src/Checkers/ src/clang/ | -| (StaticAnalyzer plugins: extract function signatures, | -| types, and usage patterns from library source code) | +| Layer 2: C++ Clang Checkers | +| analyzers/checkers/ analyzers/clang-patches/ | +| (StaticAnalyzer plugins: extract function signatures, | +| types, and usage patterns from library source code) | +---------------------------------------------------------------+ | JSON analysis data | +---------------------------------------------------------------+ -| Layer 2: Python Orchestration | -| src/python/futag-package/src/futag/ | -| | -| preprocessor.py --> generator.py --> fuzzer.py | -| (build & analyze) (gen targets) (run fuzzing) | +| Layer 3: Python Orchestration | +| futag-package/src/futag/ | +| | +| preprocessor.py --> generator.py --> fuzzer.py | +| (build & analyze) (gen targets) (run fuzzing) | +---------------------------------------------------------------+ ``` -### Layer 1: C++ Clang Checkers +### Layer 1: Build Infrastructure (optional, one-time) -Located in `src/Checkers/` and `src/clang/`, these are static analysis plugins that -run inside Clang's StaticAnalyzer framework. They extract function signatures, type -information, and usage patterns from the target library's source code and serialize -the results as JSON. - -For detailed documentation, see [docs/checkers.md](checkers.md). - -### Layer 2: Python Orchestration - -Located in `src/python/futag-package/src/futag/`, this layer provides the user-facing -Python API that drives the full pipeline: - -- **preprocessor.py** -- `Builder` builds and analyzes target libraries; - `ConsumerBuilder` handles library+consumer pairs -- **generator.py** -- `Generator` produces fuzz targets from analysis JSON; - `ContextGenerator` uses consumer usage contexts -- **fuzzer.py** -- `BaseFuzzer`, `Fuzzer`, and `NatchFuzzer` execute generated - targets with configurable timeouts, memory limits, and sanitizers -- **sysmsg.py** -- Constants and error messages (LIBFUZZER, AFLPLUSPLUS engine - identifiers, paths) - -For detailed documentation, see [docs/generators.md](generators.md) and -[docs/python-api.md](python-api.md). - -### Layer 3: Build Infrastructure +Located in `build-llvm/` and `build/`, shell scripts that: -Located in `custom-llvm/` and `build/`, shell scripts that: - -1. Download LLVM sources (`custom-llvm/prepare.sh`) +1. Download LLVM sources (`build-llvm/prepare.sh`) 2. Patch in Futag's checkers and Clang modifications 3. Build the complete toolchain via CMake (`build/build.sh`) 4. Optionally build AFLplusplus support (`futag-llvm/buildAFLplusplus.sh`) +**Note:** This layer is only needed for the full pipeline workflow. The Python +package can also work with pre-built analysis JSON and system-installed clang, +or in generation-only mode. See [Usage Modes](#usage-modes) below. + ## Data Flow ``` @@ -78,7 +56,7 @@ Located in `custom-llvm/` and `build/`, shell scripts that: v v | +----------+ +-------------+ | | .c / .h |--->| scan-build | | - | files | | (checkers) | | + | files | | (checkers) | | +----------+ +------+------+ | | | JSON analysis | @@ -117,36 +95,94 @@ Located in `custom-llvm/` and `build/`, shell scripts that: crashes / coverage ``` +### Layer 2: C++ Clang Checkers + +Located in `analyzers/checkers/` and `analyzers/clang-patches/`, these are static analysis plugins that +run inside Clang's StaticAnalyzer framework. They extract function signatures, type +information, and usage patterns from the target library's source code and serialize +the results as JSON. + +For detailed documentation, see [docs/checkers.md](checkers.md). + +### Layer 3: Python Orchestration + +Located in `futag-package/src/futag/`, this layer provides the user-facing +Python API that drives the full pipeline: + +- **preprocessor.py** -- `Builder` builds and analyzes target libraries; + `ConsumerBuilder` handles library+consumer pairs +- **generator.py** -- `Generator` produces fuzz targets from analysis JSON; + `ContextGenerator` uses consumer usage contexts +- **fuzzer.py** -- `BaseFuzzer`, `Fuzzer`, and `NatchFuzzer` execute generated + targets with configurable timeouts, memory limits, and sanitizers +- **sysmsg.py** -- Constants and error messages (LIBFUZZER, AFLPLUSPLUS engine + identifiers, paths) + +For detailed documentation, see [docs/generators.md](generators.md) and +[docs/python-api.md](python-api.md). + + +## Usage Modes + +The Python package supports three usage modes via `ToolchainConfig`: + +| Mode | Requires | Use case | +|------|----------|----------| +| Full pipeline | futag-llvm toolchain | First-time setup, complete workflow | +| Pre-built JSON + system clang | Any clang + analysis JSON | CodeQL or other backend, CI/CD | +| Generation only | Just the analysis JSON file | Produce source files, compile elsewhere | + +```python +from futag.toolchain import ToolchainConfig + +# Mode 1: Full pipeline +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) +generator = Generator(library_root, toolchain=tc) + +# Mode 2: Pre-built JSON + system clang +tc = ToolchainConfig.from_system() +generator = Generator(library_root=lib_root, json_file="analysis.json", toolchain=tc) + +# Mode 3: Generation only (no compiler needed) +generator = Generator(library_root=lib_root, json_file="analysis.json") +generator.gen_targets() # produces .c/.cpp source files +``` + +The JSON interface between analysis backends and generators is formally specified +in [analysis-schema.json](analysis-schema.json). For instructions on producing +compatible JSON from alternative tools, see [analysis-backend.md](analysis-backend.md). + ## Key Components | Component | Location | Documentation | |-----------|----------|---------------| -| Clang Checkers | `src/Checkers/`, `src/clang/` | [docs/checkers.md](checkers.md) | -| Generator Classes | `src/python/futag-package/src/futag/` | [docs/generators.md](generators.md) | -| Python API | `src/python/futag-package/` | [docs/python-api.md](python-api.md) | -| Build Scripts | `custom-llvm/`, `build/` | [README.en.md](../README.en.md) | +| Clang Checkers | `analyzers/checkers/`, `analyzers/clang-patches/` | [docs/checkers.md](checkers.md) | +| Generator Classes | `futag-package/src/futag/` | [docs/generators.md](generators.md) | +| Python API | `futag-package/` | [docs/python-api.md](python-api.md) | +| Build Scripts | `build-llvm/`, `build/` | [README.en.md](../README.en.md) | +| JSON Schema | `docs/analysis-schema.json` | [docs/analysis-backend.md](analysis-backend.md) | ## Directory Structure ``` Futag/ - build/ # Build scripts for compiling the LLVM toolchain - custom-llvm/ # Scripts to download and patch LLVM sources + analyzers/ # C++ analysis layer + checkers/ # Clang StaticAnalyzer checker sources + clang-patches/ # Clang modifications for Futag + build-llvm/ # Scripts to download and patch LLVM sources docs/ # Detailed documentation checkers.md # Clang checker documentation generators.md # Generator class documentation python-api.md # Python API reference examples/ # Example scripts and configurations - product-tests/ # Dockerized tests + futag-package/ # Python package (pip-installable) + src/futag/ # Core Python modules + tests/ # Unit tests + integration-tests/ # Dockerized tests build-test/ # Build validation (Ubuntu 20.04/22.04/24.04, Alt 11/12) libraries-test/ # End-to-end tests against real libraries package-test/ # Python package tests - src/ - Checkers/ # C++ Clang StaticAnalyzer checker sources - clang/ # Clang modifications for Futag - python/ - futag-package/ # Python package (pip-installable) - src/futag/ # Core Python modules + scripts/ # Standalone utility scripts vendors/ json/ # nlohmann/json (C++ JSON library for checkers) workshop/ # Library-specific tutorials @@ -182,5 +218,5 @@ The base (unsuffixed) file should always match the latest supported LLVM version For build instructions and setup, see [README.en.md](../README.en.md). -For a complete workflow example, see `src/python/template-script.py` and the +For a complete workflow example, see `scripts/template-script.py` and the `workshop/` directory for library-specific tutorials. diff --git a/docs/checkers.md b/docs/checkers.md index 7a938d8c..f505bda2 100644 --- a/docs/checkers.md +++ b/docs/checkers.md @@ -50,7 +50,7 @@ Consumer Source Code + futag-4consumer.json ## Checker Registration -Checkers are defined in `src/Checkers/include/Checkers.td` using Clang's TableGen format: +Checkers are defined in `analyzers/checkers/include/Checkers.td` using Clang's TableGen format: ```tablegen let ParentPackage = Futag in { @@ -78,7 +78,7 @@ scan-build -enable-checker futag.FutagAnalyzer \ ## FutagAnalyzer -**Source:** `src/Checkers/lib/FutagAnalyzer.cpp` (+ LLVM14/18 variants) +**Source:** `analyzers/checkers/lib/FutagAnalyzer.cpp` (+ LLVM14/18 variants) ### Class Hierarchy @@ -157,7 +157,7 @@ The checker hooks into Clang's `ASTDecl` callback, receiving the entire translat ## FutagConsumerAnalyzer -**Source:** `src/Checkers/lib/FutagConsumerAnalyzer.cpp` (+ LLVM14/18 variants) +**Source:** `analyzers/checkers/lib/FutagConsumerAnalyzer.cpp` (+ LLVM14/18 variants) ### Purpose @@ -243,7 +243,7 @@ Each context file (`_funcname_varname_blocks.json`) contains: ## Type System -**Source:** `src/clang/include/Futag/Basic.h`, `src/clang/lib/Futag/Basic.cpp` +**Source:** `analyzers/clang-patches/include/Futag/Basic.h`, `analyzers/clang-patches/lib/Futag/Basic.cpp` ### Type Classification @@ -296,7 +296,7 @@ The Python `sysmsg.py` defines matching constants: ## AST Matching Infrastructure -### FutagAnalyzer Matchers (`src/clang/include/Futag/MatchFinder.h`) +### FutagAnalyzer Matchers (`analyzers/clang-patches/include/Futag/MatchFinder.h`) | Callback Class | Pattern Matched | Purpose | |----------------|-----------------|---------| @@ -305,7 +305,7 @@ The Python `sysmsg.py` defines matching constants: | `FutagMatchVarDeclArgCallBack` | Variable declaration matching | Tracks variable initializations | | `FutagMatchBinOperatorArgCallBack` | Assignment operator matching | Tracks variable assignments | -### FutagConsumerAnalyzer Matchers (`src/clang/include/Futag/ConsumerFinder.h`) +### FutagConsumerAnalyzer Matchers (`analyzers/clang-patches/include/Futag/ConsumerFinder.h`) | Callback Class | Pattern Matched | Purpose | |----------------|-----------------|---------| @@ -368,7 +368,7 @@ The base file always matches the LLVM 18 version. The `build.sh` script selects Checkers are compiled into Clang's `clangStaticAnalyzerCheckers` library: ```cmake -# src/Checkers/lib/CMakeLists.txt +# analyzers/checkers/lib/CMakeLists.txt add_clang_library(clangStaticAnalyzerCheckers ... FutagAnalyzer.cpp @@ -382,7 +382,7 @@ add_clang_library(clangStaticAnalyzerCheckers Supporting code is built as `FutagLib`: ```cmake -# src/clang/lib/Futag/CMakeLists.txt +# analyzers/clang-patches/lib/Futag/CMakeLists.txt add_clang_library(FutagLib Basic.cpp MatchFinder.cpp diff --git a/futag-work.mmd b/docs/diagrams/futag-work.mmd similarity index 100% rename from futag-work.mmd rename to docs/diagrams/futag-work.mmd diff --git a/futag-work.png b/docs/diagrams/futag-work.png similarity index 100% rename from futag-work.png rename to docs/diagrams/futag-work.png diff --git a/docs/generators.md b/docs/generators.md index c9bbb9db..4caebf70 100644 --- a/docs/generators.md +++ b/docs/generators.md @@ -45,7 +45,7 @@ succeeded/ and failed/ directories with compiled fuzz drivers ## BaseGenerator (ABC) -**File:** `src/python/futag-package/src/futag/base_generator.py` +**File:** `futag-package/src/futag/base_generator.py` The abstract base class containing all shared infrastructure. Subclasses only need to implement 10 type-specific generation methods. @@ -84,8 +84,7 @@ Each returns a dict with three keys: ```python # Create generator -gen = Generator(futag_llvm_package="/path/to/futag-llvm", - library_root="/path/to/library") +gen = Generator("/path/to/library") # Generate fuzz targets gen.gen_targets(anonymous=False, max_wrappers=10, max_functions=10000) @@ -96,7 +95,7 @@ gen.compile_targets(workers=4, keep_failed=True, coverage=True) ## GeneratorState -**File:** `src/python/futag-package/src/futag/generator_state.py` +**File:** `futag-package/src/futag/generator_state.py` Dataclass encapsulating all mutable state during recursive target generation. Replaces the previous pattern of 13+ instance variables with manual save/restore. @@ -111,7 +110,7 @@ state.reset() # Clear all state for new function ### Generator (Standard) -**File:** `src/python/futag-package/src/futag/generator.py` +**File:** `futag-package/src/futag/generator.py` Uses raw `memcpy()` from a byte buffer (`uint8_t *futag_pos`). Supports both C and C++ targets. @@ -124,7 +123,7 @@ futag_pos += sizeof(int); ### FuzzDataProviderGenerator -**File:** `src/python/futag-package/src/futag/fdp_generator.py` +**File:** `futag-package/src/futag/fdp_generator.py` Uses libFuzzer's `FuzzedDataProvider` API for type-safe data consumption. C++ only. @@ -138,7 +137,7 @@ const char* param2 = s_fdp.c_str(); ### BlobStamperGenerator -**File:** `src/python/futag-package/src/futag/blob_stamper_generator.py` +**File:** `futag-package/src/futag/blob_stamper_generator.py` Inherits from `FuzzDataProviderGenerator`. Same type generation logic but supports both C and C++ targets. diff --git a/How-to-work-with-Futag.md b/docs/how-to-work-with-futag.md similarity index 96% rename from How-to-work-with-Futag.md rename to docs/how-to-work-with-futag.md index 75038c30..17631446 100644 --- a/How-to-work-with-Futag.md +++ b/docs/how-to-work-with-futag.md @@ -14,7 +14,7 @@ Есть возможность объединить шаги [4] и [6], но scan-build не собирает с флагами "-fprofile-instr-generate -fcoverage-mapping", соответственно в собранной цели будет отсутстовать инструментация, позволяющая собирать информацию о покрытии. ## Как написать python-скрипт работы с Futag -Полную документацию python-модуля в составе Futag можно посмотреть [по ссылке](https://github.com/ispras/Futag/tree/main/src/python/futag-package). +Полную документацию python-модуля в составе Futag можно посмотреть [по ссылке](https://github.com/ispras/Futag/tree/main/futag-package). Класс Builder принимает следующие параметры: ```python @@ -155,11 +155,12 @@ testing_lib.analyze() from futag.generator import * from futag.sysmsg import * +tc = ToolchainConfig.from_futag_llvm("Futag/futag-llvm/") #Путь к рабочей директории futag g = Generator( -"Futag/futag-llvm/", #Путь к рабочей директории futag "path/to/library/source/code", #Путь к директории исходных текстов исследуемого приложения target_type=AFLPLUSPLUS, # Формат оберток LIBFUZZER или AFLPLUSPLUS -json_file="/path/to/analysis/folder/futag-analysis-result.json" #Путь к файлу результата анализа +json_file="/path/to/analysis/folder/futag-analysis-result.json", #Путь к файлу результата анализа +toolchain=tc, ) g.gen_targets() # генерация фаззинг-оберток g.compile_targets( # компиляция фаззинг-оберток @@ -175,7 +176,7 @@ Futag поддерживает несколько вариантов генер ```python from futag.generator import Generator -generator = Generator(futag_llvm_path, library_root) +generator = Generator(library_root) generator.gen_targets(max_wrappers=10) generator.compile_targets(workers=4) ``` @@ -185,7 +186,7 @@ generator.compile_targets(workers=4) ```python from futag.fdp_generator import FuzzDataProviderGenerator -generator = FuzzDataProviderGenerator(futag_llvm_path, library_root) +generator = FuzzDataProviderGenerator(library_root) generator.gen_targets(max_wrappers=100) generator.compile_targets(workers=4, keep_failed=True) ``` @@ -195,7 +196,7 @@ generator.compile_targets(workers=4, keep_failed=True) ```python from futag.generator import ContextGenerator -ctx_gen = ContextGenerator(futag_llvm_path, library_root) +ctx_gen = ContextGenerator(library_root) ctx_gen.gen_context() ctx_gen.compile_targets(keep_failed=True) ``` diff --git a/Project-summary.md b/docs/project-summary.md similarity index 75% rename from Project-summary.md rename to docs/project-summary.md index f9740015..08ccb6ab 100644 --- a/Project-summary.md +++ b/docs/project-summary.md @@ -21,31 +21,33 @@ ```python from futag.preprocessor import * -from futag.generator import * -from futag.fuzzer import * -from futag.sysmsg import * +from futag.generator import * +from futag.fuzzer import * +from futag.sysmsg import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "/home/futag/Futag-tests/futag-llvm/" # Путь к директории инструмента Futag "futag-llvm" [*] library_root = "curl-7.85.0" # путь к директории содержащей исходные кода исследуемого ПО [*] +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) # конфигурация инструментов Futag lib_test = Builder( # модуль для запуска сборки и анализа - FUTAG_PATH, library_root, clean=True, # перед запуска удаляются ли созданные папки инструментом Futag [1] processes=16, # количество потоков при сборке - build_ex_params="--without-ssl --disable-ldap --disable-ldaps" # дополнительные параметры при сборке библиотеки - данные параметры запускаются на этапе конфигурации библиотеки curl [*] + build_ex_params="--without-ssl --disable-ldap --disable-ldaps", # дополнительные параметры при сборке библиотеки - данные параметры запускаются на этапе конфигурации библиотеки curl [*] + toolchain=tc, ) lib_test.auto_build() # инструмент автоматически собирает, устанавливает библиотеку в папки из вышесказанного [1] lib_test.analyze() # запуск сборки результата анализа в файл futag-analysis-result.json lib_test = Generator( # модуль для генерации - FUTAG_PATH, library_root, - target_type=LIBFUZZER, # формат фаззинг-оберток: LIBFUZZER или AFLPLUSPLUS + target_type=LIBFUZZER, # формат фаззинг-оберток: LIBFUZZER или AFLPLUSPLUS + toolchain=tc, ) lib_test.gen_targets() # генерация оберток lib_test.compile_targets( # функция для компиляции оберток - 16, + workers=16, keep_failed=True, # сохранить ли не скомпилированные обертки extra_include="-DHAVE_CONFIG_H", # дополнительные параметры включаются в строку компиляции. Данный параметр включается при сборке curl [*] extra_dynamiclink="-lgsasl -lpsl -lbrotlidec -lz -lidn2" # системые библиотеки включаются на этапе линковки. Данные библиотеки включаются при сборке curl [*] @@ -53,9 +55,9 @@ lib_test.compile_targets( # функция для компиляции обер consumer_root = "libstorj-1.0.3" consumber_builder = ConsumerBuilder( - FUTAG_PATH, # путь к директории "futag-llvm" - library_root, # путь к директории содержащей исходные кода тестируемой библиотеки - consumer_root, # путь к директории содержащей исходные кода потребительской программы + library_root, # путь к директории содержащей исходные кода тестируемой библиотеки + consumer_root, # путь к директории содержащей исходные кода потребительской программы + toolchain=tc, # clean=True, # processes=16, ) @@ -63,19 +65,19 @@ consumber_builder.auto_build() consumber_builder.analyze() context_generator = ContextGenerator( - FUTAG_PATH, - library_root, + library_root, + toolchain=tc, ) context_generator.gen_context() # генерация фаззинг-оберток для контекстов -context_generator.compile_targets( #компиляция сгенерированных фаззинг-оберток +context_generator.compile_targets( # компиляция сгенерированных фаззинг-оберток keep_failed=True, ) fuzzer = Fuzzer( # модуль для фаззинга - FUTAG_PATH, - fuzz_driver_path="curl-7.85.0/futag-fuzz-drivers", # путь к папке, содержащей скомпилированные обертки - totaltime=10, # время фаззинга одной обертки - coverage=True # показывается ли покрытие + "curl-7.85.0/futag-fuzz-drivers", # путь к папке, содержащей скомпилированные обертки + totaltime=10, # время фаззинга одной обертки + coverage=True, # показывается ли покрытие + toolchain=tc, ) fuzzer.fuzz() # функция для запуска фаззинга ``` \ No newline at end of file diff --git a/docs/python-api.md b/docs/python-api.md index 20bd00af..4209fa7e 100644 --- a/docs/python-api.md +++ b/docs/python-api.md @@ -5,14 +5,17 @@ ```python from futag.preprocessor import Builder from futag.generator import Generator +from futag.toolchain import ToolchainConfig # Step 1: Build and analyze the library -builder = Builder("/path/to/futag-llvm", "/path/to/library", clean=True) +tc = ToolchainConfig.from_futag_llvm("/path/to/futag-llvm") +builder = Builder("/path/to/library", clean=True, toolchain=tc) builder.auto_build() builder.analyze() # Step 2: Generate fuzz targets -generator = Generator("/path/to/futag-llvm", "/path/to/library") +tc = ToolchainConfig.from_futag_llvm("/path/to/futag-llvm") +generator = Generator("/path/to/library", toolchain=tc) generator.gen_targets(anonymous=False, max_wrappers=10) generator.compile_targets(workers=4, keep_failed=True) ``` @@ -27,11 +30,60 @@ generator.compile_targets(workers=4, keep_failed=True) | `futag.blob_stamper_generator` | `BlobStamperGenerator` | BlobStamper-based targets | | `futag.fuzzer` | `Fuzzer`, `NatchFuzzer` | Execute fuzz targets | | `futag.base_generator` | `BaseGenerator` (ABC) | Shared generator infrastructure | +| `futag.toolchain` | `ToolchainConfig` | External tool path configuration | | `futag.generator_state` | `GeneratorState` | State management dataclass | | `futag.sysmsg` | (constants) | Constants and error messages | --- +## ToolchainConfig + +Centralizes all external tool paths. Supports three factory methods for different usage modes. + +```python +from futag.toolchain import ToolchainConfig + +# From a compiled futag-llvm directory (existing workflow) +tc = ToolchainConfig.from_futag_llvm("/path/to/futag-llvm") + +# From system-installed tools (discovered via PATH) +tc = ToolchainConfig.from_system() + +# Generation only — no tools needed, gen_targets() produces source files +tc = ToolchainConfig.for_generation_only() +``` + +All classes accept an optional `toolchain` parameter. For `Generator` and `Fuzzer` classes, if omitted, a generation-only config is used (no compiler). For `Builder`, it is constructed from `futag_llvm_package`. + +### Usage Modes + +```python +# Mode 1: Full pipeline +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) +builder = Builder(lib_root, toolchain=tc) +builder.auto_build() +builder.analyze() +generator = Generator(lib_root, toolchain=tc) +generator.gen_targets() +generator.compile_targets(4) + +# Mode 2: Pre-built JSON + system clang +tc = ToolchainConfig.from_system() +generator = Generator(library_root=lib_root, + json_file="/path/to/analysis.json", + toolchain=tc) +generator.gen_targets() +generator.compile_targets(4) + +# Mode 3: Generation only (no compiler needed) +generator = Generator(library_root=lib_root, + json_file="/path/to/analysis.json") +generator.gen_targets() +# produces .c/.cpp source files in futag-fuzz-drivers/ +``` + +--- + ## Preprocessor Module ### Builder @@ -40,9 +92,10 @@ Builds and analyzes a target library using the Futag-patched Clang toolchain. ```python from futag.preprocessor import Builder +from futag.toolchain import ToolchainConfig +tc = ToolchainConfig.from_futag_llvm("/path/to/futag-llvm") builder = Builder( - futag_llvm_package="/path/to/futag-llvm", # Required: path to compiled toolchain library_root="/path/to/library", # Required: path to library source flags="-g -O0", # Compiler flags (default: debug + sanitizer + coverage) clean=False, # Delete futag dirs before starting @@ -51,7 +104,8 @@ builder = Builder( install_path=".futag-install", # Install directory name analysis_path=".futag-analysis", # Analysis output directory name processes=4, # Parallel build workers - build_ex_params="" # Extra build params (e.g., "--with-openssl") + build_ex_params="", # Extra build params (e.g., "--with-openssl") + toolchain=tc, ) builder.auto_build() # Auto-detect build system (configure/cmake/makefile/meson) @@ -68,13 +122,15 @@ Analyzes a consumer program to extract library usage contexts. ```python from futag.preprocessor import ConsumerBuilder +from futag.toolchain import ToolchainConfig +tc = ToolchainConfig.from_futag_llvm("/path/to/futag-llvm") consumer_builder = ConsumerBuilder( - futag_llvm_package="/path/to/futag-llvm", library_root="/path/to/library", consumer_root="/path/to/consumer", # Required: consumer program source clean=False, processes=4, + toolchain=tc, ) consumer_builder.auto_build() @@ -93,7 +149,6 @@ Generates fuzz targets using raw `memcpy()` buffer consumption. Supports both C from futag.generator import Generator generator = Generator( - futag_llvm_package="/path/to/futag-llvm", library_root="/path/to/library", alter_compiler="", # Override compiler path target_type=0, # 0=LIBFUZZER, 1=AFLPLUSPLUS @@ -132,7 +187,6 @@ Uses libFuzzer's `FuzzedDataProvider` API. C++ only, type-safe data consumption. from futag.fdp_generator import FuzzDataProviderGenerator generator = FuzzDataProviderGenerator( - futag_llvm_package="/path/to/futag-llvm", library_root="/path/to/library", ) generator.gen_targets(anonymous=False, max_wrappers=100) @@ -147,7 +201,6 @@ Uses LibBlobStamper. Inherits FDP's type generation but supports both C and C++. from futag.blob_stamper_generator import BlobStamperGenerator generator = BlobStamperGenerator( - futag_llvm_package="/path/to/futag-llvm", library_root="/path/to/library", ) ``` @@ -160,7 +213,6 @@ Generates fuzz targets from consumer program usage contexts. from futag.generator import ContextGenerator ctx_gen = ContextGenerator( - futag_llvm_package="/path/to/futag-llvm", library_root="/path/to/library", db_json_file=".futag-analysis/futag-analysis-result.json", context_json_file=".futag-consumer/futag-contexts.json", @@ -179,7 +231,6 @@ Generates fuzz targets from Natch crash trace data. from futag.generator import NatchGenerator natch_gen = NatchGenerator( - futag_llvm_package="/path/to/futag-llvm", library_root="/path/to/library", json_file="/path/to/natch-output.json", # Required: Natch JSON file ) @@ -236,7 +287,6 @@ Executes generated fuzz targets and collects crashes. from futag.fuzzer import Fuzzer fuzzer = Fuzzer( - futag_llvm_package="/path/to/futag-llvm", fuzz_driver_path="futag-fuzz-drivers", # Directory with compiled fuzz targets debug=False, # Print debug info gdb=False, # Debug crashes with GDB @@ -261,7 +311,6 @@ Same as Fuzzer but adds Natch corpus path support. from futag.fuzzer import NatchFuzzer fuzzer = NatchFuzzer( - futag_llvm_package="/path/to/futag-llvm", fuzz_driver_path="futag-fuzz-drivers", totaltime=60, debug=True, diff --git a/References.md b/docs/references.md similarity index 100% rename from References.md rename to docs/references.md diff --git a/src/python/futag-package/LICENSE b/futag-package/LICENSE similarity index 100% rename from src/python/futag-package/LICENSE rename to futag-package/LICENSE diff --git a/src/python/futag-package/README.md b/futag-package/README.md similarity index 83% rename from src/python/futag-package/README.md rename to futag-package/README.md index 5174d9f9..e4d6a946 100644 --- a/src/python/futag-package/README.md +++ b/futag-package/README.md @@ -15,7 +15,7 @@ ************************************************** ``` This python package is for building library, generating and compiling fuzz-drivers of functions. -* Package url: https://github.com/ispras/Futag/tree/main/src/python/futag-package +* Package url: https://github.com/ispras/Futag/tree/main/futag-package * Bug Tracker = https://github.com/ispras/Futag/issues ## 1. Install @@ -49,19 +49,16 @@ test_lib.analyze() For more parameters of Builder please refer to docstring of this class. ```bash class Builder(builtins.object) - | Builder(futag_llvm_package: str, library_root: str, flags: str = '-fsanitize=address', clean: bool = False, build_path: str = '.futag-build', install_path: str = '.futag-install', analysis_path: str = '.futag-analysis', process -es: int = 4, build_ex_params='') - | + | Builder(library_root: str, flags: str = '', clean: bool = False, build_path: str = '.futag-build', install_path: str = '.futag-install', analysis_path: str = '.futag-analysis', processes: int = 4, build_ex_params='', toolchain=None) + | | Futag Builder Class - | + | | Methods defined here: - | - | __init__(self, futag_llvm_package: str, library_root: str, flags: str = '-fsanitize=address', clean: bool = False, build_path: str = '.futag-build', install_path: str = '.futag-install', analysis_path: str = '.futag-analysis', -processes: int = 4, build_ex_params='') + | + | __init__(self, library_root: str, flags: str = '', clean: bool = False, build_path: str = '.futag-build', install_path: str = '.futag-install', analysis_path: str = '.futag-analysis', processes: int = 4, build_ex_params='', toolchain=None) | Constructor of class Builder - | + | | Args: - | futag_llvm_package (str): path to the futag-llvm package (with binaries, scripts, etc.). | library_root (str): path to the library root. | flags (str, optional): flags for compiling.. Defaults to COMPILER_FLAGS. | clean (bool, optional): Option for deleting futag folders if they are exist, for example futag-build, futag-install, futag-analysis. Defaults to False. @@ -140,17 +137,16 @@ The fuzz-drivers of libjson will be generated in futag-fuzz-drivers inside the l ```bash class Generator(builtins.object) - | Generator(futag_llvm_package: str, library_root: str, target_type: int = 0, json_file: str = '.futag-analysis/futag-analysis-result.json', output_path='futag-fuzz-drivers', build_path='.futag-build', install_path='.futag-install') + | Generator(library_root: str, target_type: int = 0, json_file: str = '.futag-analysis/futag-analysis-result.json', output_path='futag-fuzz-drivers', build_path='.futag-build', install_path='.futag-install', toolchain=None) | | Futag Generator | | Methods defined here: | - | __init__(self, futag_llvm_package: str, library_root: str, target_type: int = 0, json_file: str = '.futag-analysis/futag-analysis-result.json', output_path='futag-fuzz-drivers', build_path='.futag-build', install_path='.futag-install') + | __init__(self, library_root: str, target_type: int = 0, json_file: str = '.futag-analysis/futag-analysis-result.json', output_path='futag-fuzz-drivers', build_path='.futag-build', install_path='.futag-install', toolchain=None) | Constructor of Generator class. - | + | | Args: - | futag_llvm_package (str): path to the futag-llvm package (with binaries, scripts, etc.). | library_root (str): path to the library root. | target_type (int, optional): format of fuzz-drivers (LIBFUZZER or AFLPLUSPLUS). Defaults to LIBFUZZER. | json_file (str, optional): path to the futag-analysis-result.json file. Defaults to ANALYSIS_FILE_PATH. @@ -207,17 +203,15 @@ f.fuzz() ```bash class Fuzzer(builtins.object) - | Fuzzer(futag_llvm_package: str, fuzz_driver_path: str = 'futag-fuzz-drivers', debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False) + | Fuzzer(fuzz_driver_path: str = 'futag-fuzz-drivers', debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False, toolchain=None) | | Futag Fuzzer | | Methods defined here: | - | __init__(self, futag_llvm_package: str, fuzz_driver_path: str = 'futag-fuzz-drivers', debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False) + | __init__(self, fuzz_driver_path: str = 'futag-fuzz-drivers', debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False, toolchain=None) | Parameters | ---------- - | futag_llvm_package: str - | path to the futag llvm package (with binaries, scripts, etc) | fuzz_driver_path: str | location of fuzz-drivers, default "futag-fuzz-drivers" | debug: bool = False diff --git a/src/python/futag-package/param_hints.json b/futag-package/param_hints.json similarity index 100% rename from src/python/futag-package/param_hints.json rename to futag-package/param_hints.json diff --git a/src/python/futag-package/pyproject.toml b/futag-package/pyproject.toml similarity index 100% rename from src/python/futag-package/pyproject.toml rename to futag-package/pyproject.toml diff --git a/src/python/futag-package/pytest.ini b/futag-package/pytest.ini similarity index 100% rename from src/python/futag-package/pytest.ini rename to futag-package/pytest.ini diff --git a/src/python/futag-package/requirements.txt b/futag-package/requirements.txt similarity index 100% rename from src/python/futag-package/requirements.txt rename to futag-package/requirements.txt diff --git a/src/python/futag-package/setup.cfg b/futag-package/setup.cfg similarity index 88% rename from src/python/futag-package/setup.cfg rename to futag-package/setup.cfg index 24d7c963..0cd33483 100644 --- a/src/python/futag-package/setup.cfg +++ b/futag-package/setup.cfg @@ -6,7 +6,7 @@ author_email = thientcgithub@gmail.com description = Python package of Futag long_description = file: README.md long_description_content_type = text/markdown -url = https://github.com/ispras/Futag/tree/main/src/python/futag-package +url = https://github.com/ispras/Futag/tree/main/futag-package project_urls = Bug Tracker = https://github.com/ispras/Futag/issues classifiers = diff --git a/src/python/futag-package/setup.py b/futag-package/setup.py similarity index 93% rename from src/python/futag-package/setup.py rename to futag-package/setup.py index 6bac8c56..570a767d 100644 --- a/src/python/futag-package/setup.py +++ b/futag-package/setup.py @@ -11,7 +11,7 @@ url='https://github.com/ispras/Futag', project_urls={ 'Documentation': 'https://github.com/ispras/Futag/tree/main/docs', - 'Source': 'https://github.com/ispras/Futag/tree/main/src/python/futag-package', + 'Source': 'https://github.com/ispras/Futag/tree/main/futag-package', 'Bug Tracker': 'https://github.com/ispras/Futag/issues', }, license='LICENSE', diff --git a/src/python/futag-package/src/futag.egg-info/PKG-INFO b/futag-package/src/futag.egg-info/PKG-INFO similarity index 100% rename from src/python/futag-package/src/futag.egg-info/PKG-INFO rename to futag-package/src/futag.egg-info/PKG-INFO diff --git a/src/python/futag-package/src/futag.egg-info/SOURCES.txt b/futag-package/src/futag.egg-info/SOURCES.txt similarity index 100% rename from src/python/futag-package/src/futag.egg-info/SOURCES.txt rename to futag-package/src/futag.egg-info/SOURCES.txt diff --git a/src/python/futag-package/src/futag.egg-info/dependency_links.txt b/futag-package/src/futag.egg-info/dependency_links.txt similarity index 100% rename from src/python/futag-package/src/futag.egg-info/dependency_links.txt rename to futag-package/src/futag.egg-info/dependency_links.txt diff --git a/src/python/futag-package/src/futag.egg-info/requires.txt b/futag-package/src/futag.egg-info/requires.txt similarity index 100% rename from src/python/futag-package/src/futag.egg-info/requires.txt rename to futag-package/src/futag.egg-info/requires.txt diff --git a/src/python/futag-package/src/futag.egg-info/top_level.txt b/futag-package/src/futag.egg-info/top_level.txt similarity index 100% rename from src/python/futag-package/src/futag.egg-info/top_level.txt rename to futag-package/src/futag.egg-info/top_level.txt diff --git a/src/python/futag-package/src/futag/__init__.py b/futag-package/src/futag/__init__.py similarity index 56% rename from src/python/futag-package/src/futag/__init__.py rename to futag-package/src/futag/__init__.py index 17ee786d..75601e11 100644 --- a/src/python/futag-package/src/futag/__init__.py +++ b/futag-package/src/futag/__init__.py @@ -13,12 +13,13 @@ from futag.preprocessor import Builder from futag.generator import Generator + from futag.toolchain import ToolchainConfig - builder = Builder(futag_llvm_path, library_root, clean=True) + tc = ToolchainConfig.from_futag_llvm(futag_llvm_path) + builder = Builder(library_root, clean=True, toolchain=tc) builder.auto_build() builder.analyze() - - generator = Generator(futag_llvm_path, library_root) + generator = Generator(library_root, toolchain=tc) generator.gen_targets() generator.compile_targets(workers=4) @@ -38,3 +39,26 @@ import logging logging.getLogger('futag').addHandler(logging.NullHandler()) + + +def setup_console_logging(enable=True): + """Configure console logging for the futag package. + + When enabled, attaches a StreamHandler to the 'futag' logger at INFO level. + Idempotent: calling multiple times will not add duplicate handlers. + + Args: + enable: If True, add a console handler. If False, do nothing. + """ + if not enable: + return + futag_logger = logging.getLogger('futag') + # Avoid adding duplicate console handlers + for h in futag_logger.handlers: + if isinstance(h, logging.StreamHandler) and not isinstance(h, (logging.NullHandler, logging.FileHandler)): + return + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + handler.setFormatter(logging.Formatter('%(name)s - %(levelname)s - %(message)s')) + futag_logger.addHandler(handler) + futag_logger.setLevel(logging.INFO) diff --git a/src/python/futag-package/src/futag/base_generator.py b/futag-package/src/futag/base_generator.py similarity index 98% rename from src/python/futag-package/src/futag/base_generator.py rename to futag-package/src/futag/base_generator.py index 8fb9662c..0fb4b80a 100644 --- a/src/python/futag-package/src/futag/base_generator.py +++ b/futag-package/src/futag/base_generator.py @@ -30,6 +30,7 @@ from shutil import copytree from futag.sysmsg import * +from futag import setup_console_logging logger = logging.getLogger(__name__) from futag.preprocessor import delete_folder @@ -400,11 +401,10 @@ def _gen_pointer(self, param_name: str, prev_param_name: str, gen_type_info: dic # Constructor # # ------------------------------------------------------------------ # - def __init__(self, futag_llvm_package: str, library_root: str, target_type: int = LIBFUZZER, json_file: str = ANALYSIS_FILE_PATH, output_path=FUZZ_DRIVER_PATH, build_path=BUILD_PATH, install_path=INSTALL_PATH, delimiter: str = "."): + def __init__(self, library_root: str, target_type: int = LIBFUZZER, json_file: str = ANALYSIS_FILE_PATH, output_path=FUZZ_DRIVER_PATH, build_path=BUILD_PATH, install_path=INSTALL_PATH, delimiter: str = ".", toolchain=None, log_to_console: bool = True): """ Constructor of BaseGenerator class. Args: - futag_llvm_package (str): path to the futag-llvm package (with binaries, scripts, etc.). library_root (str): path to the library root. target_type (int, optional): format of fuzz-drivers (LIBFUZZER or AFLPLUSPLUS). Defaults to LIBFUZZER. json_file (str, optional): path to the futag-analysis-result.json file. Defaults to ANALYSIS_FILE_PATH. @@ -421,10 +421,11 @@ def __init__(self, futag_llvm_package: str, library_root: str, target_type: int ValueError: INVALID_INSTALLPATH: Invalid path to the library install path. """ + setup_console_logging(log_to_console) + self.output_path = None # Path for saving fuzzing drivers self.tmp_output_path = None # Path for saving fuzzing drivers self.json_file = json_file - self.futag_llvm_package = futag_llvm_package self.library_root = library_root self.target_library = None self.exclude_headers = [] @@ -445,18 +446,11 @@ def __init__(self, futag_llvm_package: str, library_root: str, target_type: int self.target_type = target_type - if pathlib.Path(self.futag_llvm_package).exists(): - self.futag_llvm_package = pathlib.Path( - self.futag_llvm_package).absolute() - else: - sys.exit(INVALID_FUTAG_PATH) - - if self.target_type == LIBFUZZER: - if not pathlib.Path(self.futag_llvm_package / "bin/clang").exists(): - sys.exit(INVALID_FUTAG_PATH) + from futag.toolchain import ToolchainConfig + if toolchain is not None: + self.toolchain = toolchain else: - if not pathlib.Path(self.futag_llvm_package / "AFLplusplus/usr/local/bin/afl-clang-fast").exists(): - sys.exit(INVALID_FUTAG_PATH) + self.toolchain = ToolchainConfig.for_generation_only() if pathlib.Path(self.library_root).exists(): self.library_root = pathlib.Path(self.library_root).absolute() @@ -1314,7 +1308,7 @@ def _wrapper_file(self, func: dict) -> dict: filename = func["qname"].replace(":", "_") filepath = self.tmp_output_path - self.target_extension = func["location"]["fullpath"].split(".")[-1] + self.target_extension = "cpp" file_index = 1 # qname = func["qname"] @@ -1383,7 +1377,7 @@ def _anonymous_wrapper_file(self, func: dict): filename = "anonymous_" + func["name"].replace(":", "_") filepath = self.tmp_output_path - self.target_extension = func["location"]["fullpath"].split(".")[-1] + self.target_extension = "cpp" file_index = 1 # qname = func["qname"] @@ -1604,10 +1598,7 @@ def _gen_target_function(self, func: dict, param_id: int, anonymous: bool = Fals func["location"]["fullpath"]) if self.target_type == LIBFUZZER: - if compiler_info["compiler"] == "CC": - f.write(LIBFUZZER_PREFIX_C) - else: - f.write(LIBFUZZER_PREFIX_CXX) + f.write(LIBFUZZER_PREFIX) else: f.write(AFLPLUSPLUS_PREFIX) @@ -1775,6 +1766,8 @@ def _gen_target_function(self, func: dict, param_id: int, anonymous: bool = Fals " //end of generation random array of dynamic file sizes\n") f.write(" uint8_t * futag_pos = Fuzz_Data;\n") + if self.harness_preamble: + f.write(self.harness_preamble) for line in self.state.gen_lines: f.write(" " + line) @@ -2423,19 +2416,11 @@ def compile_targets(self, workers: int = 4, keep_failed: bool = False, extra_par if not "-ferror-limit=1" in compiler_flags_libFuzzer: compiler_flags_libFuzzer += " -ferror-limit=1" - compiler_path = "" + self.toolchain.require_compiler(self.target_type) if self.target_type == LIBFUZZER: - if compiler_info["compiler"] == "CC": - compiler_path = self.futag_llvm_package / "bin/clang" - else: - compiler_path = self.futag_llvm_package / "bin/clang++" + compiler_path = self.toolchain.clangpp else: - if compiler_info["compiler"] == "CC": - compiler_path = self.futag_llvm_package / \ - "AFLplusplus/usr/local/bin/afl-clang-fast" - else: - compiler_path = self.futag_llvm_package / \ - "AFLplusplus/usr/local/bin/afl-clang-fast++" + compiler_path = self.toolchain.afl_clang_fastpp current_func_compilation_opts = "" compilation_opts = "" diff --git a/src/python/futag-package/src/futag/blob_stamper_generator.py b/futag-package/src/futag/blob_stamper_generator.py similarity index 91% rename from src/python/futag-package/src/futag/blob_stamper_generator.py rename to futag-package/src/futag/blob_stamper_generator.py index 18837f6a..f47e0fe7 100644 --- a/src/python/futag-package/src/futag/blob_stamper_generator.py +++ b/futag-package/src/futag/blob_stamper_generator.py @@ -37,6 +37,6 @@ def supports_c(self) -> bool: return True # BlobStamper supports both C and C++ def _wrapper_file(self, func) -> dict: - """Return wrapper file metadata, using the original source file extension.""" - self.target_extension = func["location"]["fullpath"].split(".")[-1] + """Return wrapper file metadata, always using .cpp extension.""" + self.target_extension = "cpp" return BaseGenerator._wrapper_file(self, func) diff --git a/src/python/futag-package/src/futag/context_generator.py b/futag-package/src/futag/context_generator.py similarity index 99% rename from src/python/futag-package/src/futag/context_generator.py rename to futag-package/src/futag/context_generator.py index 0fede332..7c57cc96 100644 --- a/src/python/futag-package/src/futag/context_generator.py +++ b/futag-package/src/futag/context_generator.py @@ -38,17 +38,17 @@ class ContextGenerator(Generator): code (call sequences, variable bindings, etc.). """ - def __init__(self, futag_llvm_package: str, library_root: str, + def __init__(self, library_root: str, target_type: int = LIBFUZZER, db_json_file: str = ANALYSIS_FILE_PATH, context_json_file: str = CONTEXT_FILE_PATH, output_path=CONTEXT_FUZZ_DRIVER_PATH, build_path=BUILD_PATH, install_path=INSTALL_PATH, - delimiter: str = '.'): + delimiter: str = '.', toolchain=None, + log_to_console: bool = True): """Constructor of ContextGenerator class. Args: - futag_llvm_package (str): path to the futag-llvm package. library_root (str): path to the library root. target_type (int, optional): format of fuzz-drivers. Defaults to LIBFUZZER. db_json_file (str, optional): path to the analysis JSON file. Defaults to ANALYSIS_FILE_PATH. @@ -62,10 +62,11 @@ def __init__(self, futag_llvm_package: str, library_root: str, SystemExit: on invalid paths or configuration. """ super().__init__( - futag_llvm_package, library_root, + library_root, target_type=target_type, json_file=db_json_file, output_path=output_path, build_path=build_path, install_path=install_path, delimiter=delimiter, + toolchain=toolchain, log_to_console=log_to_console, ) self.consumer_contexts = None @@ -651,10 +652,7 @@ def _gen_context_wrapper(self, func): func["location"]["fullpath"]) if self.target_type == LIBFUZZER: - if compiler_info["compiler"] == "CC": - f.write(LIBFUZZER_PREFIX_C) - else: - f.write(LIBFUZZER_PREFIX_CXX) + f.write(LIBFUZZER_PREFIX) else: f.write(AFLPLUSPLUS_PREFIX) @@ -777,6 +775,8 @@ def _gen_context_wrapper(self, func): " //end of generation random array of dynamic file sizes\n") f.write(" uint8_t * futag_pos = Fuzz_Data;\n") + if self.harness_preamble: + f.write(self.harness_preamble) for line in self.state.gen_lines: f.write(" " + line) diff --git a/src/python/futag-package/src/futag/exceptions.py b/futag-package/src/futag/exceptions.py similarity index 100% rename from src/python/futag-package/src/futag/exceptions.py rename to futag-package/src/futag/exceptions.py diff --git a/src/python/futag-package/src/futag/fdp_generator.py b/futag-package/src/futag/fdp_generator.py similarity index 91% rename from src/python/futag-package/src/futag/fdp_generator.py rename to futag-package/src/futag/fdp_generator.py index 0193edda..da5a833d 100644 --- a/src/python/futag-package/src/futag/fdp_generator.py +++ b/futag-package/src/futag/fdp_generator.py @@ -25,13 +25,15 @@ class FuzzDataProviderGenerator(BaseGenerator): """Generator using libFuzzer's FuzzedDataProvider API for type-safe data consumption.""" - def __init__(self, futag_llvm_package, library_root, target_type=LIBFUZZER, + def __init__(self, library_root, target_type=LIBFUZZER, json_file=ANALYSIS_FILE_PATH, output_path=FUZZ_DRIVER_PATH, - build_path=BUILD_PATH, install_path=INSTALL_PATH, delimiter=".") -> None: - super().__init__(futag_llvm_package, library_root, + build_path=BUILD_PATH, install_path=INSTALL_PATH, delimiter=".", + toolchain=None, log_to_console=True) -> None: + super().__init__(library_root, target_type=target_type, json_file=json_file, output_path=output_path, build_path=build_path, - install_path=install_path, delimiter=delimiter) + install_path=install_path, delimiter=delimiter, + toolchain=toolchain, log_to_console=log_to_console) self.last_string_name: str = "" @property @@ -156,17 +158,16 @@ def _gen_cxxstring(self, param_name, gen_type_info, dyn_cxxstring_size_idx) -> d } def _gen_enum(self, enum_record, param_name, gen_type_info, compiler_info, anonymous=False) -> dict: - """Declare and assign value for an enum type.""" - if anonymous: - enum_name = enum_record["name"] - else: - enum_name = enum_record["qname"] - - enum_name = gen_type_info["type_name"] + """Declare and assign value for an enum type using PickValueInArray.""" + enum_type = gen_type_info["type_name"] + # Template parameters cannot use the 'enum' keyword in C++ + template_type = enum_type.removeprefix("enum ").strip() + enum_values = [v["field_name"] for v in enum_record["enum_values"]] + values_list = "{" + ", ".join(enum_values) + "}" return { "gen_lines": [ "//GEN_ENUM\n", - "auto " + param_name + " = provider.ConsumeEnum<" + enum_name + ">()" + enum_type + " " + param_name + " = provider.PickValueInArray<" + template_type + ">(" + values_list + ");\n", ], "gen_free": [], "buffer_size": ["sizeof(unsigned int)"] diff --git a/src/python/futag-package/src/futag/fuzzer.py b/futag-package/src/futag/fuzzer.py similarity index 93% rename from src/python/futag-package/src/futag/fuzzer.py rename to futag-package/src/futag/fuzzer.py index 16d23a6d..827699b1 100644 --- a/src/python/futag-package/src/futag/fuzzer.py +++ b/futag-package/src/futag/fuzzer.py @@ -26,6 +26,7 @@ import logging from futag.sysmsg import * +from futag import setup_console_logging logger = logging.getLogger(__name__) @@ -58,11 +59,10 @@ class BaseFuzzer: """Base class containing all shared fuzzing logic.""" - def __init__(self, futag_llvm_package: str, fuzz_driver_path: str = FUZZ_DRIVER_PATH, debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False, source_path: str = "") -> None: + def __init__(self, fuzz_driver_path: str = FUZZ_DRIVER_PATH, debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False, source_path: str = "", toolchain=None, log_to_console: bool = True) -> None: """Initialize the BaseFuzzer with fuzzing configuration. Args: - futag_llvm_package: Path to the Futag LLVM package (with binaries, scripts, etc). fuzz_driver_path: Location of fuzz-drivers, default "futag-fuzz-drivers". debug: Print debug information while fuzzing, default False. gdb: Debug crashes with GDB, default False. @@ -75,15 +75,18 @@ def __init__(self, futag_llvm_package: str, fuzz_driver_path: str = FUZZ_DRIVER_ leak: Detect memory leaks, default False. introspect: Integrate with fuzz-introspector, default False. source_path: Path to source code for coverage reports, default "". + toolchain: ToolchainConfig instance. If None, uses generation-only mode. + log_to_console: Show log messages in the console, default True. """ - self.futag_llvm_package = futag_llvm_package + setup_console_logging(log_to_console) self.fuzz_driver_path = fuzz_driver_path self.source_path = source_path - if Path(self.futag_llvm_package).exists(): - self.futag_llvm_package = Path(self.futag_llvm_package).absolute() + from futag.toolchain import ToolchainConfig + if toolchain is not None: + self.toolchain = toolchain else: - sys.exit(INVALID_FUTAG_PATH) + self.toolchain = ToolchainConfig.for_generation_only() if Path(self.fuzz_driver_path).exists(): self.fuzz_driver_path = Path(self.fuzz_driver_path).absolute() @@ -639,8 +642,8 @@ def _build_single_coverage(self, object_file: str, path: str) -> None: """ my_env = os.environ.copy() my_env["LLVM_PROFILE_FILE"] = object_file + ".profraw" - llvm_profdata = self.futag_llvm_package / "bin/llvm-profdata" - llvm_cov = self.futag_llvm_package / "bin/llvm-cov" + llvm_profdata = self.toolchain.llvm_profdata + llvm_cov = self.toolchain.llvm_cov ## Merge profraw file llvm_profdata_command = [ @@ -714,8 +717,8 @@ def _build_overall_coverage(self, path) -> None: for o in object_list: object_files += ["-object", o] - llvm_profdata = self.futag_llvm_package / "bin/llvm-profdata" - llvm_cov = self.futag_llvm_package / "bin/llvm-cov" + llvm_profdata = self.toolchain.llvm_profdata + llvm_cov = self.toolchain.llvm_cov llvm_profdata_command = [ llvm_profdata.as_posix(), @@ -774,7 +777,10 @@ def _finalize_svres(self) -> None: into the svres template, writes the final futag.svres file, and removes the intermediate files. """ - template_file = self.futag_llvm_package / "svres-tmpl/svres.tmpl" + template_file = self.toolchain.svres_template + if template_file is None: + logger.warning("svres template not available, skipping svres finalization") + return warning_info_text = "" warning_info_path = Path.cwd().absolute() / "warning_info.svres" warning_info_ex_text = "" @@ -803,9 +809,18 @@ def fuzz(self, extra_param: str = "") -> None: Args: extra_param: Extra params for fuzzing. Defaults to "". """ - symbolizer = self.futag_llvm_package / "bin/llvm-symbolizer" - generated_functions = [ - x for x in (self.fuzz_driver_path / "succeeded").iterdir() if x.is_dir()] + symbolizer = self.toolchain.llvm_symbolizer + succeeded_path = self.fuzz_driver_path / "succeeded" + if not succeeded_path.exists(): + logger.error( + "Directory '%s' not found. " + "Ensure compile_targets() ran successfully and that " + "fuzz_driver_path points to the fuzz-drivers directory " + "(not the toolchain path).", + succeeded_path, + ) + return + generated_functions = [x for x in succeeded_path.iterdir() if x.is_dir()] for func_dir in generated_functions: self.backtraces = [] fuzz_driver_dirs = [x for x in func_dir.iterdir() if x.is_dir()] @@ -893,11 +908,10 @@ def fuzz(self, extra_param: str = "") -> None: class Fuzzer(BaseFuzzer): """Futag Fuzzer""" - def __init__(self, futag_llvm_package: str, fuzz_driver_path: str = FUZZ_DRIVER_PATH, debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False, source_path: str = "") -> None: + def __init__(self, fuzz_driver_path: str = FUZZ_DRIVER_PATH, debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False, source_path: str = "", toolchain=None, log_to_console: bool = True) -> None: """Initialize the Fuzzer. Args: - futag_llvm_package: Path to the Futag LLVM package (with binaries, scripts, etc). fuzz_driver_path: Location of fuzz-drivers, default "futag-fuzz-drivers". debug: Print debug information while fuzzing, default False. gdb: Debug crashes with GDB, default False. @@ -910,9 +924,10 @@ def __init__(self, futag_llvm_package: str, fuzz_driver_path: str = FUZZ_DRIVER_ leak: Detect memory leaks, default False. introspect: Integrate with fuzz-introspector, default False. source_path: Path to source code for coverage reports, default "". + toolchain: ToolchainConfig instance. If None, uses generation-only mode. + log_to_console: Show log messages in the console, default True. """ super().__init__( - futag_llvm_package=futag_llvm_package, fuzz_driver_path=fuzz_driver_path, debug=debug, gdb=gdb, @@ -925,6 +940,8 @@ def __init__(self, futag_llvm_package: str, fuzz_driver_path: str = FUZZ_DRIVER_ leak=leak, introspect=introspect, source_path=source_path, + toolchain=toolchain, + log_to_console=log_to_console, ) def _get_corpus_args(self, target_path) -> list: @@ -935,11 +952,10 @@ def _get_corpus_args(self, target_path) -> list: class NatchFuzzer(BaseFuzzer): """Futag Fuzzer for Natch""" - def __init__(self, futag_llvm_package: str, fuzz_driver_path: str = FUZZ_DRIVER_PATH, debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False) -> None: + def __init__(self, fuzz_driver_path: str = FUZZ_DRIVER_PATH, debug: bool = False, gdb: bool = False, svres: bool = False, fork: int = 1, totaltime: int = 300, timeout: int = 10, memlimit: int = 2048, coverage: bool = False, leak: bool = False, introspect: bool = False, toolchain=None, log_to_console: bool = True) -> None: """Initialize the NatchFuzzer. Args: - futag_llvm_package: Path to the Futag LLVM package (with binaries, scripts, etc). fuzz_driver_path: Location of fuzz-drivers, default "futag-fuzz-drivers". debug: Print debug information while fuzzing, default False. gdb: Debug crashes with GDB, default False. @@ -951,9 +967,9 @@ def __init__(self, futag_llvm_package: str, fuzz_driver_path: str = FUZZ_DRIVER_ coverage: Show coverage of fuzzing, default False. leak: Detect memory leaks, default False. introspect: Integrate with fuzz-introspector, default False. + log_to_console: Show log messages in the console, default True. """ super().__init__( - futag_llvm_package=futag_llvm_package, fuzz_driver_path=fuzz_driver_path, debug=debug, gdb=gdb, @@ -966,6 +982,8 @@ def __init__(self, futag_llvm_package: str, fuzz_driver_path: str = FUZZ_DRIVER_ leak=leak, introspect=introspect, source_path="", + toolchain=toolchain, + log_to_console=log_to_console, ) def _get_corpus_args(self, target_path) -> list: diff --git a/src/python/futag-package/src/futag/generator.py b/futag-package/src/futag/generator.py similarity index 97% rename from src/python/futag-package/src/futag/generator.py rename to futag-package/src/futag/generator.py index eac45bf2..a6a5eab6 100644 --- a/src/python/futag-package/src/futag/generator.py +++ b/futag-package/src/futag/generator.py @@ -36,14 +36,16 @@ class Generator(BaseGenerator): """Standard Futag Generator using raw memcpy buffer consumption.""" - def __init__(self, futag_llvm_package, library_root, alter_compiler="", + def __init__(self, library_root, alter_compiler="", target_type=LIBFUZZER, json_file=ANALYSIS_FILE_PATH, output_path=FUZZ_DRIVER_PATH, build_path=BUILD_PATH, - install_path=INSTALL_PATH, delimiter=".", exclude_headers=None) -> None: - super().__init__(futag_llvm_package, library_root, + install_path=INSTALL_PATH, delimiter=".", exclude_headers=None, + toolchain=None, log_to_console=True) -> None: + super().__init__(library_root, target_type=target_type, json_file=json_file, output_path=output_path, build_path=build_path, - install_path=install_path, delimiter=delimiter) + install_path=install_path, delimiter=delimiter, + toolchain=toolchain, log_to_console=log_to_console) self.alter_compiler = alter_compiler self.exclude_headers = exclude_headers if exclude_headers else [] diff --git a/src/python/futag-package/src/futag/generator_state.py b/futag-package/src/futag/generator_state.py similarity index 100% rename from src/python/futag-package/src/futag/generator_state.py rename to futag-package/src/futag/generator_state.py diff --git a/src/python/futag-package/src/futag/natch_generator.py b/futag-package/src/futag/natch_generator.py similarity index 98% rename from src/python/futag-package/src/futag/natch_generator.py rename to futag-package/src/futag/natch_generator.py index 3d8600e5..e4ac6f7f 100644 --- a/src/python/futag-package/src/futag/natch_generator.py +++ b/futag-package/src/futag/natch_generator.py @@ -39,14 +39,14 @@ class NatchGenerator(Generator): - Iterates over target_functions parsed from Natch JSON """ - def __init__(self, futag_llvm_package: str, library_root: str, - json_file: str, target_type: int = LIBFUZZER, + def __init__(self, library_root: str, + json_file: str = "", target_type: int = LIBFUZZER, output_path=FUZZ_DRIVER_PATH, build_path=BUILD_PATH, - install_path=INSTALL_PATH): + install_path=INSTALL_PATH, toolchain=None, + log_to_console: bool = True): """Constructor of NatchGenerator class. Args: - futag_llvm_package (str): path to the futag-llvm package. library_root (str): path to the library root. json_file (str): path to the JSON file from Natch. target_type (int, optional): format of fuzz-drivers. Defaults to LIBFUZZER. @@ -60,11 +60,13 @@ def __init__(self, futag_llvm_package: str, library_root: str, self.natch_json_file = pathlib.Path(json_file).absolute() - super().__init__(futag_llvm_package, library_root, + super().__init__(library_root, target_type=target_type, output_path=output_path, build_path=build_path, - install_path=install_path) + install_path=install_path, + toolchain=toolchain, + log_to_console=log_to_console) # Create Natch corpus directory Natch_corpus_path = self.output_path / "Natch_corpus" @@ -421,15 +423,14 @@ def _gen_target_function(self, func, param_id) -> bool: func["location"]["fullpath"]) if self.target_type == LIBFUZZER: - if compiler_info["compiler"] == "CC": - f.write(LIBFUZZER_PREFIX_C) - else: - f.write(LIBFUZZER_PREFIX_CXX) + f.write(LIBFUZZER_PREFIX) else: f.write(AFLPLUSPLUS_PREFIX) f.write(" size_t Fuzz_Size_remain = Fuzz_Size;;\n") f.write(" uint8_t * futag_pos = Fuzz_Data;\n") + if self.harness_preamble: + f.write(self.harness_preamble) for line in self.state.gen_lines: f.write(" " + line) @@ -796,15 +797,14 @@ def _gen_anonymous_function(self, func, param_id) -> bool: func["location"]["fullpath"]) if self.target_type == LIBFUZZER: - if compiler_info["compiler"] == "CC": - f.write(LIBFUZZER_PREFIX_C) - else: - f.write(LIBFUZZER_PREFIX_CXX) + f.write(LIBFUZZER_PREFIX) else: f.write(AFLPLUSPLUS_PREFIX) f.write(" size_t Fuzz_Size_remain = Fuzz_Size;\n") f.write(" uint8_t * futag_pos = Fuzz_Data;\n") + if self.harness_preamble: + f.write(self.harness_preamble) for line in self.state.gen_lines: f.write(" " + line) diff --git a/src/python/futag-package/src/futag/preprocessor.py b/futag-package/src/futag/preprocessor.py similarity index 92% rename from src/python/futag-package/src/futag/preprocessor.py rename to futag-package/src/futag/preprocessor.py index 08992775..c1ecfd38 100644 --- a/src/python/futag-package/src/futag/preprocessor.py +++ b/futag-package/src/futag/preprocessor.py @@ -28,6 +28,7 @@ import logging from futag.sysmsg import * +from futag import setup_console_logging from subprocess import Popen, PIPE logger = logging.getLogger(__name__) @@ -86,11 +87,11 @@ def _run_command(cmd, env=None, msg_prefix="", fail_msg="", succeed_msg="", return p.returncode, output, errors -def _make_build_env(futag_llvm_package, flags=None): - """Create environment dict with Futag compiler paths set.""" +def _make_build_env(toolchain, flags=None): + """Create environment dict with compiler paths from toolchain config.""" env = os.environ.copy() - env["CC"] = (futag_llvm_package / "bin/clang").as_posix() - env["CXX"] = (futag_llvm_package / "bin/clang++").as_posix() + env["CC"] = toolchain.clang.as_posix() + env["CXX"] = toolchain.clangpp.as_posix() if flags: env["CFLAGS"] = flags env["CPPFLAGS"] = flags @@ -133,9 +134,9 @@ def _parse_location(location_str): class _BaseBuilder: """Shared base for Builder and ConsumerBuilder.""" - def _validate_common(self, futag_llvm_package, library_root, processes, build_ex_params): + def _validate_common(self, library_root, processes, build_ex_params, futag_llvm_package="", toolchain=None, log_to_console=True): """Validate and set common attributes.""" - self.futag_llvm_package = futag_llvm_package + setup_console_logging(log_to_console) self.library_root = library_root try: @@ -146,11 +147,13 @@ def _validate_common(self, futag_llvm_package, library_root, processes, build_ex sys.exit(INVALID_INPUT_PROCESSES) self.processes = processes - if pathlib.Path(futag_llvm_package).absolute().exists() and (pathlib.Path(futag_llvm_package) / "bin/clang").absolute().exists(): - self.futag_llvm_package = pathlib.Path( - self.futag_llvm_package).absolute() + from futag.toolchain import ToolchainConfig + if toolchain is not None: + self.toolchain = toolchain + elif futag_llvm_package and pathlib.Path(futag_llvm_package).absolute().exists() and (pathlib.Path(futag_llvm_package) / "bin/clang").absolute().exists(): + self.toolchain = ToolchainConfig.from_futag_llvm(futag_llvm_package) else: - sys.exit(INVALID_FUTAG_PATH + futag_llvm_package) + sys.exit(INVALID_FUTAG_PATH + str(futag_llvm_package)) if pathlib.Path(library_root).absolute().exists(): self.library_root = pathlib.Path(self.library_root).absolute() @@ -167,14 +170,15 @@ def _extra_build_params(self): def _scan_build_args(self, checker_name=None, analyzer_configs=None): """Build scan-build argument prefix.""" + self.toolchain.require_scan_build() return _scan_build_checker_args( - self.futag_llvm_package / "bin/scan-build", + self.toolchain.scan_build, checker_name, analyzer_configs) def _make_env(self, with_flags=False): - """Create build environment with Futag compiler paths.""" + """Create build environment with compiler paths from toolchain.""" return _make_build_env( - self.futag_llvm_package, + self.toolchain, self.flags if with_flags else None) def _make_jobs_arg(self): @@ -187,11 +191,10 @@ def _make_jobs_arg(self): class Builder(_BaseBuilder): """Futag Builder Class""" - def __init__(self, futag_llvm_package: str, library_root: str, flags: str = "", clean: bool = False, intercept: bool = True, build_path: str = BUILD_PATH, install_path: str = INSTALL_PATH, analysis_path: str = ANALYSIS_PATH, processes: int = 4, build_ex_params=BUILD_EX_PARAMS): + def __init__(self, library_root: str, flags: str = "", clean: bool = False, intercept: bool = True, build_path: str = BUILD_PATH, install_path: str = INSTALL_PATH, analysis_path: str = ANALYSIS_PATH, processes: int = 4, build_ex_params=BUILD_EX_PARAMS, futag_llvm_package: str = "", toolchain=None, log_to_console: bool = True): """Constructor of class Builder Args: - futag_llvm_package (str): path to the futag-llvm package (with binaries, scripts, etc.). library_root (str): path to the library root. flags (str, optional): flags for compiling.. Defaults to COMPILER_FLAGS. clean (bool, optional): Option for deleting futag folders if they are exist, for example futag-build, futag-install, futag-analysis. Defaults to False. @@ -207,7 +210,7 @@ def __init__(self, futag_llvm_package: str, library_root: str, flags: str = "", ValueError: INVALID_INPUT_PROCESSES: the input value of "processes" is not a number or negative. """ - self._validate_common(futag_llvm_package, library_root, processes, build_ex_params) + self._validate_common(library_root, processes, build_ex_params, futag_llvm_package=futag_llvm_package, toolchain=toolchain, log_to_console=log_to_console) if (self.library_root / build_path).exists() and clean: delete_folder(self.library_root / build_path) @@ -280,7 +283,7 @@ def build_meson(self) -> bool: sys.exit(CMAKE_PATH_ERROR) config_cmd = [ - (self.futag_llvm_package / "bin/scan-build").as_posix(), + self.toolchain.scan_build.as_posix(), "meson", f"--prefix={self.install_path.as_posix()}", f"{(self.build_path).as_posix()}", @@ -335,7 +338,7 @@ def build_cmake(self) -> bool: config_cmd = self._scan_build_args() + [ "cmake", - f"-DLLVM_CONFIG_PATH={(self.futag_llvm_package / 'bin/llvm-config').as_posix()}", + f"-DLLVM_CONFIG_PATH={self.toolchain.llvm_config.as_posix()}", f"-DCMAKE_INSTALL_PREFIX={self.install_path.as_posix()}", f"-DCMAKE_EXPORT_COMPILE_COMMANDS=1", f"-B{(self.build_path).as_posix()}", @@ -368,11 +371,11 @@ def build_cmake(self) -> bool: config_cmd = [ "cmake", - f"-DLLVM_CONFIG_PATH={(self.futag_llvm_package / 'bin/llvm-config').as_posix()}", + f"-DLLVM_CONFIG_PATH={self.toolchain.llvm_config.as_posix()}", f"-DCMAKE_INSTALL_PREFIX={self.install_path.as_posix()}", f"-DCMAKE_CXX_FLAGS='{self.flags}'", - f"-DCMAKE_CXX_COMPILER={(self.futag_llvm_package / 'bin/clang++').as_posix()}", - f"-DCMAKE_C_COMPILER={(self.futag_llvm_package / 'bin/clang').as_posix()}", + f"-DCMAKE_CXX_COMPILER={self.toolchain.clangpp.as_posix()}", + f"-DCMAKE_C_COMPILER={self.toolchain.clang.as_posix()}", f"-DCMAKE_C_FLAGS='{self.flags}'", f"-B{(self.build_path).as_posix()}", f"-S{self.library_root.as_posix()}",f"-DCMAKE_EXPORT_COMPILE_COMMANDS=1" @@ -391,8 +394,8 @@ def build_cmake(self) -> bool: os.chdir(self.build_path.as_posix()) # Doing make for building make_command = ["make"] + self._make_jobs_arg() + [ - f"CC={(self.futag_llvm_package / 'bin/clang').as_posix()}", - f"CXX={(self.futag_llvm_package / 'bin/clang++').as_posix()}", + f"CC={self.toolchain.clang.as_posix()}", + f"CXX={self.toolchain.clangpp.as_posix()}", f"CFLAGS={self.flags}", f"CPPFLAGS={self.flags}", f"CXXFLAGS={self.flags}", @@ -454,8 +457,7 @@ def build_configure(self) -> bool: # Doing make for building my_env = self._make_env(with_flags=True) - my_env["LLVM_CONFIG"] = ( - self.futag_llvm_package / 'bin/llvm-config').as_posix() + my_env["LLVM_CONFIG"] = self.toolchain.llvm_config.as_posix() config_cmd = [ (self.library_root / "configure").as_posix(), @@ -468,7 +470,7 @@ def build_configure(self) -> bool: fail_msg=LIB_CONFIGURE_FAILED, succeed_msg=LIB_CONFIGURE_SUCCEEDED) make_command = [ - (self.futag_llvm_package / "bin/intercept-build").as_posix(), + self.toolchain.intercept_build.as_posix(), "make", ] + self._make_jobs_arg() @@ -512,16 +514,15 @@ def build_makefile(self) -> bool: # Doing make for building _run_command(["make", "clean"], capture=True, exit_on_fail=False) - my_env = _make_build_env(self.futag_llvm_package) + my_env = _make_build_env(self.toolchain) my_env["CFLAGS"] = "'" + self.flags + "'" my_env["CPPFLAGS"] = "'" + self.flags + "'" my_env["LDFLAGS"] = "'" + self.flags + "'" - my_env["LLVM_CONFIG"] = ( - self.futag_llvm_package / 'bin/llvm-config').as_posix() + my_env["LLVM_CONFIG"] = self.toolchain.llvm_config.as_posix() if self.intercept: make_command = [ - (self.futag_llvm_package / "bin/intercept-build").as_posix(), + self.toolchain.intercept_build.as_posix(), "make", ] else: @@ -744,11 +745,10 @@ def analyze(self): class ConsumerBuilder(_BaseBuilder): """Futag Builder Class for Consumer programs""" - def __init__(self, futag_llvm_package: str, library_root: str, consumer_root: str, flags: str = "", clean: bool = False, build_path: str = BUILD_PATH, consumer_report_path: str = CONSUMER_REPORT_PATH, db_filepath: str = FOR_CONSUMER_FILEPATH, processes: int = 4, build_ex_params=BUILD_EX_PARAMS): + def __init__(self, library_root: str, consumer_root: str, flags: str = "", clean: bool = False, build_path: str = BUILD_PATH, consumer_report_path: str = CONSUMER_REPORT_PATH, db_filepath: str = FOR_CONSUMER_FILEPATH, processes: int = 4, build_ex_params=BUILD_EX_PARAMS, futag_llvm_package: str = "", toolchain=None, log_to_console: bool = True): """Constructor of class Consumer Builder Args: - futag_llvm_package (str): path to the futag-llvm package (with binaries, scripts, etc.). library_root (str): path to the library root. consumer_root (str): path to the consumer program. flags (str, optional): flags for compiling.. Defaults to COMPILER_FLAGS. @@ -766,7 +766,7 @@ def __init__(self, futag_llvm_package: str, library_root: str, consumer_root: st """ self.consumer_root = consumer_root - self._validate_common(futag_llvm_package, library_root, processes, build_ex_params) + self._validate_common(library_root, processes, build_ex_params, futag_llvm_package=futag_llvm_package, toolchain=toolchain, log_to_console=log_to_console) if pathlib.Path(consumer_root).absolute().exists(): self.consumer_root = pathlib.Path(self.consumer_root).absolute() @@ -851,7 +851,7 @@ def build_cmake(self) -> bool: config_cmd = self._scan_build_args() + [ "cmake", - f"-DLLVM_CONFIG_PATH={(self.futag_llvm_package / 'bin/llvm-config').as_posix()}", + f"-DLLVM_CONFIG_PATH={self.toolchain.llvm_config.as_posix()}", f"-DCMAKE_EXPORT_COMPILE_COMMANDS=1", f"-B{(self.build_path).as_posix()}", f"-S{self.consumer_root.as_posix()}" diff --git a/src/python/futag-package/src/futag/sysmsg.py b/futag-package/src/futag/sysmsg.py similarity index 97% rename from src/python/futag-package/src/futag/sysmsg.py rename to futag-package/src/futag/sysmsg.py index 95918d4b..650afdf7 100644 --- a/src/python/futag-package/src/futag/sysmsg.py +++ b/futag-package/src/futag/sysmsg.py @@ -185,6 +185,5 @@ return 0; }''' -LIBFUZZER_PREFIX_C = '''int LLVMFuzzerTestOneInput(uint8_t * Fuzz_Data, size_t Fuzz_Size){\n''' -LIBFUZZER_PREFIX_CXX = '''extern "C" int LLVMFuzzerTestOneInput(uint8_t * Fuzz_Data, size_t Fuzz_Size){\n''' +LIBFUZZER_PREFIX = '''extern "C" int LLVMFuzzerTestOneInput(uint8_t * Fuzz_Data, size_t Fuzz_Size){\n''' LIBFUZZER_SUFFIX = ''' return 0;\n}''' diff --git a/futag-package/src/futag/toolchain.py b/futag-package/src/futag/toolchain.py new file mode 100644 index 00000000..604f6fd5 --- /dev/null +++ b/futag-package/src/futag/toolchain.py @@ -0,0 +1,156 @@ +# Copyright (c) 2023-2024 ISP RAS (https://www.ispras.ru) +# Licensed under the GNU General Public License v3.0 +# See LICENSE file in the project root for full license text. + +"""Toolchain configuration for external tool paths. + +Provides a ToolchainConfig dataclass that centralizes all external tool +path resolution. Supports three usage modes: + +1. from_futag_llvm() — backward-compatible, uses a compiled futag-llvm directory +2. from_system() — uses system-installed tools via PATH +3. for_generation_only() — no tools needed, only source code generation +""" + +import shutil +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +from futag.exceptions import InvalidPathError + + +@dataclass +class ToolchainConfig: + """Resolved paths to all external tools FUTAG needs. + + All paths are Optional — None means the tool is not available. + Use require_compiler() or require_scan_build() before invoking + a tool to get a clear error message instead of a crash. + """ + + clang: Optional[Path] = None + clangpp: Optional[Path] = None + llvm_config: Optional[Path] = None + scan_build: Optional[Path] = None + llvm_profdata: Optional[Path] = None + llvm_cov: Optional[Path] = None + llvm_symbolizer: Optional[Path] = None + intercept_build: Optional[Path] = None + afl_clang_fast: Optional[Path] = None + afl_clang_fastpp: Optional[Path] = None + svres_template: Optional[Path] = None + + @classmethod + def from_futag_llvm(cls, futag_llvm_package: str) -> "ToolchainConfig": + """Construct from a futag-llvm directory. + + Validates that the directory exists and contains bin/clang. + Optional tools (AFL++, svres template) are set only if present. + + Args: + futag_llvm_package: Path to the compiled futag-llvm directory. + + Raises: + InvalidPathError: If the directory or bin/clang doesn't exist. + """ + base = Path(futag_llvm_package).absolute() + if not base.exists() or not (base / "bin" / "clang").exists(): + raise InvalidPathError( + f"Invalid futag-llvm path: {futag_llvm_package}") + + afl_base = base / "AFLplusplus" / "usr" / "local" / "bin" + svres_path = base / "svres-tmpl" / "svres.tmpl" + + return cls( + clang=base / "bin" / "clang", + clangpp=base / "bin" / "clang++", + llvm_config=base / "bin" / "llvm-config", + scan_build=base / "bin" / "scan-build", + llvm_profdata=base / "bin" / "llvm-profdata", + llvm_cov=base / "bin" / "llvm-cov", + llvm_symbolizer=base / "bin" / "llvm-symbolizer", + intercept_build=base / "bin" / "intercept-build", + afl_clang_fast=( + afl_base / "afl-clang-fast" + if (afl_base / "afl-clang-fast").exists() else None + ), + afl_clang_fastpp=( + afl_base / "afl-clang-fast++" + if (afl_base / "afl-clang-fast++").exists() else None + ), + svres_template=svres_path if svres_path.exists() else None, + ) + + @classmethod + def from_system(cls, clang_path: str = "") -> "ToolchainConfig": + """Use system-installed tools discovered via PATH. + + Args: + clang_path: Explicit path to clang. If empty, searches PATH. + + Returns: + ToolchainConfig with paths set for tools found on the system. + Missing tools will have None paths. + """ + def find(name): + p = shutil.which(name) + return Path(p) if p else None + + if clang_path: + clang = Path(clang_path) + clangpp = Path(clang_path + "++") + else: + clang = find("clang") + clangpp = find("clang++") + + return cls( + clang=clang, + clangpp=clangpp, + llvm_config=find("llvm-config"), + scan_build=find("scan-build"), + llvm_profdata=find("llvm-profdata"), + llvm_cov=find("llvm-cov"), + llvm_symbolizer=find("llvm-symbolizer"), + intercept_build=find("intercept-build"), + afl_clang_fast=find("afl-clang-fast"), + afl_clang_fastpp=find("afl-clang-fast++"), + ) + + @classmethod + def for_generation_only(cls) -> "ToolchainConfig": + """Minimal config with all paths set to None. + + Use this when you only need gen_targets() to produce source files + without compilation. compile_targets() will raise InvalidPathError. + """ + return cls() + + def require_compiler(self, target_type: int = 0): + """Validate that a compiler is available for the given target type. + + Args: + target_type: 0 for LIBFUZZER (needs clang), 1 for AFLPLUSPLUS + (needs afl-clang-fast). + + Raises: + InvalidPathError: If the required compiler is not available. + """ + if target_type == 0: # LIBFUZZER + if not self.clang or not self.clang.exists(): + raise InvalidPathError( + "clang compiler not found in toolchain config") + else: # AFLPLUSPLUS + if not self.afl_clang_fast or not self.afl_clang_fast.exists(): + raise InvalidPathError( + "afl-clang-fast not found in toolchain config") + + def require_scan_build(self): + """Validate that scan-build is available. + + Raises: + InvalidPathError: If scan-build is not available. + """ + if not self.scan_build or not self.scan_build.exists(): + raise InvalidPathError( + "scan-build not found in toolchain config") diff --git a/src/python/futag-package/tests/__init__.py b/futag-package/tests/__init__.py similarity index 100% rename from src/python/futag-package/tests/__init__.py rename to futag-package/tests/__init__.py diff --git a/src/python/futag-package/tests/conftest.py b/futag-package/tests/conftest.py similarity index 98% rename from src/python/futag-package/tests/conftest.py rename to futag-package/tests/conftest.py index 36b21e81..dad02aaa 100644 --- a/src/python/futag-package/tests/conftest.py +++ b/futag-package/tests/conftest.py @@ -172,7 +172,9 @@ def tmp_futag_package(tmp_path): bin_dir.mkdir(parents=True) # Create placeholder binaries - for name in ["clang", "clang++", "llvm-config", "scan-build", "intercept-build"]: + for name in ["clang", "clang++", "llvm-config", "scan-build", + "intercept-build", "llvm-profdata", "llvm-cov", + "llvm-symbolizer"]: (bin_dir / name).touch() (bin_dir / name).chmod(0o755) diff --git a/src/python/futag-package/tests/test_exceptions.py b/futag-package/tests/test_exceptions.py similarity index 100% rename from src/python/futag-package/tests/test_exceptions.py rename to futag-package/tests/test_exceptions.py diff --git a/src/python/futag-package/tests/test_fdp_generator.py b/futag-package/tests/test_fdp_generator.py similarity index 91% rename from src/python/futag-package/tests/test_fdp_generator.py rename to futag-package/tests/test_fdp_generator.py index 47b88444..34355000 100644 --- a/src/python/futag-package/tests/test_fdp_generator.py +++ b/futag-package/tests/test_fdp_generator.py @@ -10,7 +10,9 @@ @pytest.fixture def fdp_generator(tmp_futag_package, tmp_library_root): - return FuzzDataProviderGenerator(tmp_futag_package, tmp_library_root) + from futag.toolchain import ToolchainConfig + tc = ToolchainConfig.from_futag_llvm(tmp_futag_package) + return FuzzDataProviderGenerator(tmp_library_root, toolchain=tc) class TestFDPProperties: diff --git a/src/python/futag-package/tests/test_fuzzer.py b/futag-package/tests/test_fuzzer.py similarity index 55% rename from src/python/futag-package/tests/test_fuzzer.py rename to futag-package/tests/test_fuzzer.py index 55a2bf4b..0010a6ce 100644 --- a/src/python/futag-package/tests/test_fuzzer.py +++ b/futag-package/tests/test_fuzzer.py @@ -1,4 +1,5 @@ """Tests for the Fuzzer module.""" +import logging import sys import os import pytest @@ -6,6 +7,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) from futag.fuzzer import BaseFuzzer, Fuzzer, NatchFuzzer +from futag.toolchain import ToolchainConfig class TestFuzzerClassHierarchy: @@ -56,3 +58,32 @@ class TestCorpusArgs: def test_fuzzer_returns_empty(self): fuzzer = Fuzzer.__new__(Fuzzer) assert fuzzer._get_corpus_args(None) == [] + + +class TestFuzzMethod: + def test_fuzz_missing_succeeded_logs_error_not_raises(self, tmp_path, caplog): + """fuzz() must log an error and return when succeeded/ is absent.""" + fuzzer = Fuzzer( + fuzz_driver_path=str(tmp_path), # exists, but no succeeded/ inside + toolchain=ToolchainConfig.for_generation_only(), + log_to_console=False, + ) + with caplog.at_level(logging.ERROR, logger="futag.fuzzer"): + fuzzer.fuzz() # must NOT raise FileNotFoundError + + error_messages = [r.message for r in caplog.records if r.levelno == logging.ERROR] + assert any("succeeded" in msg for msg in error_messages) + + def test_fuzz_empty_succeeded_dir_runs_without_error(self, tmp_path, caplog): + """fuzz() with an empty succeeded/ directory completes without error.""" + (tmp_path / "succeeded").mkdir() + fuzzer = Fuzzer( + fuzz_driver_path=str(tmp_path), + toolchain=ToolchainConfig.for_generation_only(), + log_to_console=False, + ) + with caplog.at_level(logging.ERROR, logger="futag.fuzzer"): + fuzzer.fuzz() + + # No error should be logged — succeeded/ exists, just empty + assert not any(r.levelno == logging.ERROR for r in caplog.records) diff --git a/src/python/futag-package/tests/test_generator.py b/futag-package/tests/test_generator.py similarity index 83% rename from src/python/futag-package/tests/test_generator.py rename to futag-package/tests/test_generator.py index a8c3257f..7432d26f 100644 --- a/src/python/futag-package/tests/test_generator.py +++ b/futag-package/tests/test_generator.py @@ -12,7 +12,9 @@ @pytest.fixture def generator(tmp_futag_package, tmp_library_root): """Create a Generator instance with mock paths.""" - return Generator(tmp_futag_package, tmp_library_root) + from futag.toolchain import ToolchainConfig + tc = ToolchainConfig.from_futag_llvm(tmp_futag_package) + return Generator(tmp_library_root, toolchain=tc) class TestGenBuiltin: @@ -102,3 +104,18 @@ class TestGenPointer: def test_pointer(self, generator): result = generator._gen_pointer("p_x", "x", {"type_name": "int *"}) assert any("& x" in line for line in result["gen_lines"]) + + +class TestToolchainIntegration: + def test_toolchain_kwarg(self, tmp_futag_package, tmp_library_root): + """Pass toolchain explicitly via keyword arg.""" + from futag.toolchain import ToolchainConfig + tc = ToolchainConfig.from_futag_llvm(tmp_futag_package) + gen = Generator(tmp_library_root, toolchain=tc) + assert gen.toolchain is tc + assert gen.toolchain.clang is not None + + def test_generation_only_mode(self, tmp_library_root): + """Generation-only: no toolchain, library_root is first arg.""" + gen = Generator(tmp_library_root) + assert gen.toolchain.clang is None diff --git a/src/python/futag-package/tests/test_generator_state.py b/futag-package/tests/test_generator_state.py similarity index 100% rename from src/python/futag-package/tests/test_generator_state.py rename to futag-package/tests/test_generator_state.py diff --git a/src/python/futag-package/tests/test_preprocessor.py b/futag-package/tests/test_preprocessor.py similarity index 100% rename from src/python/futag-package/tests/test_preprocessor.py rename to futag-package/tests/test_preprocessor.py diff --git a/futag-package/tests/test_toolchain.py b/futag-package/tests/test_toolchain.py new file mode 100644 index 00000000..269ccb5a --- /dev/null +++ b/futag-package/tests/test_toolchain.py @@ -0,0 +1,108 @@ +"""Tests for the ToolchainConfig dataclass.""" +import sys +import os +import pytest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src")) + +from futag.toolchain import ToolchainConfig +from futag.exceptions import InvalidPathError + + +class TestFromFutagLlvm: + def test_valid_path(self, tmp_futag_package): + """Existing tmp_futag_package fixture creates fake futag-llvm with bin/clang.""" + tc = ToolchainConfig.from_futag_llvm(tmp_futag_package) + assert tc.clang is not None + assert tc.clang.name == "clang" + assert tc.clangpp is not None + assert tc.clangpp.name == "clang++" + assert tc.scan_build is not None + assert tc.llvm_profdata is not None + assert tc.llvm_cov is not None + assert tc.llvm_symbolizer is not None + + def test_invalid_path_raises(self): + """Nonexistent directory should raise InvalidPathError.""" + with pytest.raises(InvalidPathError): + ToolchainConfig.from_futag_llvm("/nonexistent/path") + + def test_missing_clang_raises(self, tmp_path): + """Directory exists but no bin/clang should raise.""" + d = tmp_path / "empty-llvm" + d.mkdir() + with pytest.raises(InvalidPathError): + ToolchainConfig.from_futag_llvm(str(d)) + + def test_afl_none_when_missing(self, tmp_futag_package): + """AFL++ tools should be None when not present.""" + tc = ToolchainConfig.from_futag_llvm(tmp_futag_package) + assert tc.afl_clang_fast is None + assert tc.afl_clang_fastpp is None + + def test_svres_none_when_missing(self, tmp_futag_package): + """svres template should be None when not present.""" + tc = ToolchainConfig.from_futag_llvm(tmp_futag_package) + assert tc.svres_template is None + + +class TestFromSystem: + def test_returns_config(self): + """Should not raise even if tools are not found.""" + tc = ToolchainConfig.from_system() + assert isinstance(tc, ToolchainConfig) + + def test_custom_clang_path(self, tmp_path): + """Explicit clang path should be used directly.""" + fake_clang = tmp_path / "my-clang" + fake_clang.touch() + fake_clang.chmod(0o755) + tc = ToolchainConfig.from_system(str(fake_clang)) + assert tc.clang == fake_clang + + +class TestForGenerationOnly: + def test_all_none(self): + """All paths should be None in generation-only mode.""" + tc = ToolchainConfig.for_generation_only() + assert tc.clang is None + assert tc.clangpp is None + assert tc.scan_build is None + assert tc.llvm_profdata is None + assert tc.llvm_cov is None + assert tc.llvm_symbolizer is None + assert tc.afl_clang_fast is None + assert tc.afl_clang_fastpp is None + assert tc.svres_template is None + + +class TestRequireCompiler: + def test_raises_when_none_libfuzzer(self): + """Should raise for LIBFUZZER when clang is None.""" + tc = ToolchainConfig.for_generation_only() + with pytest.raises(InvalidPathError, match="clang"): + tc.require_compiler(target_type=0) + + def test_raises_when_none_aflplusplus(self): + """Should raise for AFLPLUSPLUS when afl-clang-fast is None.""" + tc = ToolchainConfig.for_generation_only() + with pytest.raises(InvalidPathError, match="afl-clang-fast"): + tc.require_compiler(target_type=1) + + def test_passes_when_set(self, tmp_futag_package): + """Should not raise when clang is available.""" + tc = ToolchainConfig.from_futag_llvm(tmp_futag_package) + tc.require_compiler(target_type=0) # should not raise + + +class TestRequireScanBuild: + def test_raises_when_none(self): + """Should raise when scan-build is None.""" + tc = ToolchainConfig.for_generation_only() + with pytest.raises(InvalidPathError, match="scan-build"): + tc.require_scan_build() + + def test_passes_when_set(self, tmp_futag_package): + """Should not raise when scan-build is available.""" + tc = ToolchainConfig.from_futag_llvm(tmp_futag_package) + tc.require_scan_build() # should not raise diff --git a/product-tests/README.md b/integration-tests/README.md similarity index 100% rename from product-tests/README.md rename to integration-tests/README.md diff --git a/product-tests/build-test/alt11/alt11.Dockerfile b/integration-tests/build-test/alt11/alt11.Dockerfile similarity index 95% rename from product-tests/build-test/alt11/alt11.Dockerfile rename to integration-tests/build-test/alt11/alt11.Dockerfile index c8d85656..c013e898 100644 --- a/product-tests/build-test/alt11/alt11.Dockerfile +++ b/integration-tests/build-test/alt11/alt11.Dockerfile @@ -14,7 +14,7 @@ RUN pwd RUN git clone https://github.com/ispras/Futag.git WORKDIR /home/futag/Futag/ RUN git checkout llvm18 -WORKDIR /home/futag/Futag/custom-llvm +WORKDIR /home/futag/Futag/build-llvm RUN ./prepare.sh 2 WORKDIR /home/futag/Futag/build RUN ./build.sh diff --git a/product-tests/build-test/alt11/build.sh b/integration-tests/build-test/alt11/build.sh similarity index 100% rename from product-tests/build-test/alt11/build.sh rename to integration-tests/build-test/alt11/build.sh diff --git a/product-tests/build-test/alt11/run.sh b/integration-tests/build-test/alt11/run.sh similarity index 100% rename from product-tests/build-test/alt11/run.sh rename to integration-tests/build-test/alt11/run.sh diff --git a/product-tests/build-test/ubuntu20/Docker-test-build-run.sh b/integration-tests/build-test/ubuntu20/Docker-test-build-run.sh similarity index 100% rename from product-tests/build-test/ubuntu20/Docker-test-build-run.sh rename to integration-tests/build-test/ubuntu20/Docker-test-build-run.sh diff --git a/product-tests/build-test/ubuntu20/Docker-test-build.Dockerfile b/integration-tests/build-test/ubuntu20/Docker-test-build.Dockerfile similarity index 96% rename from product-tests/build-test/ubuntu20/Docker-test-build.Dockerfile rename to integration-tests/build-test/ubuntu20/Docker-test-build.Dockerfile index 1cf67231..32fab712 100644 --- a/product-tests/build-test/ubuntu20/Docker-test-build.Dockerfile +++ b/integration-tests/build-test/ubuntu20/Docker-test-build.Dockerfile @@ -17,7 +17,7 @@ RUN apt install -y libncurses5 libtinfo5 gcc-multilib g++ make gdb binutils binu USER futag WORKDIR /home/futag/ RUN git clone --depth 1 https://github.com/ispras/Futag.git -WORKDIR /home/futag/Futag/custom-llvm +WORKDIR /home/futag/Futag/build-llvm RUN ./prepare.sh 1 WORKDIR /home/futag/Futag/build RUN ./build.sh diff --git a/product-tests/build-test/ubuntu20/Docker-test-build.sh b/integration-tests/build-test/ubuntu20/Docker-test-build.sh similarity index 100% rename from product-tests/build-test/ubuntu20/Docker-test-build.sh rename to integration-tests/build-test/ubuntu20/Docker-test-build.sh diff --git a/product-tests/build-test/ubuntu20/binutils-futag.tar.xz b/integration-tests/build-test/ubuntu20/binutils-futag.tar.xz similarity index 100% rename from product-tests/build-test/ubuntu20/binutils-futag.tar.xz rename to integration-tests/build-test/ubuntu20/binutils-futag.tar.xz diff --git a/product-tests/build-test/ubuntu22/Docker-test-build-run.sh b/integration-tests/build-test/ubuntu22/Docker-test-build-run.sh similarity index 100% rename from product-tests/build-test/ubuntu22/Docker-test-build-run.sh rename to integration-tests/build-test/ubuntu22/Docker-test-build-run.sh diff --git a/product-tests/build-test/ubuntu22/Docker-test-build.Dockerfile b/integration-tests/build-test/ubuntu22/Docker-test-build.Dockerfile similarity index 95% rename from product-tests/build-test/ubuntu22/Docker-test-build.Dockerfile rename to integration-tests/build-test/ubuntu22/Docker-test-build.Dockerfile index 64da4026..e3ba69dc 100644 --- a/product-tests/build-test/ubuntu22/Docker-test-build.Dockerfile +++ b/integration-tests/build-test/ubuntu22/Docker-test-build.Dockerfile @@ -14,7 +14,7 @@ RUN apt install -y libncurses5 gcc-multilib g++ make gdb binutils python3 git op WORKDIR /home/futag/ RUN pwd RUN git clone --depth 1 https://github.com/ispras/Futag.git -WORKDIR /home/futag/Futag/custom-llvm +WORKDIR /home/futag/Futag/build-llvm RUN ./prepare.sh 1 WORKDIR /home/futag/Futag/build RUN ./build.sh diff --git a/product-tests/build-test/ubuntu22/Docker-test-build.sh b/integration-tests/build-test/ubuntu22/Docker-test-build.sh similarity index 100% rename from product-tests/build-test/ubuntu22/Docker-test-build.sh rename to integration-tests/build-test/ubuntu22/Docker-test-build.sh diff --git a/product-tests/build-test/ubuntu24/build.sh b/integration-tests/build-test/ubuntu24/build.sh similarity index 100% rename from product-tests/build-test/ubuntu24/build.sh rename to integration-tests/build-test/ubuntu24/build.sh diff --git a/product-tests/build-test/ubuntu24/run.sh b/integration-tests/build-test/ubuntu24/run.sh similarity index 100% rename from product-tests/build-test/ubuntu24/run.sh rename to integration-tests/build-test/ubuntu24/run.sh diff --git a/product-tests/build-test/ubuntu24/ubuntu24.Dockerfile b/integration-tests/build-test/ubuntu24/ubuntu24.Dockerfile similarity index 95% rename from product-tests/build-test/ubuntu24/ubuntu24.Dockerfile rename to integration-tests/build-test/ubuntu24/ubuntu24.Dockerfile index 5dbe51e3..4a1cac3e 100644 --- a/product-tests/build-test/ubuntu24/ubuntu24.Dockerfile +++ b/integration-tests/build-test/ubuntu24/ubuntu24.Dockerfile @@ -15,7 +15,7 @@ WORKDIR /home/futag/ RUN git clone https://github.com/ispras/Futag.git WORKDIR /home/futag/Futag/ RUN git checkout llvm18 -WORKDIR /home/futag/Futag/custom-llvm +WORKDIR /home/futag/Futag/build-llvm RUN ./prepare.sh 1 RUN pwd WORKDIR /home/futag/Futag/build diff --git a/product-tests/libraries-test/alt11/README.md b/integration-tests/libraries-test/alt11/README.md similarity index 100% rename from product-tests/libraries-test/alt11/README.md rename to integration-tests/libraries-test/alt11/README.md diff --git a/product-tests/libraries-test/alt11/alt11.Dockerfile b/integration-tests/libraries-test/alt11/alt11.Dockerfile similarity index 100% rename from product-tests/libraries-test/alt11/alt11.Dockerfile rename to integration-tests/libraries-test/alt11/alt11.Dockerfile diff --git a/product-tests/libraries-test/alt11/build.sh b/integration-tests/libraries-test/alt11/build.sh similarity index 100% rename from product-tests/libraries-test/alt11/build.sh rename to integration-tests/libraries-test/alt11/build.sh diff --git a/product-tests/libraries-test/alt11/run.sh b/integration-tests/libraries-test/alt11/run.sh similarity index 100% rename from product-tests/libraries-test/alt11/run.sh rename to integration-tests/libraries-test/alt11/run.sh diff --git a/product-tests/libraries-test/ubuntu20/Docker-test-libs-run.sh b/integration-tests/libraries-test/ubuntu20/Docker-test-libs-run.sh similarity index 100% rename from product-tests/libraries-test/ubuntu20/Docker-test-libs-run.sh rename to integration-tests/libraries-test/ubuntu20/Docker-test-libs-run.sh diff --git a/product-tests/libraries-test/ubuntu20/Docker-test-libs.Dockerfile b/integration-tests/libraries-test/ubuntu20/Docker-test-libs.Dockerfile similarity index 100% rename from product-tests/libraries-test/ubuntu20/Docker-test-libs.Dockerfile rename to integration-tests/libraries-test/ubuntu20/Docker-test-libs.Dockerfile diff --git a/product-tests/libraries-test/ubuntu20/Docker-test-libs.sh b/integration-tests/libraries-test/ubuntu20/Docker-test-libs.sh similarity index 100% rename from product-tests/libraries-test/ubuntu20/Docker-test-libs.sh rename to integration-tests/libraries-test/ubuntu20/Docker-test-libs.sh diff --git a/product-tests/libraries-test/ubuntu22/Docker-test-libs-run.sh b/integration-tests/libraries-test/ubuntu22/Docker-test-libs-run.sh similarity index 100% rename from product-tests/libraries-test/ubuntu22/Docker-test-libs-run.sh rename to integration-tests/libraries-test/ubuntu22/Docker-test-libs-run.sh diff --git a/product-tests/libraries-test/ubuntu22/Docker-test-libs.Dockerfile b/integration-tests/libraries-test/ubuntu22/Docker-test-libs.Dockerfile similarity index 100% rename from product-tests/libraries-test/ubuntu22/Docker-test-libs.Dockerfile rename to integration-tests/libraries-test/ubuntu22/Docker-test-libs.Dockerfile diff --git a/product-tests/libraries-test/ubuntu22/Docker-test-libs.sh b/integration-tests/libraries-test/ubuntu22/Docker-test-libs.sh similarity index 100% rename from product-tests/libraries-test/ubuntu22/Docker-test-libs.sh rename to integration-tests/libraries-test/ubuntu22/Docker-test-libs.sh diff --git a/product-tests/package-test/ubuntu20/Docker-test-package-run.sh b/integration-tests/package-test/ubuntu20/Docker-test-package-run.sh similarity index 100% rename from product-tests/package-test/ubuntu20/Docker-test-package-run.sh rename to integration-tests/package-test/ubuntu20/Docker-test-package-run.sh diff --git a/product-tests/package-test/ubuntu20/Docker-test-package.Dockerfile b/integration-tests/package-test/ubuntu20/Docker-test-package.Dockerfile similarity index 100% rename from product-tests/package-test/ubuntu20/Docker-test-package.Dockerfile rename to integration-tests/package-test/ubuntu20/Docker-test-package.Dockerfile diff --git a/product-tests/package-test/ubuntu20/Docker-test-package.sh b/integration-tests/package-test/ubuntu20/Docker-test-package.sh similarity index 100% rename from product-tests/package-test/ubuntu20/Docker-test-package.sh rename to integration-tests/package-test/ubuntu20/Docker-test-package.sh diff --git a/product-tests/package-test/ubuntu22/Docker-test-package-run.sh b/integration-tests/package-test/ubuntu22/Docker-test-package-run.sh similarity index 100% rename from product-tests/package-test/ubuntu22/Docker-test-package-run.sh rename to integration-tests/package-test/ubuntu22/Docker-test-package-run.sh diff --git a/product-tests/package-test/ubuntu22/Docker-test-package.Dockerfile b/integration-tests/package-test/ubuntu22/Docker-test-package.Dockerfile similarity index 100% rename from product-tests/package-test/ubuntu22/Docker-test-package.Dockerfile rename to integration-tests/package-test/ubuntu22/Docker-test-package.Dockerfile diff --git a/product-tests/package-test/ubuntu22/Docker-test-package.sh b/integration-tests/package-test/ubuntu22/Docker-test-package.sh similarity index 100% rename from product-tests/package-test/ubuntu22/Docker-test-package.sh rename to integration-tests/package-test/ubuntu22/Docker-test-package.sh diff --git a/product-tests/prepare-package.sh b/integration-tests/prepare-package.sh similarity index 100% rename from product-tests/prepare-package.sh rename to integration-tests/prepare-package.sh diff --git a/src/python/futag-parse.py b/scripts/futag-parse.py similarity index 100% rename from src/python/futag-parse.py rename to scripts/futag-parse.py diff --git a/src/python/requirements.txt b/scripts/requirements.txt similarity index 100% rename from src/python/requirements.txt rename to scripts/requirements.txt diff --git a/src/python/template-script.py b/scripts/template-script.py similarity index 87% rename from src/python/template-script.py rename to scripts/template-script.py index 89bfe6d8..d52c984d 100644 --- a/src/python/template-script.py +++ b/scripts/template-script.py @@ -7,13 +7,14 @@ """ from futag.preprocessor import * from futag.generator import * +from futag.toolchain import ToolchainConfig # ============================================================================= # Pattern 1: Build, analyze, and generate fuzz targets for a library # ============================================================================= +tc = ToolchainConfig.from_futag_llvm("../futag-llvm/") test_build = Builder( - "../futag-llvm", # Path to the futag-llvm working directory "../json-c", # Path to the library source directory flags="-g -O0", # Compiler flags for building clean=True, # Clean futag-build/install/analysis dirs before running (default: False) @@ -21,15 +22,15 @@ install_path="../json-c/futag-install", # Path to install directory (optional) analysis_path="../json-c/futag-analysis", # Path to analysis directory (optional) processes=4, # Number of CPU cores for building (optional) - build_ex_params="--disable-zip" # Extra build parameters (optional) + build_ex_params="--disable-zip", # Extra build parameters (optional) + toolchain=tc, ) test_build.auto_build() test_build.analyze() - generator = Generator( - "../futag-llvm/", "json-c", + toolchain=tc, ) generator.gen_targets() generator.compile_targets( @@ -45,10 +46,11 @@ library_root = "json-c-json-c-0.16-20220414" consumer_root = "libstorj-1.0.3" +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) consumer_builder = ConsumerBuilder( - FUTAG_PATH, # Path to the futag-llvm directory library_root, # Path to the library source directory consumer_root, # Path to the consumer program source directory + toolchain=tc, # clean=True, # processes=16, ) @@ -59,9 +61,10 @@ # Pattern 3: Generate fuzz targets from consumer usage contexts # ============================================================================= +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) context_generator = ContextGenerator( - FUTAG_PATH, library_root, + toolchain=tc, ) context_generator.gen_context() # Generate fuzz wrappers for contexts diff --git a/workshop/json-c/futag.all-in-one.py b/workshop/json-c/futag.all-in-one.py index 748d17bc..3c985d4a 100644 --- a/workshop/json-c/futag.all-in-one.py +++ b/workshop/json-c/futag.all-in-one.py @@ -2,25 +2,29 @@ # This file is distributed under the GPL v3 license (https://www.gnu.org/licenses/gpl-3.0.en.html). from futag.preprocessor import * -from futag.fdp_generator import * -from futag.sysmsg import * -from futag.fuzzer import * +from futag.fdp_generator import * +from futag.sysmsg import * +from futag.fuzzer import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "../futag-llvm" -lib_path = "json-c-json-c-0.18-20240915" +lib_path = "json-c-json-c-0.18-20240915" + +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) build_test = Builder( - FUTAG_PATH, lib_path, clean=True, processes=16, + toolchain=tc, ) build_test.auto_build() build_test.analyze() +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) generator = FuzzDataProviderGenerator( - FUTAG_PATH, lib_path, target_type=LIBFUZZER, + toolchain=tc, ) generator.gen_targets() @@ -31,8 +35,8 @@ ) fuzzer = Fuzzer( - FUTAG_PATH, "json-c-json-c-0.18-20240915/futag-fuzz-drivers", + toolchain=tc, debug=True, svres=True, totaltime= 10, diff --git a/workshop/json-c/futag.analysis.py b/workshop/json-c/futag.analysis.py index fce92c65..be1e1661 100644 --- a/workshop/json-c/futag.analysis.py +++ b/workshop/json-c/futag.analysis.py @@ -2,17 +2,20 @@ # This file is distributed under the GPL v3 license (https://www.gnu.org/licenses/gpl-3.0.en.html). from futag.preprocessor import * -from futag.fdp_generator import * -from futag.sysmsg import * -from futag.fuzzer import * +from futag.fdp_generator import * +from futag.sysmsg import * +from futag.fuzzer import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "../futag-llvm" -lib_path = "json-c-json-c-0.18-20240915" +lib_path = "json-c-json-c-0.18-20240915" + +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) build_test = Builder( - FUTAG_PATH, lib_path, clean=True, processes=16, + toolchain=tc, ) build_test.auto_build() build_test.analyze() \ No newline at end of file diff --git a/workshop/json-c/futag.fuzzing.py b/workshop/json-c/futag.fuzzing.py index aef18dfc..fc052368 100644 --- a/workshop/json-c/futag.fuzzing.py +++ b/workshop/json-c/futag.fuzzing.py @@ -1,15 +1,17 @@ # Futag-tests (https://github.com/thientc/Futag-tests): testing repository for Futag. # This file is distributed under the GPL v3 license (https://www.gnu.org/licenses/gpl-3.0.en.html). -from futag.sysmsg import * -from futag.fuzzer import * +from futag.sysmsg import * +from futag.fuzzer import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "../futag-llvm" -lib_path = "json-c-json-c-0.18-20240915" +lib_path = "json-c-json-c-0.18-20240915" +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) fuzzer = Fuzzer( - FUTAG_PATH, "json-c-json-c-0.18-20240915/futag-fuzz-drivers", + toolchain=tc, debug=True, svres=True, totaltime= 10, diff --git a/workshop/json-c/futag.generator.py b/workshop/json-c/futag.generator.py index 8d5f96fe..6d322e84 100644 --- a/workshop/json-c/futag.generator.py +++ b/workshop/json-c/futag.generator.py @@ -2,17 +2,19 @@ # This file is distributed under the GPL v3 license (https://www.gnu.org/licenses/gpl-3.0.en.html). from futag.preprocessor import * -from futag.fdp_generator import * -from futag.sysmsg import * -from futag.fuzzer import * +from futag.fdp_generator import * +from futag.sysmsg import * +from futag.fuzzer import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "../futag-llvm" -lib_path = "json-c-json-c-0.18-20240915" +lib_path = "json-c-json-c-0.18-20240915" +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) generator = FuzzDataProviderGenerator( - FUTAG_PATH, lib_path, target_type=LIBFUZZER, + toolchain=tc, ) generator.gen_targets() diff --git a/workshop/libvirt/futag.analysis.py b/workshop/libvirt/futag.analysis.py index 31ea3112..54c18772 100644 --- a/workshop/libvirt/futag.analysis.py +++ b/workshop/libvirt/futag.analysis.py @@ -2,14 +2,17 @@ # This file is distributed under the GPL v3 license (https://www.gnu.org/licenses/gpl-3.0.en.html). from futag.preprocessor import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "/home/futag/Futag/futag-llvm" lib_path = "." + +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) test_build = Builder( - FUTAG_PATH, lib_path, clean=False, analysis_path='futag-analysis', + toolchain=tc, ) # test_build.auto_build() test_build.analyze() diff --git a/workshop/libvirt/futag.fuzzing.py b/workshop/libvirt/futag.fuzzing.py index 85af8412..2183aba5 100644 --- a/workshop/libvirt/futag.fuzzing.py +++ b/workshop/libvirt/futag.fuzzing.py @@ -1,14 +1,16 @@ # Futag-tests (https://github.com/thientc/Futag-tests): testing repository for Futag. # This file is distributed under the GPL v3 license (https://www.gnu.org/licenses/gpl-3.0.en.html). -from futag.fuzzer import * +from futag.fuzzer import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "/home/futag/Futag/futag-llvm" lib_path = "." +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) fuzzer = Fuzzer( # модуль для фаззинга - FUTAG_PATH, - fuzz_driver_path="futag-fuzz-drivers/", + fuzz_driver_path="futag-fuzz-drivers/", + toolchain=tc, totaltime=3, # время фаззинга одной обертки debug=True, gdb=True, diff --git a/workshop/libvirt/futag.generator.py b/workshop/libvirt/futag.generator.py index 21d412ce..b4c914f2 100644 --- a/workshop/libvirt/futag.generator.py +++ b/workshop/libvirt/futag.generator.py @@ -1,14 +1,16 @@ # Futag-tests (https://github.com/thientc/Futag-tests): testing repository for Futag. # This file is distributed under the GPL v3 license (https://www.gnu.org/licenses/gpl-3.0.en.html). -from futag.generator import * +from futag.generator import * +from futag.toolchain import ToolchainConfig FUTAG_PATH = "/home/futag/Futag/futag-llvm" lib_path = "." +tc = ToolchainConfig.from_futag_llvm(FUTAG_PATH) generator = Generator( - FUTAG_PATH, lib_path, + toolchain=tc, json_file='/home/futag/libvirt/futag-analysis/futag-analysis-result.json', build_path='/home/futag/RPM/BUILD/libvirt-10.7.0/x86_64-alt-linux', #for compile_command.json with cmake install_path='/home/futag/RPM/BUILD/libvirt-10.7.0/x86_64-alt-linux', #for compile_command.json with cmake,