diff --git a/scripts/gen_traf_tcp-udp/README.md b/scripts/gen_traf_tcp-udp/README.md new file mode 100644 index 0000000..aa80f85 --- /dev/null +++ b/scripts/gen_traf_tcp-udp/README.md @@ -0,0 +1,46 @@ +# Traffic Generator + +Асинхронный генератор трафика для тестирования сетевых подключений с использованием `nmap`. Запускает заданное количество запросов к случайным сайтам из списка с контролем RPS (requests per second), таймаутами и ограничением параллельных задач. Результаты сканирования портов (TCP/UDP) сохраняются в JSON-лог (по умолчанию logs/LOG: YYYY-MM-DD_HH-MM-SS-ffffff.json). + +## Требования + +- Python 3.7+ +- Nmap (устанавливается автоматически) +- Зависимости Python: `python-nmap` + +## Установка + +```bash +# Запустите скрипт установки (установит nmap и python-зависимости) +chmod +x setup.sh +./setup.sh +``` + + +## Использование +```bash +sudo python3 generate_rand_traf.py [параметры] +``` + +Параметры командной строки +Короткая Полная По умолчанию Описание +-q --quantity 10 Количество запросов +-r --rps 15 Желаемое количество запросов в секунду +-t --timeout 20 Таймаут на один запрос (секунды) +-f --file sites.txt Файл со списком сайтов +-m --max_concurrent 50 Максимальное число одновременно выполняемых задач +-l --log_level INFO Уровень логирования (DEBUG, INFO, WARNING, ERROR, CRITICAL) +-ncl --no_console_log (флаг) Отключить логирование в консоль +-n --name_folder_log logs Имя папки для логов + +## Тестирование +Проект включает интеграционные тесты для проверки основной функциональности: + +```bash +sudo python3 test.py +``` +Тесты проверяют: + Работу с заведомо существующими сайтами + Обработку несуществующих доменов (используются зарезервированные домены .test, .invalid, .local) + Смешанные сценарии с разными типами сайтов + Корректность подсчета статистики \ No newline at end of file diff --git a/scripts/gen_traf_tcp-udp/generate_rand_traf.py b/scripts/gen_traf_tcp-udp/generate_rand_traf.py new file mode 100644 index 0000000..db22580 --- /dev/null +++ b/scripts/gen_traf_tcp-udp/generate_rand_traf.py @@ -0,0 +1,296 @@ +import asyncio +import random +import sys +import argparse +from datetime import datetime +import json +import logging +from pathlib import Path +import nmap +import requests +from bs4 import BeautifulSoup +import urllib3 +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + +def pars(): + parser = argparse.ArgumentParser( + description="Traffic generator with RPS control and timeouts", + formatter_class=argparse.RawTextHelpFormatter + ) + parser.add_argument( + "--quantity", "-q", + type=int, + default=10, + help="Number of requests (10 by default)" + ) + parser.add_argument( + "--rps", "-r", + type=int, + default=15, + help="The desired number of requests per second (15 by default)" + ) + parser.add_argument( + "--timeout", "-t", + type=int, + default=20, + help="Timeout per request in seconds (20 by default)" + ) + parser.add_argument( + "--file", "-f", + type=str, + default="sites.txt", + help="A file with a list of sites (by default sites.txt )" + ) + parser.add_argument( + "--max_concurrent", "-m", + type=int, + default=50, + help="Maximum number of simultaneous tasks (50 by default)" + ) + parser.add_argument( + "--log_level", "-l", + type=str, + default="INFO", + choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"], + help="Logging level: DEBUG, INFO, WARNING, ERROR, CRITICAL (INFO by default)" + ) + parser.add_argument( + "--no_console_log", "-ncl", + action="store_true", + help="Disable console logging (by default console logging is enabled)" + ) + parser.add_argument( + "--name_folder_log", "-n", + type=str, + default="logs", + help="A name of folder logs(by default logs)" + ) + + args = parser.parse_args() + + if args.quantity <= 0: + print("Error: the number of requests must be a positive number, using the default value of 10") + args.quantity = 10 + if args.rps <= 0: + print("Error: RPS must be a positive number, using the default value of 15") + args.rps = 15 + if args.timeout <= 0: + print("Error: the timeout must be a positive number, using the default value of 20 - the optimal time for analyzing a compound is at standard values.") + args.timeout = 20 + if args.max_concurrent <= 0: + print("Error: the number of tasks being completed at the same time must be a positive number, using the default value of 50") + args.max_concurrent = 50 + + try: + with open(args.file, 'r') as f: + sites = [line.strip() for line in f if line.strip()] + except FileNotFoundError: + print(f"Error: file '{args.file}' not found") + sys.exit(1) + + if not sites: + print(f"Error: file '{args.file}' is empty") + sys.exit(1) + + return args.quantity, sites, args.rps, args.timeout, args.max_concurrent, args.log_level, args.no_console_log, args.name_folder_log + +async def check_one(site, timeout): + scan_timeout = max(2, timeout - 2) + + result = { + 'site': site, + 'tcp_ports': {}, + 'udp_ports': {}, + 'status': 'unknown', + 'ip': None + } + + try: + nm = nmap.PortScanner() + + await asyncio.wait_for( + asyncio.to_thread( + nm.scan, + site, + '53,80,443,123,161', + f'-sS -sU -T4 --host-timeout {scan_timeout}s' + ), + timeout + ) + + if nm.all_hosts(): + result['ip'] = nm.all_hosts()[0] + + for proto in nm[result['ip']].all_protocols(): + for port in nm[result['ip']][proto].keys(): + state = nm[result['ip']][proto][port]['state'] + service = nm[result['ip']][proto][port].get('name', 'unknown') + + if state == 'open': + if proto == 'tcp': + + result['tcp_ports'][port] = { + 'state': state, + 'service': service + } + + if port == 80 or port == 443: + protocol = 'https' if port == 443 else 'http' + url = f"{protocol}://{site}:{port}" + response = requests.get(url, timeout=5, verify=False) + status_code = response.status_code + soup = BeautifulSoup(response.text, 'html.parser') + title = soup.find('title').text if soup.find('title') else None + + result['tcp_ports'][port]['http_status'] = status_code + result['tcp_ports'][port]['title'] = title + + + elif proto == 'udp': + result['udp_ports'][port] = { + 'state': state, + 'service': service + } + + if result['tcp_ports'] or result['udp_ports']: + result['status'] = 'success' + else: + result['status'] = 'no_open_ports' + + except asyncio.TimeoutError: + result['status'] = 'timeout' + + except Exception as e: + result['status'] = 'error' + result['error'] = str(e) + + + + + return result + + +def setup_logger(flag_stream_handler, input_level_logging, name_folder_log): + log_dir = Path(name_folder_log) + log_dir.mkdir(exist_ok=True) + file_log = log_dir / f"LOG_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S-%f')}.json" + + + logger = logging.getLogger(__name__) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + + logger.setLevel(input_level_logging) + + + + if not flag_stream_handler: + console_handler = logging.StreamHandler() + console_handler.setLevel(input_level_logging) + console_handler.setFormatter(formatter) + logger.addHandler(console_handler) + + + return logger, file_log + +def log(quantity, rps, timeout, max_concurrent, results, logger, file_log): + + logger.info(f" Requests: {quantity}") + logger.info(f" RPS: {rps}") + logger.info(f" Timeout: {timeout}с") + logger.info(f" Max concurrent processes: {max_concurrent}") + logger.debug(f" The results file: {file_log}") + + no_ports_count = success_count = timeout_count = error_count = 0 + + log_data = { + "parameters": { + "quantity": quantity, + "rps": rps, + "timeout": timeout, + "max_concurrent": max_concurrent + }, + "results": [] + } + + for res in results: + if isinstance(res, Exception): + log_data["results"].append({ + "site": "unknown", + "status": "exception", + "details": str(res) + }) + logger.exception(f"Request - EXCEPTION: {res}") + + else: + site_data = res + + log_entry = { + "site": site_data['site'], + "ip": site_data['ip'], + "status": site_data['status'], + "tcp_ports": site_data['tcp_ports'], + "udp_ports": site_data['udp_ports'] + } + + if 'error' in site_data: + log_entry["error"] = site_data['error'] + + log_data["results"].append(log_entry) + + if site_data['status'] == 'success': + success_count += 1 + logger.debug(f"Request to {site_data['site']} - SUCCESS") + elif site_data['status'] == 'timeout': + timeout_count += 1 + logger.error(f"Request to {site_data['site']} - TIMEOUT") + elif site_data['status'] == 'no_open_ports': + no_ports_count += 1 + logger.warning(f"Request to {site_data['site']} - NO OPEN PORTS") + else: + error_count += 1 + logger.error(f"Request to {site_data['site']} - ERROR") + + + log_data["statistics"] = { + "no_ports_count" : no_ports_count, + "success": success_count, + "timeout": timeout_count, + "error": error_count, + "total": quantity + } + + try: + with open(file_log, 'w', encoding='utf-8') as file: + json.dump(log_data, file, indent=2, ensure_ascii=False) + print(f"\nLogs are saved to a file: {file_log}") + except Exception as e: + print(f"\nError saving logs: {e}") + + +async def main(): + quantity, sites, rps, timeout, max_concurrent, log_level, console_log, name_folder_log = pars() + + semaphore = asyncio.Semaphore(max_concurrent) + delay = 1.0 / rps + + tasks = [] + for _ in range(quantity): + site = random.choice(sites) + async def task_wrapper(certain_site=site): + async with semaphore: + return await check_one(certain_site, timeout) + + task = asyncio.create_task(task_wrapper()) + tasks.append(task) + + await asyncio.sleep(delay) + + results = await asyncio.gather(*tasks, return_exceptions=True) + + logger, file_log = setup_logger(console_log, log_level, name_folder_log) + log(quantity, rps, timeout, max_concurrent, results, logger, file_log) + + +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/scripts/gen_traf_tcp-udp/generate_traf.sh b/scripts/gen_traf_tcp-udp/generate_traf.sh new file mode 100755 index 0000000..785691a --- /dev/null +++ b/scripts/gen_traf_tcp-udp/generate_traf.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +count=$1 +site=$2 + +if [ -z "$count" ] || [ -z "$site" ]; then + echo "Error: the argument was not passed!" + echo "Usage: ./generate_traf.sh " + echo "Example: ./generate_traf.sh 8 google.com" + exit 1 +fi + +if ! [[ "$count" =~ ^[1-9][0-9]*$ ]]; then + echo "Error: '$count' is not a positive number!" + echo "Usage: ./generate_traf.sh " + echo "Example: ./generate_traf.sh 8 google.com" + exit 1 +fi + +for (( i=1; i<=$count;i++ )); do + nc -zv $site 80 + nc -uzv $site 80 +done diff --git a/scripts/gen_traf_tcp-udp/setup.sh b/scripts/gen_traf_tcp-udp/setup.sh new file mode 100755 index 0000000..2062ad5 --- /dev/null +++ b/scripts/gen_traf_tcp-udp/setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -e +sudo apt update +sudo apt install -y nmap +sudo apt install -y python3-pip +pip3 install python-nmap diff --git a/scripts/gen_traf_tcp-udp/sites.txt b/scripts/gen_traf_tcp-udp/sites.txt new file mode 100644 index 0000000..dd5b93d --- /dev/null +++ b/scripts/gen_traf_tcp-udp/sites.txt @@ -0,0 +1,10 @@ +4chan.org +www.reddit.com +www.yahoo.com +www.cnn.com +www.ebay.com +wikipedia.org +youtube.com +github.com +medium.com +thepiratebay.org \ No newline at end of file diff --git a/scripts/gen_traf_tcp-udp/tests.py b/scripts/gen_traf_tcp-udp/tests.py new file mode 100644 index 0000000..883cf2d --- /dev/null +++ b/scripts/gen_traf_tcp-udp/tests.py @@ -0,0 +1,135 @@ +import unittest +import asyncio +import json +import time +import tempfile +from pathlib import Path +from unittest.mock import patch +from generate_rand_traf import main +import urllib3 + +class TestIntegration(unittest.TestCase): + + def setUp(self): + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + self.temp_dir = tempfile.TemporaryDirectory() + self.sites_file = Path(self.temp_dir.name) / "sites.txt" + + self.logs_dir = Path(f"logs/test") + self.logs_dir.mkdir(exist_ok=True) + + def tearDown(self): + self.temp_dir.cleanup() + + def create_sites_file(self, sites): + with open(self.sites_file, 'w') as f: + for site in sites: + f.write(f"{site}\n") + + def get_last_log_file(self): + log_files = list(self.logs_dir.glob("LOG_*.json")) + if not log_files: + return None + return max(log_files, key=lambda p: p.stat().st_mtime) + + def run_main(self, args): + with patch('sys.argv', ['script.py'] + args + ['-f', str(self.sites_file)] + ['-n', "logs/test"] + ['-ncl'] + ['-l' "CRITICAL"]): + asyncio.run(main()) + return self.get_last_log_file() + + + + + def test_good_sites(self): + good_sites = [ + "cloudflare.com", + "4chan.org", + "www.reddit.com", + "wikipedia.org", + "github.com", + ] + + self.create_sites_file(good_sites) + log_file = self.run_main(['-q', '20', '-t', '90', '-r', '50']) + self.assertIsNotNone(log_file) + + with open(log_file, 'r') as f: + data = json.load(f) + + self.assertEqual(len(data['results']), 20) + self.assertEqual(data['statistics']['total'], 20) + self.assertTrue(data['statistics']['success'] > 12) #this condition is enough for us to confirm the success of the test. Packets can be ignored with a large number of simultaneous scanners. + + for result in data['results']: + if result['status'] == "success": + self.assertIsNotNone(result['ip']) + + + def test_bad_sites(self): + bad_sites = [ + "this-site-does-not-exist-12345.xyz", + "nonexistent.domain.test", + "project.local", + "my.project.local", + "test-project.local", + "nonexistent.invalid.test", + "nonexistent.domain.invalid", + "invalid.invalid.invalid", + ] + self.create_sites_file(bad_sites) + + log_file = self.run_main(['-q', '20',]) + self.assertIsNotNone(log_file) + + with open(log_file, 'r') as f: + data = json.load(f) + + self.assertEqual(len(data['results']), 20) + + self.assertEqual(data['statistics']['total'], 20) + self.assertEqual(data['statistics']['success'], 0) + self.assertEqual(data['statistics']['no_ports_count'] + data['statistics']['timeout'] + data['statistics']['error'], 20) + + for result in data['results']: + self.assertIsNone(result['ip']) + self.assertIn(result['status'], ['timeout', 'error', 'no_open_ports']) + + def test_mixed_sites(self): + mixed_sites = [ + "google.com", + "github.com", + "www.yahoo.com", + "www.cnn.com", + "www.ebay.com", + "this-site-does-not-exist-12345.xyz", + "nonexistent.domain.test", + "project.local", + "my.project.local" + ] + + self.create_sites_file(mixed_sites) + + log_file = self.run_main(['-q', '20', '-t', '80', '-r', '41']) + self.assertIsNotNone(log_file) + + with open(log_file, 'r') as f: + data = json.load(f) + + self.assertEqual(data['statistics']['total'], 20) + + self.assertGreater(data['statistics']['success'], 0) + self.assertGreater(data['statistics']['timeout'] + data['statistics']['error'] + data['statistics']['no_ports_count'], 0) + + self.assertEqual(data['statistics']['success'] + data['statistics']['timeout'] + + data['statistics']['error'] + data['statistics']['no_ports_count'], data['statistics']['total']) + + for result in data['results']: + if result['status'] == "success": + self.assertIsNotNone(result['ip']) + else: + self.assertIsNone(result['ip']) + + +if __name__ == '__main__': + unittest.main(verbosity=1) \ No newline at end of file