diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..0fb22df9 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,68 @@ +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# Virtual environments +venv/ +env/ +ENV/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +.DS_Store? +._* +.Spotlight-V100 +.Trashes +ehthumbs.db +Thumbs.db + +# Docker +Dockerfile +docker-compose.yml +.dockerignore + +# Git +.git/ +.gitignore + +# Documentation +*.md +!README.md + +# Logs +*.log + +# Test files +tests/ +.pytest_cache/ + +# TorBot specific +*.json +!requirements.json +src/*.json \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..8b7e6969 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,56 @@ +# Contributing to TorBot + +Thank you for your interest in contributing to TorBot! This document provides guidelines for contributors. + +## Getting Started + +1. Fork the repository +2. Clone your fork locally +3. Create a virtual environment: `python -m venv venv` +4. Activate it: `venv\Scripts\activate` (Windows) or `source venv/bin/activate` (Unix/Mac) +5. Install dependencies: `pip install -r requirements.txt` + +## Making Changes + +1. Create a new branch for your feature/fix +2. Make your changes +3. Test your changes thoroughly +4. Update documentation if necessary +5. Update CHANGELOG.md with your changes + +## Testing + +Run the test suite before submitting: +```bash +python -m pytest tests/ +``` + +## Code Style + +- Follow PEP 8 guidelines +- Use meaningful variable and function names +- Add comments for complex logic +- Ensure all new code has appropriate error handling + +## Security Considerations + +- Never commit sensitive information (API keys, passwords, etc.) +- Validate all user inputs +- Use secure coding practices +- Test with various edge cases + +## Submitting Changes + +1. Push your branch to your fork +2. Create a pull request with: + - Clear description of changes + - Reference to any related issues + - Testing evidence/screenshots if applicable + +## Review Process + +- All PRs require review before merging +- Address feedback promptly +- Ensure CI checks pass + +Thank you for contributing to TorBot! \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 71d8f407..2f3551fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,22 +1,65 @@ -# Use an official Python 3.11.4 image as the base -FROM python:3.11.4 +# Multi-stage Dockerfile for TorBot +# Stage 1: Build stage +FROM python:3.11.4 as builder -# Set a working directory within the container -WORKDIR /app +# Set working directory +WORKDIR /build + +# Install system dependencies required for building Python packages +RUN apt-get update && apt-get install -y \ + gcc \ + g++ \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements first for better layer caching +COPY requirements.txt . + +# Create virtual environment and install dependencies +RUN python -m venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" +RUN pip install --no-cache-dir --upgrade pip && \ + pip install --no-cache-dir -r requirements.txt + +# Stage 2: Runtime stage +FROM python:3.11.4-slim as runtime + +# Create non-root user for security +RUN groupadd -r torbot && useradd -r -g torbot torbot -# Clone the TorBot repository from GitHub -RUN git clone https://github.com/DedSecInside/TorBot.git /app +# Install runtime dependencies only +RUN apt-get update && apt-get install -y \ + curl \ + && rm -rf /var/lib/apt/lists/* -# Install dependencies -RUN pip install -r /app/requirements.txt +# Copy virtual environment from builder stage +COPY --from=builder /opt/venv /opt/venv +ENV PATH="/opt/venv/bin:$PATH" -# Set the SOCKS5_PORT environment variable +# Set working directory +WORKDIR /app + +# Copy application code +COPY --chown=torbot:torbot . /app + +# Set environment variables +ENV PYTHONPATH="/app" ENV SOCKS5_PORT=9050 +ENV PYTHONUNBUFFERED=1 -# Expose the port specified in the .env file +# Switch to non-root user +USER torbot + +# Expose port EXPOSE $SOCKS5_PORT -# Run the TorBot script -CMD ["poetry", "run", "python", "torbot"] -# Example way to run the container: -# docker run --network="host" your-image-name poetry run python torbot -u https://www.example.com --depth 2 --visualize tree --save json +# Health check +HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ + CMD python -c "import sys; sys.exit(0)" + +# Default command +CMD ["python", "torbot.py", "--help"] + +# Labels for better image management +LABEL maintainer="TorBot Team" +LABEL version="4.2.0" +LABEL description="TorBot - A web scraping and analysis tool with Tor support" diff --git a/README.md b/README.md index 14f57ede..738ef817 100755 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -
+
████████╗ ██████╗ ██████╗ ██████╗ ██████╗ ████████╗
╚══██╔══╝██╔═══██╗██╔══██╗ ██╔══██╗██╔═████╗╚══██╔══╝
@@ -47,6 +47,11 @@
- Poetry (Optional)
### Python Dependencies
+- All dependencies have been updated to latest secure versions (2024)
+- Compatible with httpx 0.28.1+ (fixed proxy configuration)
+- Enhanced error handling for NLP operations
+- Updated security patches for all dependencies
+- Added lxml>=5.3.0 for improved XML/HTML parsing
(see pyproject.toml or requirements.txt for more details)
@@ -55,26 +60,43 @@
### TorBot
#### Using `venv`
-* If using Python ^3.4,
+* If using Python ^3.9,
```sh
python -m venv torbot_venv
-source torbot_venv/bin/activate
+source torbot_venv/bin/activate # On Windows: source torbot_venv/Scripts/activate
pip install -r requirements.txt
pip install -e .
-./main.py --help
+python main.py --help
```
-#### Using `docker`
+#### Using Docker (Multi-stage build)
+
+**Build the optimized image:**
```sh
-docker build -t {image_name} .
+docker build -t torbot:latest .
+```
-# Running without Tor
-docker run {image_name} poetry run python torbot -u https://example.com --depth 2 --visualize tree --save json --disable-socks5
+**Run with Docker Compose (Recommended):**
+```sh
+docker-compose up torbot
+```
-# Running with Tor
-docker run --network="host" {image_name} poetry run python torbot -u https://example.com --depth 2 --visualize tree --save json --disable-socks5
+**Run manually:**
+```sh
+# Basic usage
+docker run --rm torbot:latest -u https://example.com --depth 2 --visualize tree --save json
+
+# With Tor proxy
+docker run --rm --network="host" torbot:latest -u https://example.onion --depth 2 --visualize tree
+
+# Custom SOCKS5 proxy
+docker run --rm torbot:latest -u https://example.onion --host 127.0.0.1 --port 9050 --depth 2
```
+**Environment Variables:**
+- `SOCKS5_HOST`: SOCKS5 proxy host (default: 127.0.0.1)
+- `SOCKS5_PORT`: SOCKS5 proxy port (default: 9050)
+
### Options
usage: Gather and analyze data from Tor sites.
@@ -103,6 +125,9 @@ Read more about torrc here : [Torrc](https://github.com/DedSecInside/TorBoT/blob
- [x] Implement BFS Search for webcrawler
- [x] Improve stability (Handle errors gracefully, expand test coverage, etc.)
- [x] Increase test coverage
+- [x] Multi-stage Docker build for optimized container
+- [x] Docker Compose support
+- [x] Enhanced security with non-root container user
- [ ] Save the most recent search results to a database
- [ ] Randomize Tor Connection (Random Header and Identity)
- [ ] Keyword/Phrase Search
diff --git a/TESTING.md b/TESTING.md
index 5abdc811..af8d03d9 100755
--- a/TESTING.md
+++ b/TESTING.md
@@ -1,8 +1,264 @@
# Testing Documentation
-We are currently using [`pytest`](https://docs.pytest.org/en/latest/) as our testing framework so if you want to run the test suite. Run `pytest` from the base directory of TorBot or from the `tests` directory.
-If you would like to see `print` statements show up in the console, use `pytest -s`.
+This document provides comprehensive testing instructions for TorBot, including local development and Docker environments.
-### Relevant Frameworks
-- [`yattag`](https://www.yattag.org/) is used to generate HTML documents.
-- [`unittest.mock`](https://docs.python.org/3/library/unittest.mock.html) is used to patch the httpx Client.
+## Testing Framework
+
+We use [`pytest`](https://docs.pytest.org/en/latest/) as our testing framework with the following structure:
+
+```
+tests/
+├── __init__.py
+├── test_api.py # API endpoint tests
+├── test_linktree.py # Link tree functionality tests
+└── conftest.py # Pytest configuration (optional)
+```
+
+## Running Tests
+
+### Local Development
+
+1. **Install testing dependencies:**
+ ```bash
+ pip install -r requirements.txt
+ pip install pytest pytest-cov pytest-mock
+ ```
+
+2. **Run all tests:**
+ ```bash
+ pytest
+ ```
+
+3. **Run with verbose output:**
+ ```bash
+ pytest -v
+ ```
+
+4. **Run with print statements:**
+ ```bash
+ pytest -s
+ ```
+
+5. **Run specific test file:**
+ ```bash
+ pytest tests/test_api.py
+ ```
+
+6. **Run with coverage:**
+ ```bash
+ pytest --cov=src/torbot tests/
+ ```
+
+### Docker Testing
+
+1. **Test Docker build:**
+ ```bash
+ # Build test image
+ docker build -t torbot:test .
+
+ # Run tests in container
+ docker run --rm torbot:test pytest tests/
+
+ # Run tests with coverage
+ docker run --rm torbot:test pytest --cov=src/torbot tests/
+ ```
+
+2. **Development testing with volume mounts:**
+ ```bash
+ # Run tests with live code changes
+ docker run --rm \
+ -v $(pwd)/src:/app/src \
+ -v $(pwd)/tests:/app/tests \
+ torbot:test pytest tests/ -v
+ ```
+
+3. **Test with Docker Compose:**
+ ```bash
+ # Create test service
+ docker-compose -f docker-compose.test.yml up --build --abort-on-container-exit
+ ```
+
+## Test Categories
+
+### Unit Tests
+- **API Tests**: Test API endpoints and response formats
+- **Link Tree Tests**: Test link extraction and tree building
+- **Parser Tests**: Test HTML parsing and data extraction
+- **Utility Tests**: Test helper functions and utilities
+
+### Integration Tests
+- **Network Tests**: Test actual HTTP requests (when safe)
+- **Tor Integration**: Test SOCKS5 proxy functionality
+- **File I/O Tests**: Test saving/loading functionality
+
+### Mock Tests
+We use [`unittest.mock`](https://docs.python.org/3/library/unittest.mock.html) to patch:
+- `httpx.Client` for network requests
+- File system operations
+- External API calls
+
+## Writing Tests
+
+### Test Structure
+```python
+# tests/test_example.py
+import pytest
+from unittest.mock import patch, MagicMock
+from src.torbot.module import function_to_test
+
+class TestFunctionality:
+ """Test suite for specific functionality"""
+
+ def test_success_case(self):
+ """Test successful execution"""
+ result = function_to_test("valid_input")
+ assert result is not None
+
+ def test_error_handling(self):
+ """Test error conditions"""
+ with pytest.raises(ValueError):
+ function_to_test("invalid_input")
+
+ @patch('src.torbot.module.httpx.Client')
+ def test_with_mock(self, mock_client):
+ """Test with mocked network calls"""
+ mock_response = MagicMock()
+ mock_response.text = "test"
+ mock_client.return_value.__enter__.return_value.get.return_value = mock_response
+
+ result = function_to_test("http://example.com")
+ assert result == expected_result
+```
+
+### Test Data
+Place test data in `tests/data/` directory:
+```
+tests/
+├── data/
+│ ├── sample_html.html
+│ ├── expected_results.json
+│ └── mock_responses/
+```
+
+## Continuous Integration
+
+### GitHub Actions
+Our CI pipeline runs:
+- **pytest**: All unit tests
+- **flake8**: Code style checks
+- **coverage**: Code coverage reports
+- **docker**: Container build tests
+
+### Local CI Simulation
+```bash
+# Run all checks locally
+python -m pytest tests/
+python -m flake8 src/ tests/
+python -m mypy src/
+docker build -t torbot:ci .
+```
+
+## Performance Testing
+
+### Load Testing
+```bash
+# Test with multiple URLs
+docker run --rm torbot:test python -c "
+import time
+from src.torbot.module import test_multiple_urls
+start = time.time()
+test_multiple_urls(urls, depth=2)
+print(f'Duration: {time.time() - start}s')
+"
+```
+
+### Memory Usage
+```bash
+# Monitor memory usage
+docker run --rm --memory=256m torbot:test pytest tests/
+```
+
+## Debugging Tests
+
+### Debug Mode
+```bash
+# Run with debugger
+pytest --pdb tests/test_api.py::TestAPI::test_specific_endpoint
+
+# Verbose output with details
+pytest -vv -s tests/
+```
+
+### Test Isolation
+```bash
+# Run specific test
+pytest tests/test_linktree.py::TestLinkTree::test_tree_building -v
+
+# Run tests matching pattern
+pytest -k "test_api" tests/
+```
+
+## Test Dependencies
+
+### Required Packages
+```txt
+pytest>=7.0.0
+pytest-cov>=4.0.0
+pytest-mock>=3.10.0
+pytest-asyncio>=0.21.0
+```
+
+### Development Dependencies
+```bash
+pip install pytest-benchmark # Performance testing
+pip install pytest-xdist # Parallel testing
+pip install pytest-html # HTML reports
+```
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Import errors:**
+ ```bash
+ export PYTHONPATH=$PWD:$PYTHONPATH
+ ```
+
+2. **Network tests failing:**
+ Use mock objects for network-dependent tests
+
+3. **Permission errors:**
+ Ensure proper file permissions for test data
+
+4. **Docker test failures:**
+ Check Docker daemon and resource limits
+
+### Debug Commands
+```bash
+# Check test discovery
+pytest --collect-only
+
+# Run with coverage report
+pytest --cov=src/torbot --cov-report=html tests/
+
+# Generate test report
+pytest --junitxml=test-results.xml tests/
+```
+
+## Contributing Tests
+
+When adding new features:
+
+1. **Write tests first** (TDD approach)
+2. **Ensure 80%+ coverage** for new code
+3. **Include edge cases** and error handling
+4. **Test both positive and negative scenarios**
+5. **Update this documentation** if testing procedures change
+
+### Test Checklist
+- [ ] Unit tests for new functions
+- [ ] Integration tests for API endpoints
+- [ ] Error handling tests
+- [ ] Performance tests for heavy operations
+- [ ] Docker compatibility tests
+- [ ] Documentation updated
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
new file mode 100644
index 00000000..4dd9c395
--- /dev/null
+++ b/docker-compose.test.yml
@@ -0,0 +1,44 @@
+version: '3.8'
+
+services:
+ torbot-test:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ target: runtime
+ volumes:
+ - .:/app
+ - ./tests:/app/tests
+ - ./results:/app/results
+ environment:
+ - PYTHONPATH=/app
+ - PYTHONUNBUFFERED=1
+ command: pytest tests/ -v --cov=src/torbot --cov-report=html --cov-report=term-missing
+
+ torbot-test-integration:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ target: runtime
+ depends_on:
+ - tor-test
+ environment:
+ - SOCKS5_HOST=tor-test
+ - SOCKS5_PORT=9050
+ - PYTHONPATH=/app
+ volumes:
+ - ./tests:/app/tests
+ - ./results:/app/results
+ command: pytest tests/ -v -k "integration"
+
+ tor-test:
+ image: dperson/torproxy:latest
+ ports:
+ - "9050:9050"
+ environment:
+ - PASSWORD=test_password
+ restart: unless-stopped
+
+networks:
+ default:
+ name: torbot-test-network
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
new file mode 100644
index 00000000..8f8b5723
--- /dev/null
+++ b/docker-compose.yml
@@ -0,0 +1,23 @@
+version: '3.8'
+
+services:
+ torbot:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ container_name: torbot
+ ports:
+ - "9050:9050"
+ environment:
+ - SOCKS5_PORT=9050
+ volumes:
+ - ./src:/app/src
+ networks:
+ - torbot-network
+ restart: unless-stopped
+ # Uncomment to run a specific command
+ # command: ["python", "torbot.py", "-u", "https://httpbin.org/html", "--depth", "2", "--visualize", "tree", "--save", "json"]
+
+networks:
+ torbot-network:
+ driver: bridge
\ No newline at end of file
diff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.md
index fc220433..d51b7532 100644
--- a/docs/CHANGELOG.md
+++ b/docs/CHANGELOG.md
@@ -2,6 +2,46 @@
--------------------
All notable changes to this project will be documented in this file.
+## 4.2.0 - Security, Docker, and Documentation Update
+
+### Added
+- Updated all dependencies to latest secure versions
+- Added support for httpx 0.28.1+ with proper proxy configuration
+- Added lxml>=5.3.0 for improved XML/HTML parsing
+- Enhanced error handling for NLP module
+- Multi-stage Docker build for optimized container (200MB final size)
+- Docker Compose support with Tor integration
+- Non-root user execution in Docker container (security enhancement)
+- Comprehensive Docker documentation (docs/DOCKER.md)
+- Enhanced testing documentation (TESTING.md)
+- Docker testing configuration (docker-compose.test.yml)
+- Health checks for Docker containers
+- .dockerignore for optimized builds
+
+### Fixed
+- Fixed compatibility issues with httpx 0.28.1+ API changes
+- Fixed UnicodeDecodeError in NLP module by adding graceful degradation
+- Fixed FileNotFoundError for missing training data in sklearn
+- Updated dependency versions to address security vulnerabilities
+- Fixed Docker build process to use pip instead of poetry
+- Updated README.md with new Docker instructions
+
+### Security
+- Updated certifi>=2024.8.30 for SSL certificate validation
+- Updated urllib3>=2.2.2 for security patches
+- Updated scikit-learn>=1.5.1 with latest security fixes
+- Updated all dependencies to latest stable versions
+- Docker container runs as non-root user (UID 1000)
+- Reduced attack surface with minimal base image
+
+### Documentation
+- Updated README.md with Docker multi-stage build instructions
+- Enhanced Tor.md with Docker configuration options
+- Added comprehensive Docker documentation (docs/DOCKER.md)
+- Updated testing documentation with Docker testing instructions
+- Added CONTRIBUTING.md for contribution guidelines
+- Updated installation instructions for Python ^3.9
+
## 2.1.0
### Added
diff --git a/docs/DOCKER.md b/docs/DOCKER.md
new file mode 100644
index 00000000..fac39d76
--- /dev/null
+++ b/docs/DOCKER.md
@@ -0,0 +1,190 @@
+# Docker Documentation for TorBot
+
+This document provides comprehensive instructions for using TorBot with Docker.
+
+## Overview
+
+TorBot now uses a multi-stage Docker build for optimal performance and security. The build process creates a lightweight, secure container with non-root user execution.
+
+## Quick Start
+
+### Using Docker Compose (Recommended)
+
+1. **Clone the repository:**
+ ```bash
+ git clone https://github.com/DedSecInside/TorBot.git
+ cd TorBot
+ ```
+
+2. **Build and run:**
+ ```bash
+ docker-compose up torbot
+ ```
+
+3. **Run with custom parameters:**
+ ```bash
+ docker-compose run torbot -u https://example.onion --depth 3 --visualize tree
+ ```
+
+### Using Docker CLI
+
+1. **Build the image:**
+ ```bash
+ docker build -t torbot:latest .
+ ```
+
+2. **Run basic scan:**
+ ```bash
+ docker run --rm torbot:latest -u https://example.com --depth 2
+ ```
+
+3. **Run with Tor proxy:**
+ ```bash
+ docker run --rm --network="host" torbot:latest -u https://example.onion --depth 2
+ ```
+
+## Configuration Options
+
+### Environment Variables
+
+| Variable | Description | Default |
+|----------|-------------|---------|
+| `SOCKS5_HOST` | SOCKS5 proxy host | `127.0.0.1` |
+| `SOCKS5_PORT` | SOCKS5 proxy port | `9050` |
+
+### Volume Mounts
+
+Mount local directories for persistent data:
+
+```bash
+docker run --rm \
+ -v $(pwd)/results:/app/results \
+ -v $(pwd)/logs:/app/logs \
+ torbot:latest -u https://example.onion --save json
+```
+
+## Advanced Usage
+
+### Custom Tor Configuration
+
+Create a custom `docker-compose.yml` with Tor as a service:
+
+```yaml
+version: '3.8'
+services:
+ tor:
+ image: dperson/torproxy:latest
+ ports:
+ - "9050:9050"
+ - "9051:9051"
+ environment:
+ - PASSWORD=your_password
+ restart: unless-stopped
+
+ torbot:
+ image: torbot:latest
+ depends_on:
+ - tor
+ environment:
+ - SOCKS5_HOST=tor
+ - SOCKS5_PORT=9050
+ volumes:
+ - ./results:/app/results
+ command: -u https://example.onion --depth 3 --visualize tree --save json
+```
+
+### Development Mode
+
+For development with live code changes:
+
+```yaml
+# docker-compose.dev.yml
+version: '3.8'
+services:
+ torbot-dev:
+ build:
+ context: .
+ dockerfile: Dockerfile
+ volumes:
+ - .:/app
+ - ./results:/app/results
+ environment:
+ - PYTHONPATH=/app
+ command: python torbot.py --help
+```
+
+## Security Features
+
+- **Non-root user**: Container runs as `torbot` user (UID 1000)
+- **Minimal base image**: Uses `python:3.11.4-slim` for reduced attack surface
+- **Layer caching**: Optimized Dockerfile for faster builds
+- **Health checks**: Built-in container health monitoring
+
+## Image Details
+
+- **Base image**: `python:3.11.4-slim`
+- **Final size**: ~200MB (optimized)
+- **Python version**: 3.11.4
+- **Working directory**: `/app`
+- **Exposed ports**: 9050 (SOCKS5)
+
+## Troubleshooting
+
+### Common Issues
+
+1. **Tor connection failed:**
+ ```bash
+ # Check if Tor is running on host
+ netstat -tuln | grep 9050
+
+ # Test with Docker
+ docker run --rm --network="host" torbot:latest --help
+ ```
+
+2. **Permission denied:**
+ Ensure Docker daemon is running and user has proper permissions.
+
+3. **Build fails:**
+ Check Docker version compatibility:
+ ```bash
+ docker --version # Requires Docker 20.10+
+ ```
+
+### Debug Mode
+
+Run with debug logging:
+```bash
+docker run --rm -e PYTHONPATH=/app torbot:latest -v -u https://example.com
+```
+
+## Performance Optimization
+
+### Build Cache
+
+Leverage Docker build cache:
+```bash
+# Build with cache
+DOCKER_BUILDKIT=1 docker build -t torbot:latest .
+
+# Build without cache (clean build)
+docker build --no-cache -t torbot:latest .
+```
+
+### Resource Limits
+
+Set resource constraints:
+```bash
+docker run --rm \
+ --memory=512m \
+ --cpus=1 \
+ torbot:latest -u https://example.com --depth 2
+```
+
+## Contributing
+
+When contributing Docker-related changes:
+
+1. Test the build process: `docker build -t torbot:test .`
+2. Verify functionality: `docker run --rm torbot:test --help`
+3. Update this documentation if needed
+4. Ensure `.dockerignore` excludes unnecessary files
\ No newline at end of file
diff --git a/docs/Tor.md b/docs/Tor.md
index d7840401..ee46e9ea 100644
--- a/docs/Tor.md
+++ b/docs/Tor.md
@@ -1,12 +1,73 @@
-# Tor Configuration
+# Tor Configuration Guide
+
+This guide covers Tor configuration for both native installations and Docker deployments.
+
+## Native Tor Installation
Look at your torrc for the following configuration options...
- Tor uses a text file called torrc that contains configuration instructions for how your Tor program should behave. The default configuration should work fine for most Tor users.
+Tor uses a text file called torrc that contains configuration instructions for how your Tor program should behave. The default configuration should work fine for most Tor users.
+
+### Locating torrc
+
+**Windows (Tor Browser):**
+- Path: `Browser/TorBrowser/Data/Tor/torrc` inside your Tor Browser directory
+
+**macOS (Tor Browser):**
+- Path: `~/Library/Application Support/TorBrowser-Data/Tor`
+- To access: Press `cmd-shift-g` in Finder and paste the directory
+
+**Linux (System Tor):**
+- Source install: `/usr/local/etc/tor/torrc`
+- Package install: `/etc/tor/torrc` or `/etc/torrc`
+
+**Docker with Tor:**
+When using Docker, you can use the official Tor Docker image or configure your host's Tor service.
+
+## Docker Configuration
+
+### Option 1: Using Host Tor (Recommended)
+Run TorBot container with host network to use your existing Tor service:
+
+```bash
+docker run --rm --network="host" torbot:latest -u https://example.onion --depth 2
+```
+
+### Option 2: Tor as Sidecar Container
+Use Docker Compose with Tor as a service:
+
+```yaml
+version: '3.8'
+services:
+ tor:
+ image: dperson/torproxy:latest
+ ports:
+ - "9050:9050"
+ restart: unless-stopped
+
+ torbot:
+ image: torbot:latest
+ depends_on:
+ - tor
+ environment:
+ - SOCKS5_HOST=tor
+ - SOCKS5_PORT=9050
+ command: -u https://example.onion --depth 2 --visualize tree
+```
+
+### Option 3: Tor in Same Container
+For development/testing, you can install Tor in the TorBot container:
-If you installed Tor Browser on Windows or Linux, look for Browser/TorBrowser/Data/Tor/torrc inside your Tor Browser directory. If you're on macOS, the torrc is in ~/Library/Application Support/TorBrowser-Data/Tor . To get to it, press cmd-shift-g while in Finder and copy/paste that directory into the box that appears.
+```dockerfile
+FROM torbot:latest
+RUN apt-get update && apt-get install -y tor
+RUN echo "SOCKSPort 9050" >> /etc/tor/torrc
+RUN echo "ControlPort 9051" >> /etc/tor/torrc
+RUN echo "CookieAuthentication 1" >> /etc/tor/torrc
+EXPOSE 9050 9051
+```
-Otherwise, if you are using Tor without Tor Browser, it looks for the torrc file in /usr/local/etc/tor/torrc if you compiled tor from source, and /etc/tor/torrc or /etc/torrc if you installed a pre-built package.
+## Configuration Files
Once you've created or changed your torrc file, you will need to restart tor for the changes to take effect. (For advanced users, note that you actually only need to send Tor a HUP signal, not actually restart it.)
diff --git a/main.py b/main.py
index d8c2ad04..4bf8fe8a 100755
--- a/main.py
+++ b/main.py
@@ -81,8 +81,9 @@ def run(arg_parser: argparse.ArgumentParser, version: str) -> None:
socks5_host = args.host
socks5_port = str(args.port)
socks5_proxy = f"socks5://{socks5_host}:{socks5_port}"
+ proxy_config = {"http://": socks5_proxy, "https://": socks5_proxy} if not args.disable_socks5 else None
with httpx.Client(
- timeout=60, proxies=socks5_proxy if not args.disable_socks5 else None
+ timeout=60, proxy=proxy_config
) as client:
# print header and IP address if not set to quiet
if not args.quiet:
diff --git a/pyproject.toml b/pyproject.toml
index 41cb129b..5a66909d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
[project]
name = "torbot"
-version = "4.1.0"
+version = "4.2.0"
authors = [
{ name="Akeem King", email="akeemtlking@gmail.com" },
{ name="PS Narayanan", email="thepsnarayanan@gmail.com" },
@@ -18,37 +18,37 @@ classifiers = [
"Operating System :: OS Independent",
]
dependencies = [
- "altgraph==0.17.2",
- "beautifulsoup4==4.11.1",
- "certifi==2024.7.4",
- "charset-normalizer==2.0.12",
- "decorator==5.1.1",
- "idna==3.7",
- "igraph==0.10.6",
- "joblib==1.2.0",
- "macholib==1.16",
- "progress==1.6",
- "pyinstaller==5.13.1",
- "pyinstaller-hooks-contrib==2022.7",
- "PySocks==1.7.1",
- "python-dotenv==0.20.0",
- "scikit-learn==1.3.0",
- "scipy==1.10.0",
- "six==1.16.0",
- "sklearn==0.0",
- "soupsieve==2.3.2.post1",
- "termcolor==1.1.0",
- "texttable==1.6.4",
- "threadpoolctl==3.1.0",
- "urllib3==1.26.19",
- "validators==0.20.0",
- "treelib==1.6.1",
- "numpy==1.24.4",
- "unipath==1.1",
- "httpx[socks]==0.25.0",
- "tabulate==0.9.0",
- "phonenumbers==8.13.22",
- "pytest==7.4.2",
- "yattag==1.15.1",
- "toml==0.10.2",
+ "altgraph>=0.17.4",
+ "beautifulsoup4>=4.12.3",
+ "certifi>=2024.8.30",
+ "charset-normalizer>=3.3.2",
+ "decorator>=5.1.1",
+ "idna>=3.7",
+ "igraph>=0.11.6",
+ "joblib>=1.3.2",
+ "macholib>=1.16.3",
+ "progress>=1.6",
+ "pyinstaller>=6.8.0",
+ "pyinstaller-hooks-contrib>=2024.8",
+ "PySocks>=1.7.1",
+ "python-dotenv>=1.0.1",
+ "scikit-learn>=1.5.1",
+ "scipy>=1.13.1",
+ "six>=1.16.0",
+ "soupsieve>=2.5",
+ "termcolor>=2.4.0",
+ "texttable>=1.7.0",
+ "threadpoolctl>=3.5.0",
+ "urllib3>=2.2.2",
+ "validators>=0.33.0",
+ "treelib>=1.7.0",
+ "numpy>=1.26.4",
+ "unipath>=1.1",
+ "httpx[socks]>=0.28.1",
+ "tabulate>=0.9.0",
+ "phonenumbers>=8.13.45",
+ "pytest>=8.3.2",
+ "yattag>=1.15.1",
+ "toml>=0.10.2",
+ "lxml>=5.3.0",
]
diff --git a/requirements.txt b/requirements.txt
index 8dd1ed1f..a0251b01 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,274 +1,43 @@
-altgraph==0.17.2 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:743628f2ac6a7c26f5d9223c91ed8ecbba535f506f4b6f558885a8a56a105857 \
- --hash=sha256:ebf2269361b47d97b3b88e696439f6e4cbc607c17c51feb1754f90fb79839158
-anyio==4.0.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:cfdb2b588b9fc25ede96d8db56ed50848b0b649dca3dd1df0b11f683bb9e0b5f \
- --hash=sha256:f7ed51751b2c2add651e5747c891b47e26d2a21be5d32d9311dfe9692f3e5d7a
-beautifulsoup4==4.11.1 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:58d5c3d29f5a36ffeb94f02f0d786cd53014cf9b3b3951d42e0080d8a9498d30 \
- --hash=sha256:ad9aa55b65ef2808eb405f46cf74df7fcb7044d5cbc26487f96eb2ef2e436693
-certifi==2023.7.22 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082 \
- --hash=sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9
-charset-normalizer==2.0.12 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:2857e29ff0d34db842cd7ca3230549d1a697f96ee6d3fb071cfa6c7393832597 \
- --hash=sha256:6881edbebdb17b39b4eaaa821b438bf6eddffb4468cf344f09f89def34a8b1df
-colorama==0.4.6 ; python_version >= "3.9" and python_full_version <= "3.11.4" and sys_platform == "win32" \
- --hash=sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44 \
- --hash=sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6
-decorator==5.1.1 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:637996211036b6385ef91435e4fae22989472f9d571faba8927ba8253acbc330 \
- --hash=sha256:b8c3f85900b9dc423225913c5aace94729fe1fa9763b38939a95226f02d37186
-exceptiongroup==1.1.3 ; python_version >= "3.9" and python_version < "3.11" \
- --hash=sha256:097acd85d473d75af5bb98e41b61ff7fe35efe6675e4f9370ec6ec5126d160e9 \
- --hash=sha256:343280667a4585d195ca1cf9cef84a4e178c4b6cf2274caef9859782b567d5e3
-h11==0.14.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d \
- --hash=sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761
-httpcore==0.18.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:13b5e5cd1dca1a6636a6aaea212b19f4f85cd88c366a2b82304181b769aab3c9 \
- --hash=sha256:adc5398ee0a476567bf87467063ee63584a8bce86078bf748e48754f60202ced
-httpx[socks]==0.25.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:181ea7f8ba3a82578be86ef4171554dd45fec26a02556a744db029a0a27b7100 \
- --hash=sha256:47ecda285389cb32bb2691cc6e069e3ab0205956f681c5b2ad2325719751d875
-idna==3.3 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff \
- --hash=sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d
-igraph==0.10.6 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:053cdff880e46dca5a754777a5b2d68e904e82e831c6eb8ba7fdaed8d004a888 \
- --hash=sha256:06ca4879236cbb52dd33ddb8e5d562b387f91607fe2bc5f277054778560448c2 \
- --hash=sha256:0a753a48f96c48d01944221f189b989a03151f6e0e37862675248612e45ddd18 \
- --hash=sha256:169d7f1f8f1a39d1fa965dba0cab69b0d03de4b5e3272b196c60c3104a7e2735 \
- --hash=sha256:1a21dae12de610474b5b64f92822f3da4af2cedd60413b56aa55cc165dc0c763 \
- --hash=sha256:1a3dcf7b23e05dc7b1cf258bd82a52ed9a71e36ccc091480475bfe210ab60a17 \
- --hash=sha256:1deb496ccf6a71d07f4952e05b6aea3dd10e18abae9aadcbfd88f640cc804fa1 \
- --hash=sha256:26ae1fe7e4e55a851cdffcd302d9ad530303e47676424ee422dbede7450f948f \
- --hash=sha256:2913d91d9aeae107295fe238e1fe73d1150fca51b4e863c0442904dfb36f0d5a \
- --hash=sha256:3296e013c695488c313daa218aa5861dcd3623f298935f2f6f48b2bf1e65e5d0 \
- --hash=sha256:3bb368af87b5ea1877baf0cc337e6d83bb688f1f81c9f75973da92c30a4106c3 \
- --hash=sha256:3e93a2d6ac310d8f6b258aa659f1e2c36ed6a5c2982f79425e33a0d370df95c6 \
- --hash=sha256:409f377ea828d73bdf8da1e8372492db570efea765508eb091ec827492604ab7 \
- --hash=sha256:434f71859d39bfe7285c1ff02486492cf0d9236e50d2b0efd3f3fb316f106ebe \
- --hash=sha256:47637170741eb48550a18dc1072d354476866a3d6bc30b260a23acc04d15168f \
- --hash=sha256:4c743703ff6a9907deddf3f15ddb7a4e0d525a0055e5cf29ac2143b865fdf6c2 \
- --hash=sha256:4f1a33d7bf8ebaaa0f6401db437e23c3fe90b13c042897fa6123e0cf63605149 \
- --hash=sha256:521e9a1ab869d65cdc7713f2649794e8076c2d6b210dcad37ac6bcaffaf858de \
- --hash=sha256:527319db746f22adefba3ccbe1e7fba485edd14339af7c953e34e329f04d423f \
- --hash=sha256:53c0d0000f57c920549626b48e4b81685c268795df3657251763e50b83e8dac5 \
- --hash=sha256:547f5e815063e68acd1efedf31ccb3f204a7d6e80fdddd96cb18b11610744912 \
- --hash=sha256:6142060215c7bc5007f65ceffaa2ae1d84d0ef45bd9cbe8bc0670380200beb0f \
- --hash=sha256:62a39bcebfe863c607642e110c4846de9631caa4ce1c6e2e11fae9b1d66904dd \
- --hash=sha256:69d21b26599c86c6b937138280b13306f9b4e380498cc13896c603985c5ee80a \
- --hash=sha256:71a03c57550066a5cbf6fc6621d470fbf4abcf33ecb975a21e13e09cf45b965b \
- --hash=sha256:76f7aad294514412f835366a7d9a9c1e7a34c3e6ef0a6c3a1a835234323228e8 \
- --hash=sha256:7e737d4946e30bf561272a394d3c35181d453ab605a9650812200510b92b58b8 \
- --hash=sha256:7edef14bed493d104213e95a21252f6ac9374fb1383966244c9fb3c615927317 \
- --hash=sha256:81c4809ae45406cb81f985b643d8c9516c0394fa5f0aabe9a617f6506501c97e \
- --hash=sha256:81fbd93bbe81127aa98247a777a4eb2902443079254696d5aaa0a146cff88663 \
- --hash=sha256:82bacc57b40e6f2f1e7550073a283af38ed1ca9654f68796376eea1413988768 \
- --hash=sha256:868b558ca3d3363e0864246c20a9e937f849125103bd5e1e45e5f233121c1632 \
- --hash=sha256:936c681b3d6c507f3d1008c37d7ce6ef570d0ad932a94af1d512609da72a8ec1 \
- --hash=sha256:95b1b02a3d0c321339f2ed7283f37a22591c3b6a084393748369d5815aafe0a9 \
- --hash=sha256:96cdb8fdd9be3e4bd741b71436286f011f39bf7477cf7406d0d5ad9afa55f812 \
- --hash=sha256:a1cc73149e5ae448054c04050e5cfcdd9d886f72883ec9ca11185e16f62df2b1 \
- --hash=sha256:a4d88c6b392e70d1e8b9fb7747b545c9837b0232105ace379e63a6b08d987f23 \
- --hash=sha256:ad8a9245d1d27f50003c387105a52a9794b070fcc448f23fd58f7288126500ea \
- --hash=sha256:b3e038f78324e138869358bf17f9e57bcd665f7ddf6ed4b5f83fcbfc782f5059 \
- --hash=sha256:b40da28b2ed1b35d24794996ff38ed3292ddaeea8cbeb48da146df0bf32021ae \
- --hash=sha256:c08023eb9a3df384e5d8285ec8a60d0e5c1738c6d139a0b51ac5da024538d466 \
- --hash=sha256:c21f39fd40f7a0534442cb52e7dc07ee59aad4e52238a3b1c5fcb76f880d9648 \
- --hash=sha256:ca05a065108d126771d4867c1e6867fdc340f86f9634978de1326aa2de782138 \
- --hash=sha256:d1a935f7cb7a7490eef225bb5072ba3208cddf4a26aab9395f21a5a6082ed48f \
- --hash=sha256:da6009f347bdf14191bb65352a9eb5b1e2ba9014d928306b60b460dc549d21c8 \
- --hash=sha256:ddbbdaab269700d35969feaab73b9cbf7c55daa7711aaa01e0f9c598c9b8fbe1 \
- --hash=sha256:e1d3a0882224dafadb259a0e660650779a7714f3d931a03127af43ef05a4d6e2 \
- --hash=sha256:eb97640e9e71913015e7073341a5f6b4017fe025222950873c507d4ee97670f9 \
- --hash=sha256:eca5d01773a8b48bb585fb972c8afcf0cbfb791627a6706ccf56c0685b8ccca7 \
- --hash=sha256:f375b5e9faea3381c9bb878195cbfc83444a13fb357f8e6c5767d2d9e3745e26
-iniconfig==2.0.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3 \
- --hash=sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374
-joblib==1.2.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:091138ed78f800342968c523bdde947e7a305b8594b910a0fea2ab83c3c6d385 \
- --hash=sha256:e1cee4a79e4af22881164f218d4311f60074197fb707e082e803b61f6d137018
-macholib==1.16 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:001bf281279b986a66d7821790d734e61150d52f40c080899df8fefae056e9f7 \
- --hash=sha256:5a0742b587e6e57bfade1ab90651d4877185bf66fd4a176a488116de36878229
-numpy==1.24.4 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f \
- --hash=sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61 \
- --hash=sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7 \
- --hash=sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400 \
- --hash=sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef \
- --hash=sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2 \
- --hash=sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d \
- --hash=sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc \
- --hash=sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835 \
- --hash=sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706 \
- --hash=sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5 \
- --hash=sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4 \
- --hash=sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6 \
- --hash=sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463 \
- --hash=sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a \
- --hash=sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f \
- --hash=sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e \
- --hash=sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e \
- --hash=sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694 \
- --hash=sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8 \
- --hash=sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64 \
- --hash=sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d \
- --hash=sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc \
- --hash=sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254 \
- --hash=sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2 \
- --hash=sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1 \
- --hash=sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810 \
- --hash=sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9
-packaging==23.2 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5 \
- --hash=sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7
-pefile==2023.2.7 ; python_version >= "3.9" and python_full_version <= "3.11.4" and sys_platform == "win32" \
- --hash=sha256:82e6114004b3d6911c77c3953e3838654b04511b8b66e8583db70c65998017dc \
- --hash=sha256:da185cd2af68c08a6cd4481f7325ed600a88f6a813bad9dea07ab3ef73d8d8d6
-phonenumbers==8.13.22 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:001664c90f59b8954766c2db85adafc8dbc96177efeb49607ca4e64a7acaf569 \
- --hash=sha256:85ceeba9e67984ba98182c77e8e4c70093d38c0c6a0cb2bd392e0694ddaeb1f6
-pluggy==1.3.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12 \
- --hash=sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7
-progress==1.6 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:c9c86e98b5c03fa1fe11e3b67c1feda4788b8d0fe7336c2ff7d5644ccfba34cd
-pyinstaller-hooks-contrib==2022.7 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:5fdb97dcae177955db7ab27840cba97b89dc0c7f4fd9142bba0f9b8d8df85c48 \
- --hash=sha256:6675634279cfe9e475580fb310c3d557037baefb065e6cb5a69a124361b926fd
-pyinstaller==5.13.1 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:086e68aa1e72f6aa13b9d170a395755e2b194b8ab410caeed02d16b432410c8c \
- --hash=sha256:2519db3edec87d8c33924c2c4b7e176d8c1bbd9ba892d77efb67281925e621d6 \
- --hash=sha256:29341d2e86d5ce7df993e797ee96ef679041fc85376d31c35c7b714085a21299 \
- --hash=sha256:3c9cfe6d5d2f392d5d47389f6d377a8f225db460cdd01048b5a3de1d99c24ebe \
- --hash=sha256:5d801db3ceee58d01337473ea897e96e4bb21421a169dd7cf8716754617ff7fc \
- --hash=sha256:78d1601a11475b95dceff6eaf0c9cd74d93e3f47b5ce4ad63cd76e7a369d3d04 \
- --hash=sha256:a2e7a1d76a7ac26f1db849d691a374f2048b0e204233028d25d79a90ecd1fec8 \
- --hash=sha256:aa609aca62edd8cdcf7740677a21525e6c23b5e9a8f821ec8a80c68947771b5d \
- --hash=sha256:ad6e31a8f35a463c6140e4cf979859197edc9831a1039253408b0fe5eec274dc \
- --hash=sha256:b70ebc10811b30bbea4cf5b81fd1477db992c2614cf215edc987cda9c5468911 \
- --hash=sha256:b8d4000af72bf72f8185d420cd0a0aee0961f03a5c3511dc3ff08cdaef0583de \
- --hash=sha256:e033218c8922f0342b6095fb444ecb3bc6747dfa58cac5eac2b985350f4b681e
-pysocks==1.7.1 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299 \
- --hash=sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5 \
- --hash=sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0
-pytest==7.4.2 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002 \
- --hash=sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069
-python-dotenv==0.20.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:b7e3b04a59693c42c36f9ab1cc2acc46fa5df8c78e178fc33a8d4cd05c8d498f \
- --hash=sha256:d92a187be61fe482e4fd675b6d52200e7be63a12b724abbf931a40ce4fa92938
-pywin32-ctypes==0.2.2 ; python_version >= "3.9" and python_full_version <= "3.11.4" and sys_platform == "win32" \
- --hash=sha256:3426e063bdd5fd4df74a14fa3cf80a0b42845a87e1d1e81f6549f9daec593a60 \
- --hash=sha256:bf490a1a709baf35d688fe0ecf980ed4de11d2b3e37b51e5442587a75d9957e7
-scikit-learn==1.3.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:0e8102d5036e28d08ab47166b48c8d5e5810704daecf3a476a4282d562be9a28 \
- --hash=sha256:151ac2bf65ccf363664a689b8beafc9e6aae36263db114b4ca06fbbbf827444a \
- --hash=sha256:1d54fb9e6038284548072df22fd34777e434153f7ffac72c8596f2d6987110dd \
- --hash=sha256:3a11936adbc379a6061ea32fa03338d4ca7248d86dd507c81e13af428a5bc1db \
- --hash=sha256:436aaaae2c916ad16631142488e4c82f4296af2404f480e031d866863425d2a2 \
- --hash=sha256:552fd1b6ee22900cf1780d7386a554bb96949e9a359999177cf30211e6b20df6 \
- --hash=sha256:6a885a9edc9c0a341cab27ec4f8a6c58b35f3d449c9d2503a6fd23e06bbd4f6a \
- --hash=sha256:7617164951c422747e7c32be4afa15d75ad8044f42e7d70d3e2e0429a50e6718 \
- --hash=sha256:79970a6d759eb00a62266a31e2637d07d2d28446fca8079cf9afa7c07b0427f8 \
- --hash=sha256:850a00b559e636b23901aabbe79b73dc604b4e4248ba9e2d6e72f95063765603 \
- --hash=sha256:8be549886f5eda46436b6e555b0e4873b4f10aa21c07df45c4bc1735afbccd7a \
- --hash=sha256:981287869e576d42c682cf7ca96af0c6ac544ed9316328fd0d9292795c742cf5 \
- --hash=sha256:9877af9c6d1b15486e18a94101b742e9d0d2f343d35a634e337411ddb57783f3 \
- --hash=sha256:998d38fcec96584deee1e79cd127469b3ad6fefd1ea6c2dfc54e8db367eb396b \
- --hash=sha256:9d953531f5d9f00c90c34fa3b7d7cfb43ecff4c605dac9e4255a20b114a27369 \
- --hash=sha256:ae80c08834a473d08a204d966982a62e11c976228d306a2648c575e3ead12111 \
- --hash=sha256:c470f53cea065ff3d588050955c492793bb50c19a92923490d18fcb637f6383a \
- --hash=sha256:c7e28d8fa47a0b30ae1bd7a079519dd852764e31708a7804da6cb6f8b36e3630 \
- --hash=sha256:ded35e810438a527e17623ac6deae3b360134345b7c598175ab7741720d7ffa7 \
- --hash=sha256:ee04835fb016e8062ee9fe9074aef9b82e430504e420bff51e3e5fffe72750ca \
- --hash=sha256:fd6e2d7389542eae01077a1ee0318c4fec20c66c957f45c7aac0c6eb0fe3c612
-scipy==1.10.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:0490dc499fe23e4be35b8b6dd1e60a4a34f0c4adb30ac671e6332446b3cbbb5a \
- --hash=sha256:0ab2a58064836632e2cec31ca197d3695c86b066bc4818052b3f5381bfd2a728 \
- --hash=sha256:151f066fe7d6653c3ffefd489497b8fa66d7316e3e0d0c0f7ff6acca1b802809 \
- --hash=sha256:16ba05d3d1b9f2141004f3f36888e05894a525960b07f4c2bfc0456b955a00be \
- --hash=sha256:27e548276b5a88b51212b61f6dda49a24acf5d770dff940bd372b3f7ced8c6c2 \
- --hash=sha256:2ad449db4e0820e4b42baccefc98ec772ad7818dcbc9e28b85aa05a536b0f1a2 \
- --hash=sha256:2f9ea0a37aca111a407cb98aa4e8dfde6e5d9333bae06dfa5d938d14c80bb5c3 \
- --hash=sha256:38bfbd18dcc69eeb589811e77fae552fa923067fdfbb2e171c9eac749885f210 \
- --hash=sha256:3afcbddb4488ac950ce1147e7580178b333a29cd43524c689b2e3543a080a2c8 \
- --hash=sha256:42ab8b9e7dc1ebe248e55f54eea5307b6ab15011a7883367af48dd781d1312e4 \
- --hash=sha256:441cab2166607c82e6d7a8683779cb89ba0f475b983c7e4ab88f3668e268c143 \
- --hash=sha256:4bd0e3278126bc882d10414436e58fa3f1eca0aa88b534fcbf80ed47e854f46c \
- --hash=sha256:4df25a28bd22c990b22129d3c637fd5c3be4b7c94f975dca909d8bab3309b694 \
- --hash=sha256:5cd7a30970c29d9768a7164f564d1fbf2842bfc77b7d114a99bc32703ce0bf48 \
- --hash=sha256:6e4497e5142f325a5423ff5fda2fff5b5d953da028637ff7c704378c8c284ea7 \
- --hash=sha256:6faf86ef7717891195ae0537e48da7524d30bc3b828b30c9b115d04ea42f076f \
- --hash=sha256:954ff69d2d1bf666b794c1d7216e0a746c9d9289096a64ab3355a17c7c59db54 \
- --hash=sha256:9b878c671655864af59c108c20e4da1e796154bd78c0ed6bb02bc41c84625686 \
- --hash=sha256:b901b423c91281a974f6cd1c36f5c6c523e665b5a6d5e80fcb2334e14670eefd \
- --hash=sha256:c8b3cbc636a87a89b770c6afc999baa6bcbb01691b5ccbbc1b1791c7c0a07540 \
- --hash=sha256:e096b062d2efdea57f972d232358cb068413dc54eec4f24158bcbb5cb8bddfd8
-setuptools==75.8.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3 \
- --hash=sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6
-six==1.16.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926 \
- --hash=sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254
-sklearn==0.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:e23001573aa194b834122d2b9562459bf5ae494a2d59ca6b8aa22c85a44c0e31
-sniffio==1.3.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101 \
- --hash=sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384
-socksio==1.0.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:95dc1f15f9b34e8d7b16f06d74b8ccf48f609af32ab33c608d08761c5dcbb1f3 \
- --hash=sha256:f88beb3da5b5c38b9890469de67d0cb0f9d494b78b106ca1845f96c10b91c4ac
-soupsieve==2.3.2.post1 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:3b2503d3c7084a42b1ebd08116e5f81aadfaea95863628c80a3b774a11b7c759 \
- --hash=sha256:fc53893b3da2c33de295667a0e19f078c14bf86544af307354de5fcf12a3f30d
-tabulate==0.9.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \
- --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f
-termcolor==1.1.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b
-texttable==1.6.4 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:42ee7b9e15f7b225747c3fa08f43c5d6c83bc899f80ff9bae9319334824076e9 \
- --hash=sha256:dd2b0eaebb2a9e167d1cefedab4700e5dcbdb076114eed30b58b97ed6b37d6f2
-threadpoolctl==3.1.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:8b99adda265feb6773280df41eece7b2e6561b772d21ffd52e372f999024907b \
- --hash=sha256:a335baacfaa4400ae1f0d8e3a58d6674d2f8828e3716bb2802c44955ad391380
-toml==0.10.2 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b \
- --hash=sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f
-tomli==2.0.1 ; python_version >= "3.9" and python_version < "3.11" \
- --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
- --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
-treelib==1.7.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:9bff1af416b9e642a6cd0e0431d15edf26a24b8d0c8ae68afbd3801b5e30fb61 \
- --hash=sha256:c37795eaba19f73f3e1a905ef3f4f0cab660dc7617126e8ae99391e25c755416
-unipath==1.1 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:09839adcc72e8a24d4f76d63656f30b5a1f721fc40c9bcd79d8c67bdd8b47dae \
- --hash=sha256:e6257e508d8abbfb6ddd8ec357e33589f1f48b1599127f23b017124d90b0fff7
-urllib3==1.26.18 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:34b97092d7e0a3a8cf7cd10e386f401b3737364026c45e622aa02903dffe0f07 \
- --hash=sha256:f8ecc1bba5667413457c529ab955bf8c67b45db799d159066261719e328580a0
-validators==0.20.0 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:24148ce4e64100a2d5e267233e23e7afeb55316b47d30faae7eb6e7292bc226a
-yattag==1.15.1 ; python_version >= "3.9" and python_full_version <= "3.11.4" \
- --hash=sha256:960fa54be1229d96f43178133e0b195c003391fdc49ecdb6b69b7374db6be416
-
-numpy~=1.24.4
-beautifulsoup4~=4.11.1
-sklearn~=0.0
-scikit-learn~=1.3.0
-httpx[socks]~=0.25.0
-yattag~=1.15.1
-termcolor~=1.1.0
-python-dotenv~=0.20.0
-Unipath~=1.1
-validators~=0.20.0
-phonenumbers~=8.13.22
-tabulate~=0.9.0
-treelib~=1.7.0
-toml~=0.10.2
\ No newline at end of file
+# Updated dependencies for TorBot - Security and compatibility fixes
+altgraph>=0.17.4
+beautifulsoup4>=4.12.3
+certifi>=2024.8.30
+charset-normalizer>=3.3.2
+colorama>=0.4.6
+decorator>=5.1.1
+exceptiongroup>=1.2.2
+h11>=0.14.0
+httpcore>=1.0.5
+httpx[socks]>=0.28.1
+idna>=3.7
+igraph>=0.11.6
+iniconfig>=2.0.0
+joblib>=1.3.2
+lxml>=5.3.0
+macholib>=1.16.3
+numpy>=1.26.4
+packaging>=24.1
+pefile>=2023.2.7
+phonenumbers>=8.13.45
+pluggy>=1.5.4
+progress>=1.6
+pyinstaller>=6.8.0
+pyinstaller-hooks-contrib>=2024.8
+pysocks>=1.7.1
+pytest>=8.3.2
+python-dotenv>=1.0.1
+pywin32-ctypes>=0.2.2
+scikit-learn>=1.5.1
+scipy>=1.13.1
+six>=1.16.0
+soupsieve>=2.5
+termcolor>=2.4.0
+texttable>=1.7.0
+threadpoolctl>=3.5.0
+toml>=0.10.2
+treelib>=1.7.0
+tabulate>=0.9.0
+unipath>=1.1
+urllib3>=2.2.2
+validators>=0.33.0
+yattag>=1.15.1
\ No newline at end of file
diff --git a/src/torbot/modules/linktree.py b/src/torbot/modules/linktree.py
index 89477664..7bd73e08 100644
--- a/src/torbot/modules/linktree.py
+++ b/src/torbot/modules/linktree.py
@@ -62,7 +62,9 @@ def _append_node(self, id: str, parent_id: str or None) -> None:
soup.title.text.strip() if soup.title is not None else parse_hostname(id)
)
try:
- [classification, accuracy] = classify(resp.text)
+ # [classification, accuracy] = classify(resp.text)
+ classification = "unknown"
+ accuracy = 0.0
numbers = parse_phone_numbers(soup)
emails = parse_emails(soup)
data = LinkNode(