diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index d1ffbbc..128a2b5 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -232,6 +232,13 @@ The project includes a comprehensive linting script that validates all file type - **No secrets**: Never commit SSH keys, passwords, or tokens - **Documentation**: Update docs for any infrastructure changes +#### End-to-End Smoke Testing + +For verifying the functionality of the tracker from an end-user's perspective (e.g., simulating announce/scrape requests), refer to the **Smoke Testing Guide**. This guide explains how to use the official `torrust-tracker-client` tools to perform black-box testing against a running tracker instance without needing a full BitTorrent client. + +- **Guide**: [Smoke Testing Guide](../docs/guides/smoke-testing-guide.md) +- **When to use**: After a deployment (`make apply`) or to validate that all services are working together correctly. + ### Security Guidelines #### Secrets Management diff --git a/README.md b/README.md index 531ff2c..a5e3066 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ make destroy # Clean up - [Application Overview](application/README.md) - Application components and deployment - [Production Setup](application/docs/production-setup.md) - Production - deployment + deployment with MySQL - [Deployment Guide](application/docs/deployment.md) - Deployment procedures - [Backup Procedures](application/docs/backups.md) - Data backup and recovery - [Rollback Guide](application/docs/rollbacks.md) - Application rollbacks @@ -112,6 +112,8 @@ make destroy # Clean up main Makefile is at repository root - [ADR-002: Docker for All Services](docs/adr/002-docker-for-all-services.md) - Why we use Docker for all services including UDP tracker + - [ADR-003: Use MySQL Over MariaDB](docs/adr/003-use-mysql-over-mariadb.md) - + Why we chose MySQL instead of MariaDB for the database backend ## ๐Ÿ› ๏ธ Development diff --git a/application/.env.production b/application/.env.production index 52ae2e9..45f573c 100644 --- a/application/.env.production +++ b/application/.env.production @@ -1,5 +1,20 @@ +# Torrust Tracker Demo - Production Environment Configuration +# +# This configuration uses MySQL as the default database backend. +# Make sure to change the default passwords before deployment! + USER_ID=1000 +# Database Configuration (MySQL) +MYSQL_ROOT_PASSWORD=secure_root_password_change_me +MYSQL_DATABASE=torrust_tracker +MYSQL_USER=torrust +MYSQL_PASSWORD=secure_password_change_me + +# Tracker Database Configuration +TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=mysql +TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__PATH=mysql://torrust:${MYSQL_PASSWORD}@mysql:3306/torrust_tracker + # Tracker TORRUST_TRACKER_CONFIG_TOML= TORRUST_TRACKER_CONFIG_OVERRIDE_HTTP_API__ACCESS_TOKENS__ADMIN='MyAccessToken' diff --git a/application/README.md b/application/README.md index cb2e39f..071e7d6 100644 --- a/application/README.md +++ b/application/README.md @@ -59,9 +59,30 @@ application/ - **Nginx**: Reverse proxy and SSL termination - **Prometheus**: Metrics collection and storage - **Grafana**: Metrics visualization and dashboards -- **MySQL**: Database (migrating from SQLite) +- **MySQL**: Default database backend for production - **Certbot**: SSL certificate management +### Database Configuration + +The Torrust Tracker Demo uses **MySQL as the default database backend** for +production deployments. This provides: + +- **Reliability**: Production-grade database with ACID compliance +- **Scalability**: Support for high-throughput tracking operations +- **Data Integrity**: Consistent data storage and retrieval +- **Performance**: Optimized for concurrent tracker operations + +**Database Service**: The MySQL service is automatically configured with: + +- Database initialization scripts +- Proper networking and security +- Data persistence across container restarts +- Health checks and monitoring + +For development and testing environments, you can optionally configure SQLite +by modifying the tracker configuration, though MySQL is recommended for all +production use cases. + ## ๐Ÿš€ Quick Start ### Application Deployment diff --git a/application/compose.yaml b/application/compose.yaml index 8d5d6fe..a0ce3d1 100644 --- a/application/compose.yaml +++ b/application/compose.yaml @@ -63,7 +63,7 @@ services: networks: - backend_network ports: - - "9090:9090" # This port should not be exposed to the internet + - "9090:9090" # This port should not be exposed to the internet volumes: - ./storage/prometheus/etc:/etc/prometheus:Z logging: @@ -73,6 +73,35 @@ services: depends_on: - tracker + mysql: + image: mysql:8.0 + container_name: mysql + restart: unless-stopped + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=${MYSQL_DATABASE} + - MYSQL_USER=${MYSQL_USER} + - MYSQL_PASSWORD=${MYSQL_PASSWORD} + networks: + - backend_network + ports: + - "3306:3306" # Only for debugging, remove in production + volumes: + - mysql_data:/var/lib/mysql + - ./storage/mysql/init:/docker-entrypoint-initdb.d:ro + command: > + --character-set-server=utf8mb4 + --collation-server=utf8mb4_unicode_ci + healthcheck: + test: ["CMD", "mysqladmin", "ping", "-h", "localhost", "-p${MYSQL_PASSWORD}"] + interval: 10s + timeout: 5s + retries: 5 + logging: + options: + max-size: "10m" + max-file: "10" + tracker: image: torrust/tracker:develop container_name: tracker @@ -80,8 +109,9 @@ services: restart: unless-stopped environment: - USER_ID=${USER_ID} - - TORRUST_TRACKER_DATABASE=${TORRUST_TRACKER_DATABASE:-sqlite3} - - TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=${TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER:-sqlite3} + - TORRUST_TRACKER_DATABASE=${TORRUST_TRACKER_DATABASE:-mysql} + - TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=${TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER:-mysql} + - TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__PATH=${TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__PATH:-mysql://torrust:password@mysql:3306/torrust_tracker} - TORRUST_TRACKER_CONFIG_OVERRIDE_HTTP_API__ACCESS_TOKENS__ADMIN=${TORRUST_TRACKER_CONFIG_OVERRIDE_HTTP_API__ACCESS_TOKENS__ADMIN:-MyAccessToken} networks: - backend_network @@ -98,6 +128,8 @@ services: options: max-size: "10m" max-file: "10" + depends_on: + - mysql networks: frontend_network: {} diff --git a/application/docs/deployment.md b/application/docs/deployment.md index 3b33670..f39127b 100644 --- a/application/docs/deployment.md +++ b/application/docs/deployment.md @@ -1,46 +1,93 @@ # Deployment -1. SSH into the server. -2. Execute the deployment script: `./bin/deploy-torrust-tracker-demo.com.sh`. -3. Execute the smoke tests: - - ```console - # Clone Torrust Tracker - git@github.com:torrust/torrust-tracker.git - cd torrust-tracker +This document outlines the deployment process for the Torrust Tracker demo application. + +## 1. Prerequisites + +- Ensure you have SSH access to the production server. +- The server should be provisioned and configured according to the + [Production Setup Guide](./production-setup.md). + +## 2. Deployment Steps + +1. **SSH into the server**. + +2. **Navigate to the application directory**: + + ```bash + cd /home/torrust/github/torrust/torrust-tracker-demo ``` - Execute the following commands to run the tracker client and checker. +3. **Pull the latest changes** from the repository: + + ```bash + git pull + ``` + +4. **Run the deployment script**: + + ```bash + ./share/bin/deploy-torrust-tracker-demo.com.sh + ``` + + This script handles: + + - Stopping services + - Rebuilding containers + - Starting services + +## 3. Verification and Smoke Testing + +After deployment, verify that all services are running correctly. - Simulate a torrent announce to the tracker using UDP: +### Service Status - ```console +Check the status of all Docker containers: + +```bash +docker compose ps +``` + +### Application Logs + +Check the logs for the tracker container to ensure it started without errors: + +```bash +./share/bin/tracker-filtered-logs.sh +``` + +### Smoke Tests + +Execute the following smoke tests from a machine with the `torrust-tracker` repository cloned. + +1. **UDP Announce**: + + ```bash cargo run -p torrust-tracker-client --bin udp_tracker_client announce \ udp://tracker.torrust-demo.com:6969/announce \ 9c38422213e30bff212b30c360d26f9a02136422 | jq ``` - Simulate a torrent scrape to the tracker using HTTP: +2. **HTTP Announce**: - ```console + ```bash cargo run -p torrust-tracker-client --bin http_tracker_client announce \ - https://tracker.torrust-demo.com \ + https://tracker.torrust-demo.com/announce \ 9c38422213e30bff212b30c360d26f9a02136422 | jq ``` - Make a request to the health check endpoint: +3. **Health Check Endpoint**: - ```console + ```bash + curl https://tracker.torrust-demo.com/api/health_check + ``` + +4. **Run the comprehensive tracker checker**: + + ```bash TORRUST_CHECKER_CONFIG='{ "udp_trackers": ["udp://tracker.torrust-demo.com:6969/announce"], - "http_trackers": ["https://tracker.torrust-demo.com"], + "http_trackers": ["https://tracker.torrust-demo.com/announce"], "health_checks": ["https://tracker.torrust-demo.com/api/health_check"] }' cargo run -p torrust-tracker-client --bin tracker_checker - - ``` - -4. Check the logs of the tracker container to see if everything is working: - - ```console - ./share/bin/tracker-filtered-logs.sh ``` diff --git a/application/docs/firewall-requirements.md b/application/docs/firewall-requirements.md index 0bd20ff..9186865 100644 --- a/application/docs/firewall-requirements.md +++ b/application/docs/firewall-requirements.md @@ -109,8 +109,6 @@ The tracker uses four main ports, each serving a specific purpose: - `/api/v1/metrics` - Prometheus metrics - Used by Prometheus for monitoring and Grafana dashboards -### Application Service Mapping - ### Torrust Tracker - **UDP ports 6868, 6969**: BitTorrent announce endpoints diff --git a/application/docs/production-setup.md b/application/docs/production-setup.md index d27f88c..7764718 100644 --- a/application/docs/production-setup.md +++ b/application/docs/production-setup.md @@ -1,13 +1,54 @@ -# Setup +# Production Environment Setup -Follow instructions on [Deploying Torrust To Production](https://torrust.com/blog/deploying-torrust-to-production). +This guide details the steps required to configure the production environment +for the Torrust Tracker demo. -You need to also enable a [firewall](./firewall-requirements.md). +## 1. Initial Setup + +For the initial server setup, follow the instructions on [Deploying Torrust To Production](https://torrust.com/blog/deploying-torrust-to-production). + +You also need to enable a [firewall](./firewall-requirements.md). The application is located in the directory: `/home/torrust/github/torrust/torrust-tracker-demo`. -To run docker compose commands you need to cd to the app dir: +To run Docker Compose commands, you need to be in the application directory: ```console cd /home/torrust/github/torrust/torrust-tracker-demo ``` + +## 2. Database Configuration + +The production environment uses MySQL as the database backend. + +### Environment Variables + +Create a `.env` file by copying the production template: + +```bash +cp .env.production .env +``` + +**Crucially, you must edit the `.env` file and set secure passwords** for the following variables: + +- `MYSQL_ROOT_PASSWORD`: The root password for the MySQL server. +- `MYSQL_PASSWORD`: The password for the `torrust` user. +- `TRACKER_ADMIN_TOKEN`: The admin token for the tracker API. + +### Database Initialization + +The MySQL service automatically initializes the database and creates the necessary tables on first startup. + +## 3. Running the Application + +Once the `.env` file is configured, you can start all services: + +```bash +docker compose up -d +``` + +You can check the status of the services with: + +```bash +docker compose ps +``` diff --git a/application/share/container/default/config/prometheus.yml b/application/share/container/default/config/prometheus.yml index 858d134..3240137 100644 --- a/application/share/container/default/config/prometheus.yml +++ b/application/share/container/default/config/prometheus.yml @@ -1,3 +1,4 @@ +--- global: scrape_interval: 15s # How often to scrape metrics diff --git a/docs/README.md b/docs/README.md index 3cb0c55..70cfbc3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -18,6 +18,10 @@ This directory currently contains cross-cutting documentation: - [ADR-001: Makefile Location](adr/001-makefile-location.md) - Decision to keep Makefile at repository root level +- [ADR-002: Docker for All Services](adr/002-docker-for-all-services.md) - Decision + to use Docker for all services including UDP tracker +- [ADR-003: Use MySQL Over MariaDB](adr/003-use-mysql-over-mariadb.md) - Decision + to use MySQL instead of MariaDB for database backend ### ๐Ÿ“… [`plans/`](plans/) (Ongoing Plans and Roadmaps) @@ -26,6 +30,13 @@ This directory currently contains cross-cutting documentation: - [Hetzner Migration Plan](plans/hetzner-migration-plan.md) - Comprehensive plan for migrating from Digital Ocean to Hetzner infrastructure +### ๐ŸŽฏ [`issues/`](issues/) (Implementation Plans) + +**Issue Implementation Plans:** + +- [Phase 1: MySQL Migration](issues/12-use-mysql-instead-of-sqlite-by-default.md) - + Detailed implementation plan for database migration from SQLite to MySQL + ### Future Categories The following directories can be created as needed: diff --git a/docs/adr/003-use-mysql-over-mariadb.md b/docs/adr/003-use-mysql-over-mariadb.md new file mode 100644 index 0000000..827e579 --- /dev/null +++ b/docs/adr/003-use-mysql-over-mariadb.md @@ -0,0 +1,173 @@ +# ADR-003: Use MySQL Over MariaDB for Database Backend + +## Status + +Accepted + +## Context + +As part of the migration from SQLite to a production-ready database backend for the Torrust +Tracker Demo, we needed to choose between MySQL and MariaDB. Both are popular open-source +relational database management systems, with MariaDB being a fork of MySQL that aims to +maintain compatibility while offering additional features and performance improvements. + +### Key Considerations + +1. **Official Torrust Tracker Support**: The Torrust Tracker project officially documents + and tests specific database drivers +2. **Compatibility and Reliability**: Ensuring maximum compatibility with the tracker + application and minimal integration issues +3. **Community and Documentation**: Availability of support, documentation, and + troubleshooting resources +4. **Future Maintenance**: Long-term maintainability and alignment with upstream project + decisions +5. **Performance**: Database performance for BitTorrent tracker workloads (primarily CRUD + operations) + +### Technical Requirements + +The Torrust Tracker requires: + +- Support for connection pooling via `r2d2_mysql` Rust crate +- MySQL-specific connection string format: `mysql://user:password@host:port/database` +- Compatibility with tracker's database schema and SQL syntax +- Auto-increment primary keys and foreign key constraints +- UTF-8 character encoding support + +## Decision + +We will use **MySQL 8.0** as the default database backend for the Torrust Tracker Demo instead of MariaDB. + +## Rationale + +### 1. Official Torrust Tracker Support + +- **Documented Support**: Torrust Tracker documentation explicitly mentions "SQLite and + MySQL" support, with no mention of MariaDB +- **Default Configuration**: All official examples, Docker configurations, and test + setups use MySQL 8.0 +- **Codebase Implementation**: The tracker uses the `r2d2_mysql` crate specifically + designed for MySQL, not a generic MariaDB-compatible driver + +### 2. Testing and Validation + +- **Integration Tests**: Torrust Tracker's test suite specifically uses MySQL 8.0 + containers for database driver testing +- **CI/CD Pipeline**: The official project's continuous integration validates against + MySQL, ensuring compatibility +- **Environment Variable**: Tests are controlled via + `TORRUST_TRACKER_CORE_RUN_MYSQL_DRIVER_TEST=true`, indicating MySQL-specific testing + +### 3. Community and Ecosystem Alignment + +- **BitTorrent Community Standard**: The broader BitTorrent tracker community + predominantly uses MySQL +- **Documentation Consistency**: All troubleshooting guides, configuration examples, and + community solutions reference MySQL +- **Upstream Alignment**: Following the same database choice as the upstream Torrust + Tracker project ensures consistent behavior + +### 4. Technical Implementation Benefits + +- **Connection String Compatibility**: MySQL connection strings are guaranteed to work + with the tracker's database driver +- **SQL Syntax Alignment**: The tracker's SQL queries are written and tested against + MySQL 8.0 specifically +- **Migration Path**: Future updates to Torrust Tracker will be tested against MySQL, + ensuring smooth upgrades + +### 5. Reduced Risk Profile + +- **Known Configuration**: Using MySQL eliminates uncertainty about MariaDB compatibility edge cases +- **Proven Compatibility**: MySQL 8.0 is the tested and validated database backend for Torrust Tracker +- **Support Availability**: Issues and solutions are more readily available for MySQL configurations + +## Consequences + +### Positive + +- **Maximum Compatibility**: Guaranteed compatibility with current and future Torrust Tracker versions +- **Better Documentation**: All configuration examples and troubleshooting guides apply directly +- **Simplified Maintenance**: No need to test or validate MariaDB-specific compatibility issues +- **Community Support**: Easier to get help from the Torrust community when using the standard database +- **Future-Proof**: Alignment with upstream project decisions reduces migration risks + +### Neutral + +- **Performance**: Both MySQL and MariaDB would provide similar performance for tracker workloads +- **Resource Usage**: Memory and CPU usage are comparable between the two databases +- **Feature Set**: The tracker uses basic SQL features available in both databases + +### Negative + +- **Vendor Lock-in**: Choosing MySQL over MariaDB means following Oracle's MySQL development path +- **License Considerations**: MySQL has Oracle's commercial licensing model (though GPL version is used) +- **Missing MariaDB Features**: We won't benefit from MariaDB-specific performance improvements or features + +## Implementation + +The MySQL 8.0 configuration is implemented as follows: + +### Docker Compose Configuration + +```yaml +mysql: + image: mysql:8.0 + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=${MYSQL_DATABASE} + - MYSQL_USER=${MYSQL_USER} + - MYSQL_PASSWORD=${MYSQL_PASSWORD} + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci +``` + +### Tracker Configuration + +```toml +[core.database] +driver = "mysql" +path = "mysql://torrust:password@mysql:3306/torrust_tracker" +``` + +### Environment Variables + +```env +TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=mysql +TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__PATH=mysql://torrust:password@mysql:3306/torrust_tracker +``` + +## Alternative Considered + +### MariaDB 10.x + +**Pros:** + +- Generally faster than MySQL in some benchmarks +- More storage engines and advanced features +- Fully open-source governance model +- Active development and regular releases + +**Cons:** + +- Not officially supported or tested by Torrust Tracker +- Potential compatibility issues with MySQL-specific driver code +- Less documentation and community support for tracker use cases +- Risk of subtle differences causing issues in production +- May require additional validation and testing + +**Conclusion:** While MariaDB offers technical advantages, the benefits don't outweigh +the risks and compatibility concerns for this specific use case. + +## References + +- [Torrust Tracker Documentation](https://docs.rs/torrust-tracker/) +- [Torrust Tracker Database Configuration](https://docs.rs/torrust-tracker-configuration/latest/torrust_tracker_configuration/v2_0_0/database/struct.Database.html) +- [Torrust Tracker MySQL Driver Implementation](https://github.com/torrust/torrust-tracker/blob/main/packages/tracker-core/src/databases/driver/mysql.rs) +- [Torrust Tracker Docker Compose Example](https://github.com/torrust/torrust-tracker/blob/develop/docs/containers.md) +- [Issue #12: Use MySQL instead of SQLite by default](../issues/12-use-mysql-instead-of-sqlite-by-default.md) + +## Revision History + +- **2025-07-08**: Initial decision document created +- **Decision maker**: Development team +- **Reviewed by**: N/A (initial implementation) diff --git a/docs/guides/integration-testing-guide.md b/docs/guides/integration-testing-guide.md index 1f904ab..1d1696b 100644 --- a/docs/guides/integration-testing-guide.md +++ b/docs/guides/integration-testing-guide.md @@ -93,15 +93,15 @@ echo "=== Verifying Clean State ===" # Check VMs virsh list --all | grep torrust-tracker-demo && \ - echo "โŒ VM still exists!" || echo "โœ… No VM conflicts" + echo 'โŒ VM still exists!' || echo 'โœ… No VM conflicts' # Check volumes in user-default pool virsh vol-list user-default 2>/dev/null | grep torrust-tracker-demo && \ - echo "โŒ Volumes still exist!" || echo "โœ… No volume conflicts" + echo 'โŒ Volumes still exist!' || echo 'โœ… No volume conflicts' # Check OpenTofu state ls infrastructure/terraform/terraform.tfstate* 2>/dev/null && \ - echo "โŒ State files still exist!" || echo "โœ… No state file conflicts" + echo 'โŒ State files still exist!' || echo 'โœ… No state file conflicts' ``` **Expected Output**: All checks should show "โœ…" (no conflicts). @@ -120,7 +120,7 @@ virsh vol-delete torrust-tracker-demo.qcow2 user-default # Verify cleanup virsh vol-list user-default 2>/dev/null | grep torrust-tracker-demo && \ - echo "โŒ Volumes still exist!" || echo "โœ… No volume conflicts" + echo 'โŒ Volumes still exist!' || echo 'โœ… No volume conflicts' ``` **Expected Output**: Should show "โœ… No volume conflicts" after manual cleanup. @@ -392,8 +392,8 @@ while true; do echo "$(date): Testing SSH to $VM_IP..." if timeout 10 ssh -o StrictHostKeyChecking=no \ -o ConnectTimeout=10 torrust@$VM_IP \ - "echo 'SSH works!'" 2>/dev/null; then - echo "โœ… SSH connection successful!" + 'echo "SSH works!"' 2>/dev/null; then + echo 'โœ… SSH connection successful!' break fi echo "โณ Cloud-init still running... waiting 30 seconds" diff --git a/docs/issues/12-use-mysql-instead-of-sqlite-by-default.md b/docs/issues/12-use-mysql-instead-of-sqlite-by-default.md new file mode 100644 index 0000000..62f9814 --- /dev/null +++ b/docs/issues/12-use-mysql-instead-of-sqlite-by-default.md @@ -0,0 +1,310 @@ +# Phase 1: MySQL Migration Implementation Plan + +## ๐ŸŽฏ Overview + +This document outlines the detailed implementation plan for migrating from SQLite to MySQL +as the default database for the Torrust Tracker Demo deployment. + +**Parent Issue**: Phase 1: Database Migration to MySQL +**Migration Plan Reference**: +[docs/plans/hetzner-migration-plan.md](../plans/hetzner-migration-plan.md) +**Database Choice Decision**: +[ADR-003: Use MySQL Over MariaDB](../adr/003-use-mysql-over-mariadb.md) + +## ๐Ÿ“‹ Implementation Steps + +### Step 1: MySQL Service Configuration + +**File**: `application/compose.yaml` + +**Changes Required**: + +- Add MySQL service definition to Docker Compose +- Configure MySQL environment variables +- Set up proper networking and volumes +- Add dependency relationships + +**Implementation Details**: + +```yaml +mysql: + image: mysql:8.0 + container_name: mysql + restart: unless-stopped + environment: + - MYSQL_ROOT_PASSWORD=${MYSQL_ROOT_PASSWORD} + - MYSQL_DATABASE=${MYSQL_DATABASE} + - MYSQL_USER=${MYSQL_USER} + - MYSQL_PASSWORD=${MYSQL_PASSWORD} + networks: + - backend_network + ports: + - "3306:3306" # Only for debugging, remove in production + volumes: + - mysql_data:/var/lib/mysql + - ./storage/mysql/init:/docker-entrypoint-initdb.d:ro + command: --character-set-server=utf8mb4 --collation-server=utf8mb4_unicode_ci + logging: + options: + max-size: "10m" + max-file: "10" +``` + +**Dependencies**: + +- Update tracker service to depend on MySQL +- Ensure mysql_data volume is properly utilized + +### Step 2: Environment Variables Configuration + +**File**: `application/.env.production` + +**Changes Required**: + +- Add MySQL connection parameters +- Add tracker configuration overrides for MySQL +- Maintain backward compatibility documentation + +**Implementation Details**: + +```bash +# Database Configuration +MYSQL_ROOT_PASSWORD=secure_root_password_change_me +MYSQL_DATABASE=torrust_tracker +MYSQL_USER=torrust +MYSQL_PASSWORD=secure_password_change_me + +# Tracker Database Configuration +TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=mysql +TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__PATH=mysql://torrust:${MYSQL_PASSWORD}@mysql:3306/torrust_tracker +``` + +### Step 3: Tracker Configuration Updates + +**File**: `application/storage/tracker/etc/tracker.toml` + +**Changes Required**: + +- Update database section to use MySQL by default +- Remove SQLite-specific configurations +- Use MySQL connection string format + +**Implementation Details**: + +```toml +[core.database] +driver = "mysql" +path = "mysql://torrust:password_will_be_overridden_by_env@mysql:3306/torrust_tracker" +``` + +**Note**: Environment variables will override these default values in production. + +### Step 4: MySQL Initialization Directory + +**Directory**: `application/storage/mysql/init/` + +**Files to Create**: + +1. `README.md` - Documentation for initialization directory + +**Implementation Details**: + +`README.md`: + +```markdown +# MySQL Initialization Directory + +This directory is available for MySQL initialization scripts if needed in the future. + +## Notes + +- Scripts in this directory would be executed automatically by MySQL container on first startup +- The database and user are created automatically via environment variables +- **Torrust Tracker handles its own database migrations automatically** +- Database tables are created and updated by the tracker on startup +- No manual schema setup is required +``` + +**Important Note**: The Torrust Tracker automatically handles database migrations and table +creation. The database schema is managed by the tracker itself through its built-in +migration system in the database drivers. + +### Step 5: Docker Compose Service Dependencies + +**File**: `application/compose.yaml` + +**Changes Required**: + +- Add MySQL to tracker service dependencies +- Ensure proper startup order + +**Implementation Details**: + +```yaml +tracker: + # ...existing configuration... + depends_on: + - mysql + environment: + # ...existing environment variables... + - TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER=${TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__DRIVER:-mysql} + - TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__PATH=${TORRUST_TRACKER_CONFIG_OVERRIDE_CORE__DATABASE__PATH:-mysql://torrust:password@mysql:3306/torrust_tracker} +``` + +### Step 6: Documentation Updates + +**Files to Update**: + +1. `application/README.md` +2. `application/docs/production-setup.md` +3. `application/docs/deployment.md` (if exists) + +**Changes Required**: + +- Update database requirements section +- Add MySQL setup instructions +- Document environment variables +- Add migration notes for existing SQLite users + +## ๐Ÿงช Testing Strategy + +### Phase 1: Local Docker Compose Testing + +**Recommendation**: Test the MySQL integration locally with Docker Compose first, before +deploying to VMs. This saves time during development and avoids creating/destroying VMs +while iterating on the configuration. + +**Prerequisites**: + +```bash +# Ensure you're in the application directory +cd application/ +``` + +**Local Testing Steps**: + +1. **Local Service Testing**: + + ```bash + # Stop any existing services + docker compose down + + # Start only MySQL to test it independently + docker compose up mysql -d + + # Verify MySQL is running + docker compose ps + docker compose logs mysql + ``` + +2. **Full Stack Local Testing**: + + ```bash + # Start all services with the new MySQL configuration + docker compose up -d + + # Check all services are running + docker compose ps + + # Check MySQL startup logs + docker compose logs mysql + + # Check tracker connection logs + docker compose logs tracker + ``` + + **Expected Output**: + + ```bash + # Verify tracker is running and connected to MySQL + docker compose logs tracker | grep "Successfully connected to database" + ``` + +3. **Local Data Persistence Testing**: + + - **Status**: โœ… **Completed** + - **Verification**: Data persists in MySQL after service restarts. + + ```bash + # Restart services + docker compose restart + + # Verify tables still exist in MySQL + docker compose exec mysql mysql -u torrust -p torrust_tracker -e "SHOW TABLES;" + ``` + +4. **VM Integration Testing**: + + - **Status**: โœ… **Completed** (2025-07-08) + - **Description**: Deploy the complete stack on a local VM to test the full + infrastructure integration. + - **Solution**: MySQL 8.0 x86-64-v2 CPU requirement resolved by configuring VM with + `host-model` CPU mode to enable modern instruction sets. + + ```bash + # From the repository root + make apply # Deploy VM + make ssh # Connect to VM + # Run smoke tests from the smoke testing guide + ``` + + **Results**: + + - โœ… All Docker containers running successfully + - โœ… MySQL container: `Up 48 minutes (healthy)` - no more restart loops + - โœ… Tracker container: `Up 48 minutes (healthy)` - connected to MySQL + - โœ… All services responding to health checks + + **Technical Solution**: + + - **Issue**: MySQL 8.0 Docker image requires x86-64-v2 CPU instruction set + - **Fix**: Updated `infrastructure/terraform/main.tf` to use `host-model` CPU mode + - **Result**: VM CPU now supports x86-64-v2 instructions required by MySQL 8.0 + +### Phase 2: Documentation and Cleanup + +**Status**: โœ… **Completed** (2025-07-08) + +**Description**: Update all relevant documentation to reflect the MySQL migration and +remove any outdated SQLite references. + +**Files Updated**: + +- โœ… `application/README.md` - Added database configuration section explaining MySQL as default +- โœ… `application/docs/production-setup.md` - Already documented MySQL properly +- โœ… `application/.env.production` - Added header comments about MySQL configuration +- โœ… `docs/guides/smoke-testing-guide.md` - No database-specific changes needed (external testing) +- โœ… `.github/copilot-instructions.md` - Already updated with smoke testing guide references + +**Results**: + +- All documentation now reflects MySQL as the default database +- Added clear explanations about database configuration and requirements +- Maintained references to SQLite as a development/testing option +- Updated environment file with clear MySQL configuration comments +- Legacy SQLite configuration files preserved for reference and rollback scenarios + +## โœ… Completion Checklist + +- [x] MySQL service added to `compose.yaml` +- [x] Environment variables configured in `.env.production` +- [x] Tracker `tracker.toml` defaults to MySQL +- [x] MySQL initialization directory documented +- [x] Docker Compose service dependencies updated +- [x] Local functionality testing passed +- [x] Local data persistence testing passed +- [x] VM integration testing passed +- [x] All documentation updated +- [x] Old SQLite configurations documented as legacy +- [ ] Final PR reviewed and approved + +## Rollback Plan + +If critical issues arise, the following steps can be taken to revert to SQLite: + +1. **Revert `compose.yaml`**: Remove the MySQL service and dependencies. +2. **Revert `.env.production`**: Restore SQLite environment variables. +3. **Revert `tracker.toml`**: Set the database driver back to `sqlite3`. +4. **Restart Services**: Run `docker compose up -d --force-recreate`. + +This ensures a quick rollback path if the MySQL integration causes unforeseen problems. diff --git a/docs/issues/README.md b/docs/issues/README.md new file mode 100644 index 0000000..3d47eb2 --- /dev/null +++ b/docs/issues/README.md @@ -0,0 +1,39 @@ +# Issues Documentation + +This directory contains detailed implementation plans and documentation for GitHub issues +related to the Torrust Tracker Demo project. + +## Structure + +Each implementation plan follows a consistent structure: + +- **Overview**: Clear objective and context +- **Implementation Steps**: Detailed step-by-step implementation guide +- **Testing Strategy**: Comprehensive testing approach +- **Implementation Order**: Phased approach for safe deployment +- **Risk Assessment**: Potential issues and mitigation strategies +- **Success Criteria**: Clear validation requirements + +## Current Issues + +- [Phase 1: MySQL Migration Implementation Plan](12-use-mysql-instead-of-sqlite-by-default.md) + - Database migration from SQLite to MySQL + - Part of the Hetzner migration initiative + +## Contributing + +When creating new issue implementation plans: + +1. Follow the established naming convention: `{issue-number}-{short-description}.md` +2. Use the structure from existing plans as a template +3. Ensure all markdown passes linting with `./scripts/lint.sh --markdown` +4. Link to parent issues and related documentation +5. Include comprehensive testing strategies +6. Document all file changes and configuration updates + +## Related Documentation + +- [Migration Plans](../plans/) - High-level migration strategies +- [Architecture Decision Records](../adr/) - Design decisions and rationale +- [Infrastructure Documentation](../../infrastructure/docs/) - Infrastructure setup guides +- [Application Documentation](../../application/docs/) - Application deployment guides diff --git a/infrastructure/terraform/main.tf b/infrastructure/terraform/main.tf index 8aee128..dbb083a 100644 --- a/infrastructure/terraform/main.tf +++ b/infrastructure/terraform/main.tf @@ -106,7 +106,12 @@ resource "libvirt_domain" "vm" { cloudinit = libvirt_cloudinit_disk.commoninit.id - + # CPU configuration - use a modern CPU model that supports x86-64-v2 + # Enable modern CPU model for x86-64-v2 instruction set support (required by MySQL 8.0) + # Reference: https://github.com/docker-library/mysql/issues/1055 + cpu { + mode = "host-model" + } disk { volume_id = libvirt_volume.vm_disk.id diff --git a/project-words.txt b/project-words.txt index 3202a2f..5914f3b 100644 --- a/project-words.txt +++ b/project-words.txt @@ -29,8 +29,10 @@ ethernets findtime fullchain genisoimage +healthcheck hetznercloud INFOHASH +initdb journalctl keyrings letsencrypt @@ -44,6 +46,7 @@ maxretry misprocess mkisofs mktemp +mysqladmin netdev newgrp newtrackon