Add optional DynamoDB backend for stats

README.md

@@ -23,7 +23,7 @@ RushTI transforms sequential TurboIntegrator execution into intelligent, paralle
 - **Self-Optimization** — EWMA-based learning reorders tasks from historical performance
 - **Checkpoint & Resume** — Automatic progress saving with failure recovery
 - **Exclusive Mode** — Prevents concurrent runs on shared TM1 servers
-- **SQLite Statistics** — Persistent execution history with dashboards and analysis
+- **Statistics Storage (SQLite or DynamoDB)** — Persistent execution history with dashboards and analysis
 - **TM1 Integration** — Read tasks from and write results to a TM1 cube
 - **100% Backwards Compatible** — Legacy TXT task files work without changes
 

config/settings.ini.template

@@ -196,23 +196,43 @@
 # auto_resume = false
 
 # ------------------------------------------------------------------------------
-# [stats] - SQLite stats database for execution history
+# [stats] - Execution history storage (SQLite or DynamoDB)
 # ------------------------------------------------------------------------------
 [stats]
 
 # Enable the stats database for storing execution history
-# The stats database stores execution statistics for:
+# Stats storage stores execution statistics for:
 # - Optimization features (EWMA runtime estimation)
 # - TM1 cube logging data source
 # - Historical analysis via 'rushti db' commands
 # Default: false
 # enabled = false
 
+# Storage backend: sqlite or dynamodb
+# Default: sqlite
+# backend = sqlite
+
 # Path to the SQLite database file
 # Relative paths are resolved from the application directory
 # Default: data/rushti_stats.db
 # db_path = data/rushti_stats.db
 
+# AWS region for DynamoDB backend (required when backend = dynamodb)
+# Example: eu-west-1
+# dynamodb_region = eu-west-1
+
+# DynamoDB runs table name (backend = dynamodb)
+# Default: rushti_runs
+# dynamodb_runs_table = rushti_runs
+
+# DynamoDB task results table name (backend = dynamodb)
+# Default: rushti_task_results
+# dynamodb_task_results_table = rushti_task_results
+
+# Optional custom DynamoDB endpoint URL (for LocalStack/testing)
+# Example: http://localhost:4566
+# dynamodb_endpoint_url =
+
 # Number of days to retain execution history
 # Valid range: 1-365
 # Default: 90

docs/advanced/settings-reference.md

@@ -184,12 +184,22 @@ Controls checkpoint saving and resume capability. When enabled, RushTI periodica
 
 ### [stats]
 
-Controls the SQLite statistics database that stores execution history. The stats database powers several features: EWMA optimization, the `rushti stats` commands, dashboard visualization, and historical analysis.
+Controls stats storage for execution history. RushTI supports two backends:
+- `sqlite` (default): local file storage
+- `dynamodb`: AWS DynamoDB tables
+
+Stats storage powers several features: EWMA optimization, the `rushti stats` commands, dashboard visualization, and historical analysis.
 
 | Setting | Type | Default | Description |
 |---------|------|---------|-------------|
-| `enabled` | bool | `false` | Enable the SQLite stats database. When enabled, every run records task-level execution data (timing, status, errors). |
-| `retention_days` | int | `90` | Days to keep execution history. Records older than this value are deleted at startup. Valid range: 1--365. Use `0` to keep data indefinitely. |
+| `enabled` | bool | `false` | Enable stats storage. When enabled, every run records task-level execution data (timing, status, errors). |
+| `backend` | str | `sqlite` | Storage backend: `sqlite` or `dynamodb`. |
+| `db_path` | str | `data/rushti_stats.db` | SQLite file path (used when `backend = sqlite`). |
+| `dynamodb_region` | str | `` | AWS region for DynamoDB (used when `backend = dynamodb`). |
+| `dynamodb_runs_table` | str | `rushti_runs` | DynamoDB table name for run-level records. |
+| `dynamodb_task_results_table` | str | `rushti_task_results` | DynamoDB table name for task-level records. |
+| `dynamodb_endpoint_url` | str | `` | Optional custom endpoint URL (for local testing tools such as LocalStack). |
+| `retention_days` | int | `90` | Days to keep execution history. Records older than this value are deleted at startup. Valid range: 1-365. Use `0` to keep data indefinitely. |
 
 **Required by:** `[optimization]`, `rushti stats` commands, `rushti stats visualize`
 
@@ -353,7 +363,7 @@ Copy this template to `config/settings.ini` and uncomment the settings you want
 # auto_resume = false
 
 # ------------------------------------------------------------------------------
-# [stats] - SQLite stats database for execution history
+# [stats] - Execution history storage (SQLite or DynamoDB)
 # ------------------------------------------------------------------------------
 [stats]
 
@@ -365,11 +375,31 @@ Copy this template to `config/settings.ini` and uncomment the settings you want
 # Default: false
 # enabled = false
 
+# Storage backend: sqlite or dynamodb
+# Default: sqlite
+# backend = sqlite
+
 # Path to the SQLite database file
 # Relative paths are resolved from the application directory
 # Default: data/rushti_stats.db
 # db_path = data/rushti_stats.db
 
+# AWS region for DynamoDB backend (required when backend = dynamodb)
+# Example: eu-west-1
+# dynamodb_region = eu-west-1
+
+# DynamoDB runs table name (backend = dynamodb)
+# Default: rushti_runs
+# dynamodb_runs_table = rushti_runs
+
+# DynamoDB task results table name (backend = dynamodb)
+# Default: rushti_task_results
+# dynamodb_task_results_table = rushti_task_results
+
+# Optional custom DynamoDB endpoint URL (for LocalStack/testing)
+# Example: http://localhost:4566
+# dynamodb_endpoint_url =
+
 # Number of days to retain execution history
 # Valid range: 1-365
 # Default: 90

pyproject.toml

@@ -38,6 +38,9 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
+dynamodb = [
+    "boto3>=1.34.0",
+]
 dev = [
     "pytest>=7.0.0",
     "pytest-asyncio",

src/rushti/cli.py

@@ -960,12 +960,17 @@ def main() -> int:
     results = list()
 
     # Initialize stats database if enabled
-    from rushti.stats import get_db_path
+    from rushti.stats import get_db_path, get_stats_backend
 
     db_kwargs = dict(
         enabled=settings.stats.enabled,
         retention_days=settings.stats.retention_days,
+        backend=get_stats_backend(settings),
         db_path=get_db_path(settings),
+        dynamodb_region=settings.stats.dynamodb_region or None,
+        dynamodb_runs_table=settings.stats.dynamodb_runs_table,
+        dynamodb_task_results_table=settings.stats.dynamodb_task_results_table,
+        dynamodb_endpoint_url=settings.stats.dynamodb_endpoint_url or None,
     )
     ctx = ExecutionContext(
         stats_db=create_stats_database(**db_kwargs),