From ee4f6cf47e0ec8a2b3f60ca7a687ef73e79b169e Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Wed, 14 Jan 2026 13:01:26 -0300 Subject: [PATCH 01/23] infra(aggregation_mode): deploy services with ansible --- .gitignore | 4 + Makefile | 186 +++++ infra/aggregation_mode/ansible/README.md | 659 ++++++++++++++++++ .../ansible/hoodi-inventory.yaml | 73 ++ .../ansible/mainnet-inventory.yaml | 73 ++ .../ansible/playbooks/deploy_all.yaml | 22 + .../ansible/playbooks/gateway.yaml | 143 ++++ .../ansible/playbooks/gateway_stack.yaml | 13 + .../ansible/playbooks/grafana_agg_mode.yaml | 109 +++ .../ansible/playbooks/ini/config-hoodi.ini | 88 +++ .../ansible/playbooks/ini/config-mainnet.ini | 91 +++ .../ansible/playbooks/metrics_stack.yaml | 13 + .../playbooks/pg_autofailover_common.yaml | 100 +++ .../ansible/playbooks/pg_monitor.yaml | 101 +++ .../ansible/playbooks/pg_node.yaml | 110 +++ .../ansible/playbooks/poller.yaml | 107 +++ .../ansible/playbooks/postgres_cluster.yaml | 24 + .../playbooks/postgres_migrations.yaml | 40 ++ .../playbooks/prometheus_agg_mode.yaml | 84 +++ .../ansible/playbooks/rust.yaml | 39 ++ .../ansible/playbooks/setup.yaml | 40 ++ .../config-agg-mode-gateway.yaml.j2 | 17 + .../config-agg-mode-poller.yaml.j2 | 11 + .../templates/grafana/grafana_env.j2 | 9 + .../prometheus/prometheus_agg_mode.yaml.j2 | 51 ++ .../templates/services/gateway.service.j2 | 18 + .../templates/services/poller.service.j2 | 15 + .../services/prometheus_agg_mode.service.j2 | 16 + .../templates/sudoers/gateway-service.j2 | 3 + 29 files changed, 2259 insertions(+) create mode 100644 infra/aggregation_mode/ansible/README.md create mode 100644 infra/aggregation_mode/ansible/hoodi-inventory.yaml create mode 100644 infra/aggregation_mode/ansible/mainnet-inventory.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/deploy_all.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/gateway.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/gateway_stack.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini create mode 100644 infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini create mode 100644 infra/aggregation_mode/ansible/playbooks/metrics_stack.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/pg_node.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/poller.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/rust.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/setup.yaml create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-gateway.yaml.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-poller.yaml.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/services/gateway.service.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/services/poller.service.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/services/prometheus_agg_mode.service.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/sudoers/gateway-service.j2 diff --git a/.gitignore b/.gitignore index 041927db8..a156fb7b1 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,7 @@ docs/dead_links_report.txt terraform.tfstate terraform.tfstate.backup + +# Aggregation Mode Ansible INI files (track config-*.ini templates, ignore others) +infra/aggregation_mode/ansible/playbooks/ini/*.ini +!infra/aggregation_mode/ansible/playbooks/ini/config-*.ini diff --git a/Makefile b/Makefile index 27dcb36e1..9dce05b8b 100644 --- a/Makefile +++ b/Makefile @@ -1658,3 +1658,189 @@ __NODE_EXPORTER_: install_node_exporter: @./scripts/install_node_exporter.sh + +# ============================================================================== +# Aggregation Mode Ansible Deployment +# ============================================================================== + +AGG_MODE_ANSIBLE_DIR = infra/aggregation_mode/ansible +AGG_MODE_PLAYBOOKS_DIR = $(AGG_MODE_ANSIBLE_DIR)/playbooks +AGG_MODE_INI_DIR = $(AGG_MODE_PLAYBOOKS_DIR)/ini + +# ------------------------------------------------------------------------------ +# Setup: Create INI configuration files +# ------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------ +# PostgreSQL Cluster Deployment +# ------------------------------------------------------------------------------ + +.PHONY: postgres_deploy +postgres_deploy: ## Deploy PostgreSQL Auto-Failover Cluster. Usage: make postgres_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/postgres_cluster.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "env=$(ENV)" + +.PHONY: postgres_monitor_deploy +postgres_monitor_deploy: ## Deploy PostgreSQL Monitor only. Usage: make postgres_monitor_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/pg_monitor.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=postgres_monitor" \ + -e "env=$(ENV)" + +.PHONY: postgres_nodes_deploy +postgres_nodes_deploy: ## Deploy PostgreSQL Primary & Secondary. Usage: make postgres_nodes_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/pg_node.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=postgres_primary" \ + -e "env=$(ENV)" + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/pg_node.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=postgres_secondary" \ + -e "env=$(ENV)" + +.PHONY: postgres_migrations +postgres_migrations: ## Run database migrations. Usage: make postgres_migrations ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/postgres_migrations.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=postgres_primary" \ + -e "env=$(ENV)" + +.PHONY: postgres_status +postgres_status: ## Check PostgreSQL cluster status. Usage: make postgres_status ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible postgres_monitor -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -m shell -a "sudo -u postgres pg_autoctl show state --monitor postgres://autoctl_node@localhost:5432/pg_auto_failover" --become + +# ------------------------------------------------------------------------------ +# Gateway & Poller Deployment +# ------------------------------------------------------------------------------ + +.PHONY: gateway_deploy +gateway_deploy: ## Deploy Gateway & Poller on both servers. Usage: make gateway_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=gateway_primary" \ + -e "env=$(ENV)" + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=gateway_secondary" \ + -e "env=$(ENV)" + +.PHONY: gateway_primary_deploy +gateway_primary_deploy: ## Deploy Gateway & Poller on primary only. Usage: make gateway_primary_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=gateway_primary" \ + -e "env=$(ENV)" + +.PHONY: gateway_secondary_deploy +gateway_secondary_deploy: ## Deploy Gateway & Poller on secondary only. Usage: make gateway_secondary_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=gateway_secondary" \ + -e "env=$(ENV)" + +# ------------------------------------------------------------------------------ +# Metrics Deployment +# ------------------------------------------------------------------------------ + +.PHONY: metrics_deploy +metrics_deploy: ## Deploy Prometheus & Grafana. Usage: make metrics_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/metrics_stack.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=metrics" \ + -e "env=$(ENV)" + +.PHONY: prometheus_deploy +prometheus_deploy: ## Deploy Prometheus only. Usage: make prometheus_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/prometheus_agg_mode.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=metrics" \ + -e "env=$(ENV)" + +.PHONY: grafana_deploy +grafana_deploy: ## Deploy Grafana only. Usage: make grafana_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/grafana_agg_mode.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=metrics" \ + -e "env=$(ENV)" + +# ------------------------------------------------------------------------------ +# Full Deployment +# ------------------------------------------------------------------------------ + +.PHONY: agg_mode_deploy_all +agg_mode_deploy_all: ## Deploy entire aggregation mode stack. Usage: make agg_mode_deploy_all ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/deploy_all.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "env=$(ENV)" + +# ------------------------------------------------------------------------------ +# Service Management +# ------------------------------------------------------------------------------ + +.PHONY: gateway_restart +gateway_restart: ## Restart gateway service. Usage: make gateway_restart ENV=hoodi HOST=gateway_primary + @if [ -z "$(ENV)" ] || [ -z "$(HOST)" ]; then \ + echo "Error: ENV and HOST must be set"; \ + exit 1; \ + fi + @ansible $(HOST) -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -m shell -a "sudo systemctl restart gateway" --become + +.PHONY: poller_restart +poller_restart: ## Restart poller service. Usage: make poller_restart ENV=hoodi HOST=gateway_primary + @if [ -z "$(ENV)" ] || [ -z "$(HOST)" ]; then \ + echo "Error: ENV and HOST must be set"; \ + exit 1; \ + fi + @ansible $(HOST) -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -m shell -a "systemctl --user restart poller" diff --git a/infra/aggregation_mode/ansible/README.md b/infra/aggregation_mode/ansible/README.md new file mode 100644 index 000000000..24b04bbac --- /dev/null +++ b/infra/aggregation_mode/ansible/README.md @@ -0,0 +1,659 @@ +# Aggregation Mode Ansible Automation + +This directory contains Ansible playbooks and configuration for automating the deployment and management of the Aligned Layer aggregation mode infrastructure. + +## Table of Contents + +- [Overview](#overview) +- [Architecture](#architecture) +- [Prerequisites](#prerequisites) +- [Initial Setup](#initial-setup) +- [Deployment](#deployment) +- [Service Management](#service-management) +- [Verification](#verification) +- [Troubleshooting](#troubleshooting) +- [Advanced Usage](#advanced-usage) + +## Overview + +The Ansible automation deploys a complete aggregation mode stack consisting of: + +1. **PostgreSQL Auto-Failover Cluster** (3 servers) + - 1 Monitor node (EC2) + - 2 Data nodes (Primary + Secondary) with automatic failover (Scaleway Elastic Metal) + - Password authentication with scram-sha-256 + +2. **Gateway Service** (2 servers) + - Rust-based gateway with TLS support + - Runs on port 8080 (non-TLS) and port 443 (TLS) + - Systemd service with automatic restart + +3. **Poller Service** (2 servers, colocated with gateway) + - Payment poller service + - User-level systemd service + +4. **Metrics Stack** (1 server) + - Prometheus for metrics collection + - Grafana for visualization + - 90-day retention + +## Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ Tailscale VPN │ +│ (100.64.0.0/10) │ +├─────────────────────────────────────────────────────────────────┤ +│ │ +│ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ +│ │ PG Monitor │ │ PG Primary │ │ PG Secondary │ │ +│ │ (EC2) │ │ (Scaleway) │ │ (Scaleway) │ │ +│ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ +│ │ │ │ │ +│ └──────────────────┴──────────────────┘ │ +│ pg_auto_failover │ +│ │ +│ ┌────────────────────────┐ ┌────────────────────────┐ │ +│ │ Gateway Primary │ │ Gateway Secondary │ │ +│ │ ├─ Gateway (8080+443)│ │ ├─ Gateway (8080+443) │ │ +│ │ └─ Poller │ │ └─ Poller │ │ +│ └────────────────────────┘ └────────────────────────┘ │ +│ │ +│ ┌────────────────────────┐ │ +│ │ Metrics Server │ │ +│ │ ├─ Prometheus (9090) │ │ +│ │ └─ Grafana (3000) │ │ +│ └────────────────────────┘ │ +│ │ +└─────────────────────────────────────────────────────────────────┘ +``` + +## Prerequisites + +### Local Machine + +1. **Ansible** (version 2.9 or higher) + ```bash + pip install ansible + ``` + +2. **SSH access** to all servers via Tailscale + - Ensure you're connected to the Tailscale VPN + - SSH keys configured for `admin` user on all servers + +3. **TLS Certificates** for Gateway + - Valid TLS certificate and key files + - Can be Let's Encrypt, CA-issued, or self-signed + +### Remote Servers + +All servers are provisioned via Terraform and connected via Tailscale VPN. They should have: +- Ubuntu/Debian-based OS +- `admin` user with sudo privileges +- `app` user for application services (gateway servers) +- `postgres` user will be created automatically for PostgreSQL services +- Tailscale VPN configured + +## Initial Setup + +All configuration is consolidated into environment-specific files with predefined values. You only need to fill in sensitive values (passwords, certificate paths). + +### 1. Configure Hoodi Environment + +Edit `playbooks/ini/config-hoodi.ini`: + +All non-sensitive values are already pre-filled. You only need to set: + +```ini +[DEFAULT] +# ... (all values pre-filled) ... + +# REQUIRED: Set a strong password before deploying +db_password=your_secure_password_here + +# REQUIRED: Same password for gateway/poller database access +gateway_db_password=your_secure_password_here + +# REQUIRED: Provide local paths to your TLS certificate files +tls_cert_source_path=/path/to/your/cert.pem +tls_key_source_path=/path/to/your/key.pem + +# REQUIRED: Same password for Grafana Postgres datasource +grafana_postgres_password=your_secure_password_here +``` + +**⚠️ CRITICAL**: All three password fields must be set to the same value before deploying! + +### 2. Configure Mainnet Environment (if needed) + +Edit `playbooks/ini/config-mainnet.ini`: + +Similar to Hoodi, fill in the required values: + +```ini +[DEFAULT] +# ... (most values pre-filled) ... + +# REQUIRED: Set passwords (same as above) +db_password=your_secure_password_here +gateway_db_password=your_secure_password_here +grafana_postgres_password=your_secure_password_here + +# REQUIRED: TLS certificate paths +tls_cert_source_path=/path/to/your/cert.pem +tls_key_source_path=/path/to/your/key.pem + +# TODO: Update these for mainnet deployment +gateway_payment_service_address=0xYourMainnetPaymentServiceAddress +gateway_eth_rpc_url=https://your-mainnet-rpc-url +grafana_rpc_url=https://your-mainnet-rpc-url +``` + +### Configuration File Structure + +The consolidated config files contain all settings organized by component: + +```ini +# config-hoodi.ini structure: +[DEFAULT] +environment=hoodi +git_branch=staging + +# PostgreSQL Configuration +postgres_monitor_hostname=agg-mode-hoodi-postgres-monitor +postgres_primary_hostname=agg-mode-hoodi-postgres-1 +postgres_secondary_hostname=agg-mode-hoodi-postgres-2 +db_name=agg_mode +db_user=autoctl_node +db_password= # ← FILL THIS IN + +# Gateway & Poller Configuration +gateway_network=Hoodi +gateway_payment_service_address=0x7222E0183cE1A96619d0c883e9bfc6b76D4e780e +gateway_eth_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net +gateway_db_password= # ← FILL THIS IN (same as db_password) +# ... other gateway settings ... + +# TLS Certificate Management +tls_cert_source_path= # ← FILL THIS IN +tls_key_source_path= # ← FILL THIS IN + +# Metrics Configuration +grafana_postgres_password= # ← FILL THIS IN (same as db_password) +# ... other metrics settings ... +``` + +The Ansible templates will automatically generate two separate database connection URLs for failover: +- `postgres://autoctl_node:password@agg-mode-hoodi-postgres-1:5432/agg_mode` +- `postgres://autoctl_node:password@agg-mode-hoodi-postgres-2:5432/agg_mode` + +The sqlx driver will try them in order for automatic failover + +## Deployment + +### Full Stack Deployment + +To deploy everything in one command: + +```bash +make agg_mode_deploy_all ENV=hoodi +``` + +This will: +1. Deploy PostgreSQL cluster (monitor, primary, secondary) +2. Run database migrations +3. Deploy gateway and poller on both servers +4. Deploy Prometheus and Grafana + +### Step-by-Step Deployment + +For more control, deploy each component separately: + +#### 1. Deploy PostgreSQL Cluster + +```bash +# Deploy complete postgres cluster with password authentication +make postgres_deploy ENV=hoodi +``` + +This will: +- Deploy monitor with scram-sha-256 auth +- Set password for autoctl_node user +- Deploy primary and secondary nodes +- Configure replication with password auth +- Run database migrations + +**Verify cluster status:** +```bash +make postgres_status ENV=hoodi +``` + +Expected output: +``` + Name | Node | Host:Port | TLI: LSN | Connection | Reported State | Assigned State +----------+-------+--------------------+----------------+--------------+---------------------+-------------------- +monitor | 1 | 100.x.x.x:5432 | | | | +node_1 | 2 | 100.x.x.x:5432 | 1: 0/... | read-write | primary | primary +node_2 | 3 | 100.x.x.x:5432 | 1: 0/... | read-only | secondary | secondary +``` + +#### 2. Deploy Gateway & Poller + +```bash +# Deploy on both servers +make gateway_deploy ENV=hoodi + +# Or deploy individually +make gateway_primary_deploy ENV=hoodi +make gateway_secondary_deploy ENV=hoodi +``` + +**Verify gateway is running:** +```bash +ssh app@agg-mode-hoodi-gateway-1 "sudo systemctl status gateway" +ssh app@agg-mode-hoodi-gateway-1 "systemctl --user status poller" +``` + +**Test endpoint:** +```bash +curl -k https://agg-mode-hoodi-gateway-1/health +``` + +#### 3. Deploy Metrics Stack + +```bash +# Deploy both Prometheus and Grafana +make metrics_deploy ENV=hoodi + +# Or deploy individually +make prometheus_deploy ENV=hoodi +make grafana_deploy ENV=hoodi +``` + +**Access dashboards:** +- Prometheus: `http://:9090` +- Grafana: `http://:3000` (default credentials: admin/admin) + +## Service Management + +### Restart Services + +**Gateway:** +```bash +make gateway_restart ENV=hoodi HOST=gateway_primary +make gateway_restart ENV=hoodi HOST=gateway_secondary +``` + +**Poller:** +```bash +make poller_restart ENV=hoodi HOST=gateway_primary +make poller_restart ENV=hoodi HOST=gateway_secondary +``` + +### Check Service Status + +**PostgreSQL Cluster:** +```bash +make postgres_status ENV=hoodi +``` + +**Gateway:** +```bash +ssh app@agg-mode-hoodi-gateway-1 "sudo systemctl status gateway" +ssh app@agg-mode-hoodi-gateway-1 "sudo journalctl -u gateway -n 50" +``` + +**Poller:** +```bash +ssh app@agg-mode-hoodi-gateway-1 "systemctl --user status poller" +ssh app@agg-mode-hoodi-gateway-1 "journalctl --user -u poller -n 50" +``` + +**Prometheus:** +```bash +ssh admin@agg-mode-hoodi-metrics "systemctl --user status prometheus" +``` + +**Grafana:** +```bash +ssh admin@agg-mode-hoodi-metrics "sudo systemctl status grafana-server" +``` + +### View Logs + +**Gateway:** +```bash +ssh app@agg-mode-hoodi-gateway-1 "sudo journalctl -u gateway -f" +``` + +**Poller:** +```bash +ssh app@agg-mode-hoodi-gateway-1 "journalctl --user -u poller -f" +``` + +**PostgreSQL:** +```bash +ssh admin@agg-mode-hoodi-postgres-1 "sudo journalctl -u pgautofailover -f" +``` + +## Verification + +### PostgreSQL Cluster Health + +1. **Check cluster state:** + ```bash + make postgres_status ENV=hoodi + ``` + +2. **Test password authentication:** + ```bash + ssh admin@agg-mode-hoodi-postgres-1 "PGPASSWORD='your_password' psql -U autoctl_node -h localhost -d agg_mode -c 'SELECT 1'" + ``` + +3. **Verify replication:** + ```bash + ssh admin@agg-mode-hoodi-postgres-1 "sudo -u postgres psql -d agg_mode -c 'SELECT * FROM pg_stat_replication'" + ``` + +4. **Test failover (optional):** + ```bash + # Stop primary + ssh admin@agg-mode-hoodi-postgres-1 "sudo systemctl stop pgautofailover" + + # Wait 30 seconds, check status + make postgres_status ENV=hoodi + # Secondary should now be primary + + # Restart original primary + ssh admin@agg-mode-hoodi-postgres-1 "sudo systemctl start pgautofailover" + ``` + +### Gateway Health + +1. **Check HTTP health endpoint:** + ```bash + curl -k https://agg-mode-hoodi-gateway-1/health + ``` + +2. **Check metrics:** + ```bash + curl http://agg-mode-hoodi-gateway-1:9094/metrics + ``` + +3. **Verify database connectivity:** + ```bash + ssh app@agg-mode-hoodi-gateway-1 + PGPASSWORD='your_password' psql -U autoctl_node -h agg-mode-hoodi-postgres-1 -d agg_mode -c "SELECT 1" + ``` + +### Poller Health + +1. **Check last processed block:** + ```bash + ssh app@agg-mode-hoodi-gateway-1 "cat ~/config/proof-aggregator.last_block_fetched.json" + ``` + + The block number should increase over time. + +2. **Check metrics:** + ```bash + curl http://agg-mode-hoodi-gateway-1:9095/metrics + ``` + +### Metrics Stack + +1. **Prometheus targets:** + - Navigate to `http://:9090/targets` + - All targets should show as "UP" + +2. **Grafana datasources:** + - Navigate to `http://:3000` + - Go to Configuration → Data Sources + - Verify Prometheus and PostgreSQL datasources are connected + +## Troubleshooting + +### PostgreSQL Issues + +**Problem: Node fails to join cluster** + +Check monitor logs: +```bash +ssh admin@agg-mode-hoodi-postgres-monitor "sudo journalctl -u pgautofailover -n 100" +``` + +Check node logs: +```bash +ssh admin@agg-mode-hoodi-postgres-1 "sudo journalctl -u pgautofailover -n 100" +``` + +**Problem: Password authentication fails** + +Verify password is set correctly in your environment config file (`config-hoodi.ini` or `config-mainnet.ini`). All three password fields must match: +- `db_password` +- `gateway_db_password` +- `grafana_postgres_password` + +Check pg_hba.conf: +```bash +ssh admin@agg-mode-hoodi-postgres-1 "sudo -u postgres cat /var/lib/postgresql/node/pg_hba.conf" +``` + +Should contain: +``` +host all all 100.64.0.0/10 scram-sha-256 +``` + +### Gateway Issues + +**Problem: Gateway won't start** + +Check logs for errors: +```bash +ssh app@agg-mode-hoodi-gateway-1 "sudo journalctl -u gateway -n 100" +``` + +Common issues: +- Missing TLS certificates → Check paths in `config-{{ env }}.ini` (tls_cert_source_path, tls_key_source_path) +- Database connection failed → Verify password in `config-{{ env }}.ini` (gateway_db_password) +- Port 443 already in use → Check with `sudo lsof -i :443` + +**Problem: TLS certificate errors** + +Verify certificates exist: +```bash +ssh app@agg-mode-hoodi-gateway-1 "ls -la ~/.ssl/" +``` + +Check certificate validity: +```bash +ssh app@agg-mode-hoodi-gateway-1 "openssl x509 -in ~/.ssl/cert.pem -text -noout" +``` + +### Poller Issues + +**Problem: Poller not syncing blocks** + +Check logs: +```bash +ssh app@agg-mode-hoodi-gateway-1 "journalctl --user -u poller -n 100" +``` + +Verify RPC connectivity: +```bash +ssh app@agg-mode-hoodi-gateway-1 "curl -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}' https://aligned-hoodi-rpc-geth.tail665ae.ts.net" +``` + +### Metrics Issues + +**Problem: Prometheus not scraping targets** + +Check Prometheus logs: +```bash +ssh admin@agg-mode-hoodi-metrics "journalctl --user -u prometheus -n 100" +``` + +Verify targets are reachable from metrics server: +```bash +ssh admin@agg-mode-hoodi-metrics "curl http://agg-mode-hoodi-gateway-1:9094/metrics" +``` + +Check Prometheus config: +```bash +ssh admin@agg-mode-hoodi-metrics "cat ~/config/prometheus.yaml" +``` + +### General Debugging + +**Check Tailscale connectivity:** +```bash +tailscale status +``` + +**Test SSH access to servers:** +```bash +ssh admin@agg-mode-hoodi-postgres-monitor "echo 'Connection successful'" +ssh app@agg-mode-hoodi-gateway-1 "echo 'Connection successful'" +``` + +**Verify Ansible inventory:** +```bash +ansible-inventory -i infra/aggregation_mode/ansible/hoodi-inventory.yaml --list +``` + +## Advanced Usage + +### Running Individual Playbooks + +You can run any playbook directly with ansible-playbook: + +```bash +# Deploy only postgres monitor +ansible-playbook infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml \ + -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ + -e "host=postgres_monitor" \ + -e "env=hoodi" + +# Deploy only gateway (no poller) +ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ + -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ + -e "host=gateway_primary" \ + -e "env=hoodi" +``` + +### Updating Services + +**Update gateway code:** +```bash +ssh app@agg-mode-hoodi-gateway-1 +cd ~/repos/gateway/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/gateway --bin gateway --features tls --locked +sudo systemctl restart gateway +``` + +**Update poller code:** +```bash +ssh app@agg-mode-hoodi-gateway-1 +cd ~/repos/poller/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/payments_poller --bin payments_poller --locked +systemctl --user restart poller +``` + +### Redeploy with Latest Code + +To redeploy with the latest code from git, simply run the deployment again: + +```bash +make gateway_deploy ENV=hoodi +``` + +The playbooks will: +1. Pull latest code from the configured branch +2. Rebuild the binaries +3. Restart the services + +### Changing Configuration + +1. Update INI files in `playbooks/ini/` +2. Redeploy the affected service: + ```bash + make gateway_deploy ENV=hoodi + # or + make postgres_deploy ENV=hoodi + ``` + +### Rotating Passwords + +1. Update all three password fields in your environment config file (`config-hoodi.ini` or `config-mainnet.ini`): + - `db_password` + - `gateway_db_password` + - `grafana_postgres_password` +2. Run password update on PostgreSQL: + ```bash + ssh admin@agg-mode-hoodi-postgres-monitor "sudo -u postgres psql -d pg_auto_failover -c \"ALTER USER autoctl_node PASSWORD 'new_password'\"" + ``` +3. Redeploy gateway and metrics: + ```bash + make gateway_deploy ENV=hoodi + make metrics_deploy ENV=hoodi + ``` + +## File Structure + +``` +infra/aggregation_mode/ansible/ +├── README.md # This file +├── hoodi-inventory.yaml # Hoodi environment inventory +├── mainnet-inventory.yaml # Mainnet environment inventory +└── playbooks/ + ├── ini/ # Configuration files + │ ├── config-hoodi.ini # Hoodi config (tracked, fill in passwords) + │ └── config-mainnet.ini # Mainnet config (tracked, fill in passwords) + ├── templates/ # Jinja2 templates + │ ├── config-files/ # Service config templates + │ ├── services/ # Systemd service templates + │ ├── sudoers/ # Sudoers templates + │ ├── prometheus/ # Prometheus config templates + │ └── grafana/ # Grafana config templates + ├── rust.yaml # Rust installation + ├── pg_autofailover_common.yaml # PostgreSQL + pg_auto_failover setup + ├── pg_monitor.yaml # PostgreSQL monitor deployment + ├── pg_node.yaml # PostgreSQL node deployment + ├── postgres_migrations.yaml # Database migrations + ├── gateway.yaml # Gateway deployment + ├── poller.yaml # Poller deployment + ├── prometheus_agg_mode.yaml # Prometheus deployment + ├── grafana_agg_mode.yaml # Grafana deployment + ├── postgres_cluster.yaml # Postgres orchestration + ├── gateway_stack.yaml # Gateway + poller orchestration + ├── metrics_stack.yaml # Metrics orchestration + └── deploy_all.yaml # Full stack orchestration +``` + +## Security Notes + +1. **Passwords**: Config files are tracked in git with empty password fields. Fill in passwords locally. Use `git update-index --assume-unchanged config-*.ini` after filling passwords to prevent accidentally committing them. + +2. **TLS Certificates**: Keep private keys secure. The playbooks set appropriate permissions (0600). + +3. **SSH Access**: All servers are only accessible via Tailscale VPN (100.64.0.0/10). + +4. **PostgreSQL**: Uses scram-sha-256 password authentication, not trust mode. + +5. **Firewall**: UFW is configured on all servers with deny-by-default policy. + +## Support + +For issues or questions: +- Check the [Troubleshooting](#troubleshooting) section +- Review logs on the affected server +- Contact the infrastructure team + +## References + +- [PostgreSQL Auto-Failover Documentation](https://pg-auto-failover.readthedocs.io/) +- [Ansible Documentation](https://docs.ansible.com/) +- [Prometheus Documentation](https://prometheus.io/docs/) +- [Grafana Documentation](https://grafana.com/docs/) diff --git a/infra/aggregation_mode/ansible/hoodi-inventory.yaml b/infra/aggregation_mode/ansible/hoodi-inventory.yaml new file mode 100644 index 000000000..4e2f1c91b --- /dev/null +++ b/infra/aggregation_mode/ansible/hoodi-inventory.yaml @@ -0,0 +1,73 @@ +# PostgreSQL Monitor +postgres_monitor: + hosts: + agg-mode-hoodi-postgres-monitor: + ansible_host: agg-mode-hoodi-postgres-monitor + admin_user: admin + ansible_user: admin + ansible_python_interpreter: /usr/bin/python3 + +# PostgreSQL Primary +postgres_primary: + hosts: + agg-mode-hoodi-postgres-1: + ansible_host: agg-mode-hoodi-postgres-1 + admin_user: admin + ansible_user: admin + ansible_python_interpreter: /usr/bin/python3 + +# PostgreSQL Secondary +postgres_secondary: + hosts: + agg-mode-hoodi-postgres-2: + ansible_host: agg-mode-hoodi-postgres-2 + admin_user: admin + ansible_user: admin + ansible_python_interpreter: /usr/bin/python3 + +# PostgreSQL Cluster (all postgres nodes) +postgres_cluster: + children: + postgres_monitor: + postgres_primary: + postgres_secondary: + +# Gateway Primary +gateway_primary: + hosts: + agg-mode-hoodi-gateway-1: + ansible_host: agg-mode-hoodi-gateway-1 + admin_user: admin + ansible_user: app + ansible_python_interpreter: /usr/bin/python3 + +# Gateway Secondary +gateway_secondary: + hosts: + agg-mode-hoodi-gateway-2: + ansible_host: agg-mode-hoodi-gateway-2 + admin_user: admin + ansible_user: app + ansible_python_interpreter: /usr/bin/python3 + +# Gateway Cluster (all gateway nodes) +gateway_cluster: + children: + gateway_primary: + gateway_secondary: + +# Metrics Server +metrics: + hosts: + agg-mode-hoodi-metrics: + ansible_host: agg-mode-hoodi-metrics + admin_user: admin + ansible_user: admin + ansible_python_interpreter: /usr/bin/python3 + +# All aggregation mode servers +aggregation_mode: + children: + postgres_cluster: + gateway_cluster: + metrics: diff --git a/infra/aggregation_mode/ansible/mainnet-inventory.yaml b/infra/aggregation_mode/ansible/mainnet-inventory.yaml new file mode 100644 index 000000000..489cc9d8d --- /dev/null +++ b/infra/aggregation_mode/ansible/mainnet-inventory.yaml @@ -0,0 +1,73 @@ +# PostgreSQL Monitor +postgres_monitor: + hosts: + agg-mode-mainnet-postgres-monitor: + ansible_host: agg-mode-mainnet-postgres-monitor + admin_user: admin + ansible_user: admin + ansible_python_interpreter: /usr/bin/python3 + +# PostgreSQL Primary +postgres_primary: + hosts: + agg-mode-mainnet-postgres-1: + ansible_host: agg-mode-mainnet-postgres-1 + admin_user: admin + ansible_user: admin + ansible_python_interpreter: /usr/bin/python3 + +# PostgreSQL Secondary +postgres_secondary: + hosts: + agg-mode-mainnet-postgres-2: + ansible_host: agg-mode-mainnet-postgres-2 + admin_user: admin + ansible_user: admin + ansible_python_interpreter: /usr/bin/python3 + +# PostgreSQL Cluster (all postgres nodes) +postgres_cluster: + children: + postgres_monitor: + postgres_primary: + postgres_secondary: + +# Gateway Primary +gateway_primary: + hosts: + agg-mode-mainnet-gateway-1: + ansible_host: agg-mode-mainnet-gateway-1 + admin_user: admin + ansible_user: app + ansible_python_interpreter: /usr/bin/python3 + +# Gateway Secondary +gateway_secondary: + hosts: + agg-mode-mainnet-gateway-2: + ansible_host: agg-mode-mainnet-gateway-2 + admin_user: admin + ansible_user: app + ansible_python_interpreter: /usr/bin/python3 + +# Gateway Cluster (all gateway nodes) +gateway_cluster: + children: + gateway_primary: + gateway_secondary: + +# Metrics Server +metrics: + hosts: + agg-mode-mainnet-metrics: + ansible_host: agg-mode-mainnet-metrics + admin_user: admin + ansible_user: admin + ansible_python_interpreter: /usr/bin/python3 + +# All aggregation mode servers +aggregation_mode: + children: + postgres_cluster: + gateway_cluster: + metrics: diff --git a/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml b/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml new file mode 100644 index 000000000..160cfe4f1 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml @@ -0,0 +1,22 @@ +- name: Deploy Full Aggregation Mode Stack + hosts: localhost + gather_facts: no + + tasks: + - name: Deploy PostgreSQL Cluster + ansible.builtin.import_playbook: postgres_cluster.yaml + + - name: Deploy Gateway and Poller on Primary + ansible.builtin.import_playbook: gateway_stack.yaml + vars: + host: gateway_primary + + - name: Deploy Gateway and Poller on Secondary + ansible.builtin.import_playbook: gateway_stack.yaml + vars: + host: gateway_secondary + + - name: Deploy Metrics Stack + ansible.builtin.import_playbook: metrics_stack.yaml + vars: + host: metrics diff --git a/infra/aggregation_mode/ansible/playbooks/gateway.yaml b/infra/aggregation_mode/ansible/playbooks/gateway.yaml new file mode 100644 index 000000000..04d257a23 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/gateway.yaml @@ -0,0 +1,143 @@ +- name: Gateway Setup + hosts: "{{ host }}" + + tasks: + - name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + + - name: Import rust playbook + ansible.builtin.import_playbook: rust.yaml + vars: + host: "{{ host }}" + +- hosts: "{{ host }}" + vars: + git_branch: "{{ lookup('ini', 'git_branch', file='ini/config-{{ env }}.ini') }}" + gateway_port: "{{ lookup('ini', 'gateway_port', file='ini/config-{{ env }}.ini', default='8080') }}" + gateway_tls_enabled: "{{ lookup('ini', 'gateway_tls_enabled', file='ini/config-{{ env }}.ini', default='true') }}" + gateway_tls_port: "{{ lookup('ini', 'gateway_tls_port', file='ini/config-{{ env }}.ini', default='443') }}" + gateway_tls_cert_path: "{{ lookup('ini', 'gateway_tls_cert_path', file='ini/config-{{ env }}.ini') }}" + gateway_tls_key_path: "{{ lookup('ini', 'gateway_tls_key_path', file='ini/config-{{ env }}.ini') }}" + gateway_db_user: "{{ lookup('ini', 'gateway_db_user', file='ini/config-{{ env }}.ini') }}" + gateway_db_password: "{{ lookup('ini', 'gateway_db_password', file='ini/config-{{ env }}.ini') }}" + gateway_db_name: "{{ lookup('ini', 'gateway_db_name', file='ini/config-{{ env }}.ini') }}" + gateway_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file='ini/config-{{ env }}.ini') }}" + gateway_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file='ini/config-{{ env }}.ini') }}" + gateway_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file='ini/config-{{ env }}.ini', default='5432') }}" + gateway_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file='ini/config-{{ env }}.ini') }}" + gateway_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file='ini/config-{{ env }}.ini') }}" + gateway_network: "{{ lookup('ini', 'gateway_network', file='ini/config-{{ env }}.ini') }}" + gateway_max_daily_proofs: "{{ lookup('ini', 'gateway_max_daily_proofs', file='ini/config-{{ env }}.ini', default='100') }}" + gateway_last_block_fetched_filepath: "{{ lookup('ini', 'gateway_last_block_fetched_filepath', file='ini/config-{{ env }}.ini') }}" + gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file='ini/config-{{ env }}.ini', default='9094') }}" + poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file='ini/config-{{ env }}.ini', default='9095') }}" + tls_cert_source_path: "{{ lookup('ini', 'tls_cert_source_path', file='ini/config-{{ env }}.ini') }}" + tls_key_source_path: "{{ lookup('ini', 'tls_key_source_path', file='ini/config-{{ env }}.ini') }}" + + tasks: + - name: Install required system packages + become: true + apt: + pkg: + - pkg-config + - libssl-dev + - build-essential + state: latest + update_cache: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Create SSL directory + file: + path: /home/{{ ansible_user }}/.ssl + state: directory + mode: '0700' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Copy TLS certificate + copy: + src: "{{ tls_cert_source_path }}" + dest: "{{ gateway_tls_cert_path }}" + mode: '0600' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Copy TLS key + copy: + src: "{{ tls_key_source_path }}" + dest: "{{ gateway_tls_key_path }}" + mode: '0600' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + no_log: true + + - name: Clone aligned_layer repository + git: + repo: https://github.com/yetanotherco/aligned_layer.git + dest: /home/{{ ansible_user }}/repos/gateway/aligned_layer + version: "{{ git_branch }}" + update: yes + + - name: Build gateway with TLS + shell: | + export PATH=$HOME/.cargo/bin:$PATH + cargo install --path /home/{{ ansible_user }}/repos/gateway/aligned_layer/aggregation_mode/gateway --bin gateway --features tls --locked + args: + creates: /home/{{ ansible_user }}/.cargo/bin/gateway + + - name: Create config directory + file: + path: /home/{{ ansible_user }}/config + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Template gateway config file + template: + src: config-files/config-agg-mode-gateway.yaml.j2 + dest: /home/{{ ansible_user }}/config/config-agg-mode-gateway.yaml + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Template gateway systemd service + become: true + template: + src: services/gateway.service.j2 + dest: /etc/systemd/system/gateway.service + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Allow port 443 (TLS) through UFW + become: true + ufw: + rule: allow + port: '443' + proto: tcp + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Create sudoers file for gateway service management + become: true + template: + src: sudoers/gateway-service.j2 + dest: /etc/sudoers.d/gateway-service + mode: '0440' + validate: 'visudo -cf %s' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Enable and start gateway service + become: true + systemd_service: + name: gateway + state: started + enabled: true + daemon_reload: true + vars: + ansible_ssh_user: "{{ admin_user }}" diff --git a/infra/aggregation_mode/ansible/playbooks/gateway_stack.yaml b/infra/aggregation_mode/ansible/playbooks/gateway_stack.yaml new file mode 100644 index 000000000..eceba3b91 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/gateway_stack.yaml @@ -0,0 +1,13 @@ +- name: Deploy Gateway and Poller Stack + hosts: "{{ host }}" + + tasks: + - name: Deploy Gateway + ansible.builtin.import_playbook: gateway.yaml + vars: + host: "{{ host }}" + + - name: Deploy Poller + ansible.builtin.import_playbook: poller.yaml + vars: + host: "{{ host }}" diff --git a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml new file mode 100644 index 000000000..388d53ab5 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml @@ -0,0 +1,109 @@ +- name: Grafana Setup for Aggregation Mode + hosts: "{{ host }}" + + tasks: + - name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + +- hosts: "{{ host }}" + vars: + grafana_prometheus_url: "{{ lookup('ini', 'grafana_prometheus_url', file='ini/config-{{ env }}.ini') }}" + grafana_rpc_url: "{{ lookup('ini', 'grafana_rpc_url', file='ini/config-{{ env }}.ini') }}" + grafana_postgres_host: "{{ lookup('ini', 'grafana_postgres_host', file='ini/config-{{ env }}.ini') }}" + grafana_postgres_port: "{{ lookup('ini', 'grafana_postgres_port', file='ini/config-{{ env }}.ini', default='5432') }}" + grafana_postgres_db: "{{ lookup('ini', 'grafana_postgres_db', file='ini/config-{{ env }}.ini') }}" + grafana_postgres_user: "{{ lookup('ini', 'grafana_postgres_user', file='ini/config-{{ env }}.ini') }}" + grafana_postgres_password: "{{ lookup('ini', 'grafana_postgres_password', file='ini/config-{{ env }}.ini') }}" + + tasks: + - name: Install required packages + become: true + apt: + pkg: + - apt-transport-https + - software-properties-common + - wget + - gnupg + state: latest + update_cache: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Create apt keyrings directory + become: true + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Download Grafana GPG key + become: true + get_url: + url: https://apt.grafana.com/gpg.key + dest: /tmp/grafana.key + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Add Grafana GPG key + become: true + shell: | + gpg --dearmor < /tmp/grafana.key > /etc/apt/keyrings/grafana.gpg + args: + creates: /etc/apt/keyrings/grafana.gpg + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Add Grafana APT repository + become: true + lineinfile: + path: /etc/apt/sources.list.d/grafana.list + line: "deb [signed-by=/etc/apt/keyrings/grafana.gpg] https://apt.grafana.com stable main" + create: yes + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Update apt cache + become: true + apt: + update_cache: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Install Grafana + become: true + apt: + name: grafana + state: latest + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Template Grafana environment configuration + become: true + template: + src: grafana/grafana_env.j2 + dest: /etc/default/grafana-server-custom + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Append custom environment to grafana-server defaults + become: true + shell: | + cat /etc/default/grafana-server-custom >> /etc/default/grafana-server + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Enable and start Grafana service + become: true + systemd_service: + name: grafana-server + state: restarted + enabled: true + daemon_reload: true + vars: + ansible_ssh_user: "{{ admin_user }}" diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini new file mode 100644 index 000000000..d40277519 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -0,0 +1,88 @@ +[DEFAULT] +# ============================================ +# Hoodi Environment Configuration +# ============================================ +# This file contains all configuration for the Hoodi environment. +# Only sensitive values (passwords, certificate paths) need to be filled in. + +# Environment +environment=hoodi +git_branch=staging + +# ============================================ +# PostgreSQL Configuration +# ============================================ +postgres_monitor_hostname=agg-mode-hoodi-postgres-monitor +postgres_primary_hostname=agg-mode-hoodi-postgres-1 +postgres_secondary_hostname=agg-mode-hoodi-postgres-2 +db_name=agg_mode +db_user=autoctl_node +# REQUIRED: Set this password before deploying postgres +db_password= + +# PostgreSQL Monitor +monitor_pgdata=/var/lib/postgresql/monitor +monitor_port=5432 + +# PostgreSQL Nodes +node_pgdata=/var/lib/postgresql/node +node_port=5432 +backup_dir=/var/lib/backup + +# ============================================ +# Gateway & Poller Configuration +# ============================================ +gateway_network=hoodi +gateway_max_daily_proofs=100 +gateway_payment_service_address=0x7222E0183cE1A96619d0c883e9bfc6b76D4e780e +gateway_eth_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net + +# Database connection (uses same credentials as postgres) +gateway_db_user=autoctl_node +# REQUIRED: Set to same password as db_password +gateway_db_password= +gateway_db_name=agg_mode +gateway_postgres_primary=agg-mode-hoodi-postgres-1 +gateway_postgres_secondary=agg-mode-hoodi-postgres-2 +gateway_postgres_port=5432 + +# Metrics ports +gateway_metrics_port=9094 +poller_metrics_port=9095 + +# Gateway Service Settings (same for all gateways) +gateway_port=8080 +gateway_tls_enabled=true +gateway_tls_port=443 +gateway_tls_cert_path=/home/app/.ssl/cert.pem +gateway_tls_key_path=/home/app/.ssl/key.pem +gateway_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json + +# Poller Service Settings (same for all pollers) +poller_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json +last_block_fetched_initial_value=0 + +# ============================================ +# TLS Certificate Management +# ============================================ +# REQUIRED: Provide paths to existing certificates on your local machine +# These will be copied to the gateway servers +tls_cert_source_path= +tls_key_source_path= + +# ============================================ +# Metrics Configuration +# ============================================ +prometheus_version=3.6.0 +gateway_primary_hostname=agg-mode-hoodi-gateway-1 +gateway_secondary_hostname=agg-mode-hoodi-gateway-2 + +# Grafana Configuration +grafana_prometheus_url=http://localhost:9090 +grafana_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net +grafana_postgres_host=agg-mode-hoodi-postgres-1 +grafana_postgres_port=5432 +grafana_postgres_db=agg_mode +grafana_postgres_user=autoctl_node +# REQUIRED: Set to same password as db_password +grafana_postgres_password= diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini new file mode 100644 index 000000000..70d25c24f --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -0,0 +1,91 @@ +[DEFAULT] +# ============================================ +# Mainnet Environment Configuration +# ============================================ +# This file contains all configuration for the Mainnet environment. +# Only sensitive values (passwords, certificate paths) need to be filled in. + +# Environment +environment=mainnet +git_branch=staging + +# ============================================ +# PostgreSQL Configuration +# ============================================ +postgres_monitor_hostname=agg-mode-mainnet-postgres-monitor +postgres_primary_hostname=agg-mode-mainnet-postgres-1 +postgres_secondary_hostname=agg-mode-mainnet-postgres-2 +db_name=agg_mode +db_user=autoctl_node +# REQUIRED: Set this password before deploying postgres +db_password= + +# PostgreSQL Monitor +monitor_pgdata=/var/lib/postgresql/monitor +monitor_port=5432 + +# PostgreSQL Nodes +node_pgdata=/var/lib/postgresql/node +node_port=5432 +backup_dir=/var/lib/backup + +# ============================================ +# Gateway & Poller Configuration +# ============================================ +gateway_network=mainnet +gateway_max_daily_proofs=100 +# TODO: Update with mainnet payment service address +gateway_payment_service_address= +# TODO: Update with mainnet RPC URL +gateway_eth_rpc_url= + +# Database connection (uses same credentials as postgres) +gateway_db_user=autoctl_node +# REQUIRED: Set to same password as db_password +gateway_db_password= +gateway_db_name=agg_mode +gateway_postgres_primary=agg-mode-mainnet-postgres-1 +gateway_postgres_secondary=agg-mode-mainnet-postgres-2 +gateway_postgres_port=5432 + +# Metrics ports +gateway_metrics_port=9094 +poller_metrics_port=9095 + +# Gateway Service Settings (same for all gateways) +gateway_port=8080 +gateway_tls_enabled=true +gateway_tls_port=443 +gateway_tls_cert_path=/home/app/.ssl/cert.pem +gateway_tls_key_path=/home/app/.ssl/key.pem +gateway_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json + +# Poller Service Settings (same for all pollers) +poller_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json +last_block_fetched_initial_value=0 + +# ============================================ +# TLS Certificate Management +# ============================================ +# REQUIRED: Provide paths to existing certificates on your local machine +# These will be copied to the gateway servers +tls_cert_source_path= +tls_key_source_path= + +# ============================================ +# Metrics Configuration +# ============================================ +prometheus_version=3.6.0 +gateway_primary_hostname=agg-mode-mainnet-gateway-1 +gateway_secondary_hostname=agg-mode-mainnet-gateway-2 + +# Grafana Configuration +grafana_prometheus_url=http://localhost:9090 +# TODO: Update with mainnet RPC URL +grafana_rpc_url= +grafana_postgres_host=agg-mode-mainnet-postgres-1 +grafana_postgres_port=5432 +grafana_postgres_db=agg_mode +grafana_postgres_user=autoctl_node +# REQUIRED: Set to same password as db_password +grafana_postgres_password= diff --git a/infra/aggregation_mode/ansible/playbooks/metrics_stack.yaml b/infra/aggregation_mode/ansible/playbooks/metrics_stack.yaml new file mode 100644 index 000000000..7303a0320 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/metrics_stack.yaml @@ -0,0 +1,13 @@ +- name: Deploy Metrics Stack + hosts: "{{ host }}" + + tasks: + - name: Deploy Prometheus + ansible.builtin.import_playbook: prometheus_agg_mode.yaml + vars: + host: "{{ host }}" + + - name: Deploy Grafana + ansible.builtin.import_playbook: grafana_agg_mode.yaml + vars: + host: "{{ host }}" diff --git a/infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml b/infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml new file mode 100644 index 000000000..655637704 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml @@ -0,0 +1,100 @@ +- name: PostgreSQL Auto-Failover Common Setup + hosts: "{{ host }}" + + tasks: + - name: Install postgresql-common and ca-certificates + become: true + apt: + pkg: + - postgresql-common + - ca-certificates + - curl + state: latest + update_cache: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Download PostgreSQL APT repository setup script + become: true + get_url: + url: https://www.postgresql.org/media/keys/ACCC4CF8.asc + dest: /tmp/postgresql.asc + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Add PostgreSQL APT key + become: true + shell: | + cat /tmp/postgresql.asc | gpg --dearmor | tee /etc/apt/keyrings/postgresql.gpg > /dev/null + args: + creates: /etc/apt/keyrings/postgresql.gpg + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Add PostgreSQL APT repository + become: true + lineinfile: + path: /etc/apt/sources.list.d/pgdg.list + line: "deb [signed-by=/etc/apt/keyrings/postgresql.gpg] https://apt.postgresql.org/pub/repos/apt bookworm-pgdg main" + create: yes + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Set create_main_cluster to false + become: true + lineinfile: + path: /etc/postgresql-common/createcluster.conf + line: "create_main_cluster = false" + create: yes + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Update apt cache + become: true + apt: + update_cache: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Install postgresql-16 and pg-auto-failover-cli + become: true + apt: + pkg: + - postgresql-16 + - pg-auto-failover-cli + state: latest + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Stop and disable default PostgreSQL service + become: true + systemd_service: + name: "{{ item }}" + state: stopped + enabled: false + with_items: + - postgresql + - postgresql@16-main + ignore_errors: yes + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Remove default PostgreSQL data directory + become: true + file: + path: /var/lib/postgresql + state: absent + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Create PostgreSQL data directory + become: true + file: + path: /var/lib/postgresql + state: directory + owner: postgres + group: postgres + mode: '0700' + vars: + ansible_ssh_user: "{{ admin_user }}" diff --git a/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml b/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml new file mode 100644 index 000000000..36c19f039 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml @@ -0,0 +1,101 @@ +- name: PostgreSQL Monitor Setup + hosts: "{{ host }}" + + tasks: + - name: Import pg_autofailover_common playbook + ansible.builtin.import_playbook: pg_autofailover_common.yaml + vars: + host: "{{ host }}" + +- hosts: "{{ host }}" + vars: + db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}" + monitor_pgdata: "{{ lookup('ini', 'monitor_pgdata', file='ini/config-{{ env }}.ini', default='/var/lib/postgresql/monitor') }}" + monitor_port: "{{ lookup('ini', 'monitor_port', file='ini/config-{{ env }}.ini', default='5432') }}" + + tasks: + - name: Install postgresql-16-auto-failover + become: true + apt: + pkg: + - postgresql-16-auto-failover + state: latest + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Check if monitor is already initialized + stat: + path: "{{ monitor_pgdata }}/PG_VERSION" + register: monitor_initialized + become: true + become_user: postgres + + - name: Get Tailscale IP + shell: tailscale ip --4 + register: tailscale_ip + when: not monitor_initialized.stat.exists + + - name: Create pg_autoctl monitor + become: true + become_user: postgres + shell: | + pg_autoctl create monitor \ + --pgdata {{ monitor_pgdata }} \ + --pgctl /usr/lib/postgresql/16/bin/pg_ctl \ + --auth scram-sha-256 \ + --ssl-self-signed \ + --pgport {{ monitor_port }} \ + --hostname {{ tailscale_ip.stdout }} + when: not monitor_initialized.stat.exists + args: + creates: "{{ monitor_pgdata }}/PG_VERSION" + + - name: Wait for monitor to be ready + wait_for: + port: "{{ monitor_port }}" + delay: 5 + timeout: 60 + + - name: Set password for autoctl_node user + become: true + become_user: postgres + shell: | + psql -d pg_auto_failover -c "ALTER USER autoctl_node PASSWORD '{{ db_password }}';" + no_log: true + + - name: Update pg_hba.conf for Tailscale network + become: true + become_user: postgres + blockinfile: + path: "{{ monitor_pgdata }}/pg_hba.conf" + marker: "# {mark} ANSIBLE MANAGED BLOCK - Tailscale" + block: | + # Tailscale network connections + host all all 100.64.0.0/10 scram-sha-256 + host replication all 100.64.0.0/10 scram-sha-256 + + - name: Generate systemd service file + become: true + become_user: postgres + shell: | + pg_autoctl show systemd --pgdata {{ monitor_pgdata }} + register: systemd_service_content + + - name: Create pgautofailover systemd service + become: true + copy: + content: "{{ systemd_service_content.stdout }}" + dest: /etc/systemd/system/pgautofailover.service + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Enable and start pgautofailover service + become: true + systemd_service: + name: pgautofailover + state: started + enabled: true + daemon_reload: true + vars: + ansible_ssh_user: "{{ admin_user }}" diff --git a/infra/aggregation_mode/ansible/playbooks/pg_node.yaml b/infra/aggregation_mode/ansible/playbooks/pg_node.yaml new file mode 100644 index 000000000..b271d4d10 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/pg_node.yaml @@ -0,0 +1,110 @@ +- name: PostgreSQL Node Setup + hosts: "{{ host }}" + + tasks: + - name: Import pg_autofailover_common playbook + ansible.builtin.import_playbook: pg_autofailover_common.yaml + vars: + host: "{{ host }}" + +- hosts: "{{ host }}" + vars: + db_name: "{{ lookup('ini', 'db_name', file='ini/config-{{ env }}.ini') }}" + db_user: "{{ lookup('ini', 'db_user', file='ini/config-{{ env }}.ini') }}" + db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}" + postgres_monitor_hostname: "{{ lookup('ini', 'postgres_monitor_hostname', file='ini/config-{{ env }}.ini') }}" + node_pgdata: "{{ lookup('ini', 'node_pgdata', file='ini/config-{{ env }}.ini', default='/var/lib/postgresql/node') }}" + node_port: "{{ lookup('ini', 'node_port', file='ini/config-{{ env }}.ini', default='5432') }}" + backup_dir: "{{ lookup('ini', 'backup_dir', file='ini/config-{{ env }}.ini', default='/var/lib/backup') }}" + + tasks: + - name: Create backup directory + become: true + file: + path: "{{ backup_dir }}" + state: directory + owner: postgres + group: postgres + mode: '0700' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Check if node is already initialized + stat: + path: "{{ node_pgdata }}/PG_VERSION" + register: node_initialized + become: true + become_user: postgres + + - name: Get Tailscale IP + shell: tailscale ip --4 + register: tailscale_ip + when: not node_initialized.stat.exists + + - name: Create pg_autoctl postgres node + become: true + become_user: postgres + shell: | + PGPASSWORD='{{ db_password }}' pg_autoctl create postgres \ + --pgdata {{ node_pgdata }} \ + --pgctl /usr/lib/postgresql/16/bin/pg_ctl \ + --auth scram-sha-256 \ + --ssl-self-signed \ + --username {{ db_user }} \ + --dbname {{ db_name }} \ + --pgport {{ node_port }} \ + --hostname {{ tailscale_ip.stdout }} \ + --monitor 'postgres://{{ db_user }}:{{ db_password }}@{{ postgres_monitor_hostname }}:5432/pg_auto_failover?sslmode=require' + when: not node_initialized.stat.exists + no_log: true + args: + creates: "{{ node_pgdata }}/PG_VERSION" + + - name: Set replication password + become: true + become_user: postgres + shell: | + pg_autoctl config set replication.password '{{ db_password }}' --pgdata {{ node_pgdata }} + no_log: true + + - name: Update pg_hba.conf for Tailscale network + become: true + become_user: postgres + blockinfile: + path: "{{ node_pgdata }}/pg_hba.conf" + marker: "# {mark} ANSIBLE MANAGED BLOCK - Tailscale" + block: | + # Tailscale network connections + host all all 100.64.0.0/10 scram-sha-256 + + - name: Generate systemd service file + become: true + become_user: postgres + shell: | + pg_autoctl show systemd --pgdata {{ node_pgdata }} + register: systemd_service_content + + - name: Create pgautofailover systemd service + become: true + copy: + content: "{{ systemd_service_content.stdout }}" + dest: /etc/systemd/system/pgautofailover.service + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Enable and start pgautofailover service + become: true + systemd_service: + name: pgautofailover + state: started + enabled: true + daemon_reload: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Wait for node to join cluster + wait_for: + port: "{{ node_port }}" + delay: 5 + timeout: 60 diff --git a/infra/aggregation_mode/ansible/playbooks/poller.yaml b/infra/aggregation_mode/ansible/playbooks/poller.yaml new file mode 100644 index 000000000..7c1de48c0 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/poller.yaml @@ -0,0 +1,107 @@ +- name: Poller Setup + hosts: "{{ host }}" + + tasks: + - name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + + - name: Import rust playbook + ansible.builtin.import_playbook: rust.yaml + vars: + host: "{{ host }}" + +- hosts: "{{ host }}" + vars: + git_branch: "{{ lookup('ini', 'git_branch', file='ini/config-{{ env }}.ini') }}" + poller_db_user: "{{ lookup('ini', 'gateway_db_user', file='ini/config-{{ env }}.ini') }}" + poller_db_password: "{{ lookup('ini', 'gateway_db_password', file='ini/config-{{ env }}.ini') }}" + poller_db_name: "{{ lookup('ini', 'gateway_db_name', file='ini/config-{{ env }}.ini') }}" + poller_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file='ini/config-{{ env }}.ini') }}" + poller_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file='ini/config-{{ env }}.ini') }}" + poller_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file='ini/config-{{ env }}.ini', default='5432') }}" + poller_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file='ini/config-{{ env }}.ini') }}" + poller_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file='ini/config-{{ env }}.ini') }}" + poller_network: "{{ lookup('ini', 'gateway_network', file='ini/config-{{ env }}.ini') }}" + poller_max_daily_proofs: "{{ lookup('ini', 'gateway_max_daily_proofs', file='ini/config-{{ env }}.ini', default='100') }}" + poller_last_block_fetched_filepath: "{{ lookup('ini', 'poller_last_block_fetched_filepath', file='ini/config-{{ env }}.ini') }}" + last_block_fetched_initial_value: "{{ lookup('ini', 'last_block_fetched_initial_value', file='ini/config-{{ env }}.ini', default='0') }}" + gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file='ini/config-{{ env }}.ini', default='9094') }}" + poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file='ini/config-{{ env }}.ini', default='9095') }}" + + tasks: + - name: Install required system packages + become: true + apt: + pkg: + - pkg-config + - libssl-dev + - build-essential + state: latest + update_cache: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Clone aligned_layer repository + git: + repo: https://github.com/yetanotherco/aligned_layer.git + dest: /home/{{ ansible_user }}/repos/poller/aligned_layer + version: "{{ git_branch }}" + update: yes + + - name: Build poller + shell: | + export PATH=$HOME/.cargo/bin:$PATH + cargo install --path /home/{{ ansible_user }}/repos/poller/aligned_layer/aggregation_mode/payments_poller --bin payments_poller --locked + args: + creates: /home/{{ ansible_user }}/.cargo/bin/payments_poller + + - name: Create config directory + file: + path: /home/{{ ansible_user }}/config + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Create last_block_fetched file + copy: + content: '{"last_block_fetched":{{ last_block_fetched_initial_value }}}' + dest: "{{ poller_last_block_fetched_filepath }}" + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + force: no + + - name: Template poller config file + template: + src: config-files/config-agg-mode-poller.yaml.j2 + dest: /home/{{ ansible_user }}/config/config-agg-mode-poller.yaml + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Create systemd user directory + file: + path: /home/{{ ansible_user }}/.config/systemd/user + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Template poller systemd service + template: + src: services/poller.service.j2 + dest: /home/{{ ansible_user }}/.config/systemd/user/poller.service + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Enable and start poller service + systemd_service: + name: poller + state: started + enabled: true + scope: user + daemon_reload: true diff --git a/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml b/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml new file mode 100644 index 000000000..24b033d5e --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml @@ -0,0 +1,24 @@ +- name: Deploy PostgreSQL Auto-Failover Cluster + hosts: localhost + gather_facts: no + + tasks: + - name: Deploy PostgreSQL Monitor + ansible.builtin.import_playbook: pg_monitor.yaml + vars: + host: postgres_monitor + + - name: Deploy PostgreSQL Primary Node + ansible.builtin.import_playbook: pg_node.yaml + vars: + host: postgres_primary + + - name: Deploy PostgreSQL Secondary Node + ansible.builtin.import_playbook: pg_node.yaml + vars: + host: postgres_secondary + + - name: Run Database Migrations + ansible.builtin.import_playbook: postgres_migrations.yaml + vars: + host: postgres_primary diff --git a/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml b/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml new file mode 100644 index 000000000..a6c526009 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml @@ -0,0 +1,40 @@ +- name: PostgreSQL Migrations Setup + hosts: "{{ host }}" + + tasks: + - name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + + - name: Import rust playbook + ansible.builtin.import_playbook: rust.yaml + vars: + host: "{{ host }}" + +- hosts: "{{ host }}" + vars: + db_name: "{{ lookup('ini', 'db_name', file='ini/config-{{ env }}.ini') }}" + db_user: "{{ lookup('ini', 'db_user', file='ini/config-{{ env }}.ini') }}" + db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}" + postgres_primary_hostname: "{{ lookup('ini', 'postgres_primary_hostname', file='ini/config-{{ env }}.ini') }}" + git_branch: "{{ lookup('ini', 'git_branch', file='ini/config-{{ env }}.ini') }}" + + tasks: + - name: Clone aligned_layer repository + git: + repo: https://github.com/yetanotherco/aligned_layer.git + dest: /home/{{ ansible_user }}/repos/migrations/aligned_layer + version: "{{ git_branch }}" + update: yes + + - name: Run database migrations + shell: | + export PATH=$HOME/.cargo/bin:$PATH + cargo run --manifest-path /home/{{ ansible_user }}/repos/migrations/aligned_layer/aggregation_mode/Cargo.toml --release --bin migrate -- "postgres://{{ db_user }}:{{ db_password }}@{{ postgres_primary_hostname }}:5432/{{ db_name }}" + register: migration_result + no_log: true + + - name: Display migration result + debug: + msg: "{{ migration_result.stdout_lines }}" diff --git a/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml new file mode 100644 index 000000000..c7f82c747 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml @@ -0,0 +1,84 @@ +- name: Prometheus Setup for Aggregation Mode + hosts: "{{ host }}" + + tasks: + - name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + +- hosts: "{{ host }}" + vars: + prometheus_version: "{{ lookup('ini', 'prometheus_version', file='ini/config-{{ env }}.ini', default='3.6.0') }}" + gateway_primary_hostname: "{{ lookup('ini', 'gateway_primary_hostname', file='ini/config-{{ env }}.ini') }}" + gateway_secondary_hostname: "{{ lookup('ini', 'gateway_secondary_hostname', file='ini/config-{{ env }}.ini') }}" + gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file='ini/config-{{ env }}.ini', default='9094') }}" + poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file='ini/config-{{ env }}.ini', default='9095') }}" + + tasks: + - name: Check if Prometheus is installed + stat: + path: /home/{{ ansible_user }}/prometheus-{{ prometheus_version }}.linux-amd64/prometheus + register: prometheus_exists + + - name: Download Prometheus + when: not prometheus_exists.stat.exists + get_url: + url: "https://github.com/prometheus/prometheus/releases/download/v{{ prometheus_version }}/prometheus-{{ prometheus_version }}.linux-amd64.tar.gz" + dest: "/tmp/prometheus-{{ prometheus_version }}.tar.gz" + mode: '0644' + + - name: Extract Prometheus + when: not prometheus_exists.stat.exists + unarchive: + src: "/tmp/prometheus-{{ prometheus_version }}.tar.gz" + dest: /home/{{ ansible_user }}/ + remote_src: yes + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Create config directory + file: + path: /home/{{ ansible_user }}/config + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Template Prometheus config file + template: + src: prometheus/prometheus_agg_mode.yaml.j2 + dest: /home/{{ ansible_user }}/config/prometheus.yaml + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Create systemd user directory + file: + path: /home/{{ ansible_user }}/.config/systemd/user + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Template Prometheus systemd service + template: + src: services/prometheus_agg_mode.service.j2 + dest: /home/{{ ansible_user }}/.config/systemd/user/prometheus.service + mode: '0644' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Enable and start Prometheus service + systemd_service: + name: prometheus + state: started + enabled: true + scope: user + daemon_reload: true + + - name: Clean up Prometheus tar + when: not prometheus_exists.stat.exists + file: + path: "/tmp/prometheus-{{ prometheus_version }}.tar.gz" + state: absent diff --git a/infra/aggregation_mode/ansible/playbooks/rust.yaml b/infra/aggregation_mode/ansible/playbooks/rust.yaml new file mode 100644 index 000000000..c3eac848f --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/rust.yaml @@ -0,0 +1,39 @@ +- name: Rust Setup + hosts: "{{ host }}" + vars: + rust_version: 1.83.0 + + tasks: + - name: Check if cargo is installed + stat: + path: /home/{{ ansible_user }}/.cargo/bin/cargo + register: cargo_exists + + - name: Download Rust installer + when: not cargo_exists.stat.exists + get_url: + url: https://sh.rustup.rs + dest: /tmp/rustup-init.sh + mode: '0755' + + - name: Install Rust + when: not cargo_exists.stat.exists + shell: | + /tmp/rustup-init.sh -y --default-toolchain {{ rust_version }} + args: + creates: /home/{{ ansible_user }}/.cargo/bin/cargo + + - name: Update user PATH + when: not cargo_exists.stat.exists + lineinfile: + path: "/home/{{ ansible_user }}/.bashrc" + line: "{{ item }}" + state: present + with_items: + - 'export PATH=$HOME/.cargo/bin:$PATH' + + - name: Clean up Rust installer + when: not cargo_exists.stat.exists + file: + path: /tmp/rustup-init.sh + state: absent diff --git a/infra/aggregation_mode/ansible/playbooks/setup.yaml b/infra/aggregation_mode/ansible/playbooks/setup.yaml new file mode 100644 index 000000000..a521324d3 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/setup.yaml @@ -0,0 +1,40 @@ +- name: Server setup + hosts: "{{ host }}" + + tasks: + # Install required packages + - name: Update apt and install required system packages + become: true + apt: + pkg: + - curl + - vim + - git + - make + - gcc + state: latest + update_cache: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + # Create basic directories + - name: Create basic directories if do not exist + ansible.builtin.file: + path: /home/{{ ansible_user }}/{{ item }} + state: directory + mode: '0755' + owner: '{{ ansible_user }}' + group: '{{ ansible_user }}' + loop: + - repos + - config + - services + - .keystores + + - name: Enable linger for {{ ansible_user }} + become: true + command: sudo loginctl enable-linger {{ ansible_user }} + args: + creates: /var/lib/systemd/linger/{{ ansible_user }} + vars: + ansible_ssh_user: "{{ admin_user }}" diff --git a/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-gateway.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-gateway.yaml.j2 new file mode 100644 index 000000000..fab21b85b --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-gateway.yaml.j2 @@ -0,0 +1,17 @@ +ip: "0.0.0.0" +port: {{ gateway_port }} +{% if gateway_tls_enabled == 'true' %} +tls_cert_path: "{{ gateway_tls_cert_path }}" +tls_key_path: "{{ gateway_tls_key_path }}" +tls_port: {{ gateway_tls_port }} +{% endif %} +db_connection_urls: + - "postgres://{{ gateway_db_user }}:{{ gateway_db_password }}@{{ gateway_postgres_primary }}:{{ gateway_postgres_port }}/{{ gateway_db_name }}" + - "postgres://{{ gateway_db_user }}:{{ gateway_db_password }}@{{ gateway_postgres_secondary }}:{{ gateway_postgres_port }}/{{ gateway_db_name }}" +eth_rpc_url: "{{ gateway_eth_rpc_url }}" +payment_service_address: "{{ gateway_payment_service_address }}" +network: "{{ gateway_network }}" +max_daily_proofs_per_user: {{ gateway_max_daily_proofs }} + +# Metrics +gateway_metrics_port: {{ gateway_metrics_port }} diff --git a/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-poller.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-poller.yaml.j2 new file mode 100644 index 000000000..8f8114287 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-poller.yaml.j2 @@ -0,0 +1,11 @@ +db_connection_urls: + - "postgres://{{ poller_db_user }}:{{ poller_db_password }}@{{ poller_postgres_primary }}:{{ poller_postgres_port }}/{{ poller_db_name }}" + - "postgres://{{ poller_db_user }}:{{ poller_db_password }}@{{ poller_postgres_secondary }}:{{ poller_postgres_port }}/{{ poller_db_name }}" +eth_rpc_url: "{{ poller_eth_rpc_url }}" +payment_service_address: "{{ poller_payment_service_address }}" +network: "{{ poller_network }}" +max_daily_proofs_per_user: {{ poller_max_daily_proofs }} +last_block_fetched_filepath: "{{ poller_last_block_fetched_filepath }}" + +# Metrics +poller_metrics_port: {{ poller_metrics_port }} diff --git a/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 b/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 new file mode 100644 index 000000000..f1a809492 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 @@ -0,0 +1,9 @@ +GF_USERS_ALLOW_SIGN_UP=false +GF_INSTALL_PLUGINS=yesoreyeram-infinity-datasource +PROMETHEUS_URL={{ grafana_prometheus_url }} +RPC_URL={{ grafana_rpc_url }} +POSTGRES_HOST={{ grafana_postgres_host }} +POSTGRES_PORT={{ grafana_postgres_port }} +POSTGRES_DB={{ grafana_postgres_db }} +POSTGRES_USER={{ grafana_postgres_user }} +POSTGRES_PASSWORD={{ grafana_postgres_password }} diff --git a/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 new file mode 100644 index 000000000..2886c16fc --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 @@ -0,0 +1,51 @@ +global: + scrape_interval: 15s + +scrape_configs: +- job_name: "gateway-primary-http" + scrape_interval: 60s + static_configs: + - targets: ["{{ gateway_primary_hostname }}:8080"] + labels: + service: "gateway" + instance: "primary" + +- job_name: "gateway-secondary-http" + scrape_interval: 60s + static_configs: + - targets: ["{{ gateway_secondary_hostname }}:8080"] + labels: + service: "gateway" + instance: "secondary" + +- job_name: "gateway-primary" + scrape_interval: 15s + static_configs: + - targets: ["{{ gateway_primary_hostname }}:{{ gateway_metrics_port }}"] + labels: + service: "gateway" + instance: "primary" + +- job_name: "gateway-secondary" + scrape_interval: 15s + static_configs: + - targets: ["{{ gateway_secondary_hostname }}:{{ gateway_metrics_port }}"] + labels: + service: "gateway" + instance: "secondary" + +- job_name: "poller-primary" + scrape_interval: 15s + static_configs: + - targets: ["{{ gateway_primary_hostname }}:{{ poller_metrics_port }}"] + labels: + service: "poller" + instance: "primary" + +- job_name: "poller-secondary" + scrape_interval: 15s + static_configs: + - targets: ["{{ gateway_secondary_hostname }}:{{ poller_metrics_port }}"] + labels: + service: "poller" + instance: "secondary" diff --git a/infra/aggregation_mode/ansible/playbooks/templates/services/gateway.service.j2 b/infra/aggregation_mode/ansible/playbooks/templates/services/gateway.service.j2 new file mode 100644 index 000000000..4d9724047 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/services/gateway.service.j2 @@ -0,0 +1,18 @@ +[Unit] +Description=Aggregation Mode Gateway +After=network.target + +[Service] +Type=simple +User={{ ansible_user }} +Group={{ ansible_user }} +WorkingDirectory=/home/{{ ansible_user }}/repos/gateway/aligned_layer/aggregation_mode +ExecStart=/home/{{ ansible_user }}/.cargo/bin/gateway /home/{{ ansible_user }}/config/config-agg-mode-gateway.yaml +Restart=always +RestartSec=1 +StartLimitBurst=100 +LimitNOFILE=100000 +AmbientCapabilities=CAP_NET_BIND_SERVICE + +[Install] +WantedBy=multi-user.target diff --git a/infra/aggregation_mode/ansible/playbooks/templates/services/poller.service.j2 b/infra/aggregation_mode/ansible/playbooks/templates/services/poller.service.j2 new file mode 100644 index 000000000..a1baeffac --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/services/poller.service.j2 @@ -0,0 +1,15 @@ +[Unit] +Description=Aggregation Mode Payments Poller +After=network.target + +[Service] +Type=simple +WorkingDirectory=/home/{{ ansible_user }}/repos/poller/aligned_layer/aggregation_mode +ExecStart=/home/{{ ansible_user }}/.cargo/bin/payments_poller /home/{{ ansible_user }}/config/config-agg-mode-poller.yaml +Restart=always +RestartSec=1 +StartLimitBurst=100 +LimitNOFILE=100000 + +[Install] +WantedBy=multi-user.target diff --git a/infra/aggregation_mode/ansible/playbooks/templates/services/prometheus_agg_mode.service.j2 b/infra/aggregation_mode/ansible/playbooks/templates/services/prometheus_agg_mode.service.j2 new file mode 100644 index 000000000..cf1563018 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/services/prometheus_agg_mode.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=Prometheus +After=network.target + +[Service] +Type=simple +WorkingDirectory=/home/{{ ansible_user }} +ExecStart=/home/{{ ansible_user }}/prometheus-{{ prometheus_version }}.linux-amd64/prometheus \ + --config.file=/home/{{ ansible_user }}/config/prometheus.yaml \ + --storage.tsdb.retention.time=90d +Restart=always +RestartSec=1 +StartLimitBurst=100 + +[Install] +WantedBy=multi-user.target diff --git a/infra/aggregation_mode/ansible/playbooks/templates/sudoers/gateway-service.j2 b/infra/aggregation_mode/ansible/playbooks/templates/sudoers/gateway-service.j2 new file mode 100644 index 000000000..192f06f07 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/sudoers/gateway-service.j2 @@ -0,0 +1,3 @@ +{{ ansible_user }} ALL=(ALL) NOPASSWD: /bin/systemctl start gateway, /bin/systemctl stop gateway, /bin/systemctl restart gateway, /bin/systemctl status gateway +{{ ansible_user }} ALL=(ALL) NOPASSWD: /bin/journalctl -u gateway* +{{ ansible_user }} ALL=(ALL) NOPASSWD: /bin/journalctl --user-unit=gateway* From 1bb05d0a87217051b1f6ddd11dc0c59202496b7c Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Wed, 14 Jan 2026 13:08:36 -0300 Subject: [PATCH 02/23] remove unused variables --- infra/aggregation_mode/ansible/playbooks/gateway.yaml | 2 -- infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini | 1 - infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini | 1 - infra/aggregation_mode/ansible/playbooks/poller.yaml | 1 - 4 files changed, 5 deletions(-) diff --git a/infra/aggregation_mode/ansible/playbooks/gateway.yaml b/infra/aggregation_mode/ansible/playbooks/gateway.yaml index 04d257a23..d553d0cdb 100644 --- a/infra/aggregation_mode/ansible/playbooks/gateway.yaml +++ b/infra/aggregation_mode/ansible/playbooks/gateway.yaml @@ -30,9 +30,7 @@ gateway_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file='ini/config-{{ env }}.ini') }}" gateway_network: "{{ lookup('ini', 'gateway_network', file='ini/config-{{ env }}.ini') }}" gateway_max_daily_proofs: "{{ lookup('ini', 'gateway_max_daily_proofs', file='ini/config-{{ env }}.ini', default='100') }}" - gateway_last_block_fetched_filepath: "{{ lookup('ini', 'gateway_last_block_fetched_filepath', file='ini/config-{{ env }}.ini') }}" gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file='ini/config-{{ env }}.ini', default='9094') }}" - poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file='ini/config-{{ env }}.ini', default='9095') }}" tls_cert_source_path: "{{ lookup('ini', 'tls_cert_source_path', file='ini/config-{{ env }}.ini') }}" tls_key_source_path: "{{ lookup('ini', 'tls_key_source_path', file='ini/config-{{ env }}.ini') }}" diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index d40277519..9a7e907f1 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -56,7 +56,6 @@ gateway_tls_enabled=true gateway_tls_port=443 gateway_tls_cert_path=/home/app/.ssl/cert.pem gateway_tls_key_path=/home/app/.ssl/key.pem -gateway_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json # Poller Service Settings (same for all pollers) poller_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index 70d25c24f..268091b6a 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -58,7 +58,6 @@ gateway_tls_enabled=true gateway_tls_port=443 gateway_tls_cert_path=/home/app/.ssl/cert.pem gateway_tls_key_path=/home/app/.ssl/key.pem -gateway_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json # Poller Service Settings (same for all pollers) poller_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json diff --git a/infra/aggregation_mode/ansible/playbooks/poller.yaml b/infra/aggregation_mode/ansible/playbooks/poller.yaml index 7c1de48c0..26dbb03f1 100644 --- a/infra/aggregation_mode/ansible/playbooks/poller.yaml +++ b/infra/aggregation_mode/ansible/playbooks/poller.yaml @@ -27,7 +27,6 @@ poller_max_daily_proofs: "{{ lookup('ini', 'gateway_max_daily_proofs', file='ini/config-{{ env }}.ini', default='100') }}" poller_last_block_fetched_filepath: "{{ lookup('ini', 'poller_last_block_fetched_filepath', file='ini/config-{{ env }}.ini') }}" last_block_fetched_initial_value: "{{ lookup('ini', 'last_block_fetched_initial_value', file='ini/config-{{ env }}.ini', default='0') }}" - gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file='ini/config-{{ env }}.ini', default='9094') }}" poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file='ini/config-{{ env }}.ini', default='9095') }}" tasks: From df26df2841fb5805cb72f79b75559d5cde5b5f98 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Wed, 14 Jan 2026 16:49:36 -0300 Subject: [PATCH 03/23] fix postgres cluster --- .../ansible/playbooks/ini/config-hoodi.ini | 2 +- .../ansible/playbooks/ini/config-mainnet.ini | 2 +- .../playbooks/pg_autofailover_common.yaml | 1 + .../ansible/playbooks/pg_monitor.yaml | 60 ++++++++++++------- .../ansible/playbooks/pg_node.yaml | 55 ++++++++++++----- .../ansible/playbooks/postgres_cluster.yaml | 41 +++++++------ .../playbooks/postgres_migrations.yaml | 40 +++++++------ .../ansible/playbooks/rust.yaml | 2 +- 8 files changed, 124 insertions(+), 79 deletions(-) diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index 9a7e907f1..aca8d8743 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -1,4 +1,4 @@ -[DEFAULT] +[global] # ============================================ # Hoodi Environment Configuration # ============================================ diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index 268091b6a..17dacb20f 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -1,4 +1,4 @@ -[DEFAULT] +[global] # ============================================ # Mainnet Environment Configuration # ============================================ diff --git a/infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml b/infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml index 655637704..5d3382613 100644 --- a/infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml +++ b/infra/aggregation_mode/ansible/playbooks/pg_autofailover_common.yaml @@ -9,6 +9,7 @@ - postgresql-common - ca-certificates - curl + - acl state: latest update_cache: true vars: diff --git a/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml b/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml index 36c19f039..3a7da6aa7 100644 --- a/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml +++ b/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml @@ -1,19 +1,33 @@ +- name: Import pg_autofailover_common playbook + ansible.builtin.import_playbook: pg_autofailover_common.yaml + vars: + host: "{{ host }}" + - name: PostgreSQL Monitor Setup hosts: "{{ host }}" + vars: + ansible_common_remote_group: postgres tasks: - - name: Import pg_autofailover_common playbook - ansible.builtin.import_playbook: pg_autofailover_common.yaml - vars: - host: "{{ host }}" + - name: Set config file path + set_fact: + config_file: "ini/config-{{ env }}.ini" -- hosts: "{{ host }}" - vars: - db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}" - monitor_pgdata: "{{ lookup('ini', 'monitor_pgdata', file='ini/config-{{ env }}.ini', default='/var/lib/postgresql/monitor') }}" - monitor_port: "{{ lookup('ini', 'monitor_port', file='ini/config-{{ env }}.ini', default='5432') }}" + - name: Debug config file + debug: + msg: "Using config file: {{ config_file }}" + + - name: Set config vars from INI file + set_fact: + db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" + monitor_pgdata: "{{ lookup('ini', 'monitor_pgdata', file=config_file, default='/var/lib/postgresql/monitor') }}" + monitor_port: "{{ lookup('ini', 'monitor_port', file=config_file, default='5432') }}" + no_log: true + + - name: Debug vars + debug: + msg: "env={{ env }}, monitor_pgdata={{ monitor_pgdata }}" - tasks: - name: Install postgresql-16-auto-failover become: true apt: @@ -50,19 +64,6 @@ args: creates: "{{ monitor_pgdata }}/PG_VERSION" - - name: Wait for monitor to be ready - wait_for: - port: "{{ monitor_port }}" - delay: 5 - timeout: 60 - - - name: Set password for autoctl_node user - become: true - become_user: postgres - shell: | - psql -d pg_auto_failover -c "ALTER USER autoctl_node PASSWORD '{{ db_password }}';" - no_log: true - - name: Update pg_hba.conf for Tailscale network become: true become_user: postgres @@ -99,3 +100,16 @@ daemon_reload: true vars: ansible_ssh_user: "{{ admin_user }}" + + - name: Wait for monitor to be ready + wait_for: + port: "{{ monitor_port }}" + delay: 5 + timeout: 60 + + - name: Set password for autoctl_node user + become: true + become_user: postgres + shell: | + psql -d pg_auto_failover -c "ALTER USER autoctl_node PASSWORD '{{ db_password }}';" + no_log: true diff --git a/infra/aggregation_mode/ansible/playbooks/pg_node.yaml b/infra/aggregation_mode/ansible/playbooks/pg_node.yaml index b271d4d10..9b394de62 100644 --- a/infra/aggregation_mode/ansible/playbooks/pg_node.yaml +++ b/infra/aggregation_mode/ansible/playbooks/pg_node.yaml @@ -1,23 +1,29 @@ +- name: Import pg_autofailover_common playbook + ansible.builtin.import_playbook: pg_autofailover_common.yaml + vars: + host: "{{ host }}" + - name: PostgreSQL Node Setup hosts: "{{ host }}" + vars: + ansible_common_remote_group: postgres tasks: - - name: Import pg_autofailover_common playbook - ansible.builtin.import_playbook: pg_autofailover_common.yaml - vars: - host: "{{ host }}" + - name: Set config file path + set_fact: + config_file: "ini/config-{{ env }}.ini" -- hosts: "{{ host }}" - vars: - db_name: "{{ lookup('ini', 'db_name', file='ini/config-{{ env }}.ini') }}" - db_user: "{{ lookup('ini', 'db_user', file='ini/config-{{ env }}.ini') }}" - db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}" - postgres_monitor_hostname: "{{ lookup('ini', 'postgres_monitor_hostname', file='ini/config-{{ env }}.ini') }}" - node_pgdata: "{{ lookup('ini', 'node_pgdata', file='ini/config-{{ env }}.ini', default='/var/lib/postgresql/node') }}" - node_port: "{{ lookup('ini', 'node_port', file='ini/config-{{ env }}.ini', default='5432') }}" - backup_dir: "{{ lookup('ini', 'backup_dir', file='ini/config-{{ env }}.ini', default='/var/lib/backup') }}" + - name: Set config vars from INI file + set_fact: + db_name: "{{ lookup('ini', 'db_name', file=config_file) }}" + db_user: "{{ lookup('ini', 'db_user', file=config_file) }}" + db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" + postgres_monitor_hostname: "{{ lookup('ini', 'postgres_monitor_hostname', file=config_file) }}" + node_pgdata: "{{ lookup('ini', 'node_pgdata', file=config_file, default='/var/lib/postgresql/node') }}" + node_port: "{{ lookup('ini', 'node_port', file=config_file, default='5432') }}" + backup_dir: "{{ lookup('ini', 'backup_dir', file=config_file, default='/var/lib/backup') }}" + no_log: true - tasks: - name: Create backup directory become: true file: @@ -36,6 +42,10 @@ become: true become_user: postgres + - name: Debug node_initialized + debug: + msg: "Node initialized: {{ node_initialized.stat.exists }}, path: {{ node_pgdata }}/PG_VERSION" + - name: Get Tailscale IP shell: tailscale ip --4 register: tailscale_ip @@ -56,7 +66,6 @@ --hostname {{ tailscale_ip.stdout }} \ --monitor 'postgres://{{ db_user }}:{{ db_password }}@{{ postgres_monitor_hostname }}:5432/pg_auto_failover?sslmode=require' when: not node_initialized.stat.exists - no_log: true args: creates: "{{ node_pgdata }}/PG_VERSION" @@ -108,3 +117,19 @@ port: "{{ node_port }}" delay: 5 timeout: 60 + + - name: Check if node is writable (primary) + become: true + become_user: postgres + shell: | + psql -d {{ db_name }} -tAc "SELECT NOT pg_is_in_recovery();" + register: is_writable + changed_when: false + + - name: Set password for pgautofailover_replicator user + become: true + become_user: postgres + shell: | + psql -d {{ db_name }} -c "ALTER USER pgautofailover_replicator PASSWORD '{{ db_password }}';" + when: is_writable.stdout == 't' + no_log: true diff --git a/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml b/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml index 24b033d5e..9cc845212 100644 --- a/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml +++ b/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml @@ -1,24 +1,23 @@ -- name: Deploy PostgreSQL Auto-Failover Cluster - hosts: localhost - gather_facts: no +- name: Deploy PostgreSQL Monitor + ansible.builtin.import_playbook: pg_monitor.yaml + vars: + host: postgres_monitor + env: "{{ env }}" - tasks: - - name: Deploy PostgreSQL Monitor - ansible.builtin.import_playbook: pg_monitor.yaml - vars: - host: postgres_monitor +- name: Deploy PostgreSQL Primary Node + ansible.builtin.import_playbook: pg_node.yaml + vars: + host: postgres_primary + env: "{{ env }}" - - name: Deploy PostgreSQL Primary Node - ansible.builtin.import_playbook: pg_node.yaml - vars: - host: postgres_primary +- name: Deploy PostgreSQL Secondary Node + ansible.builtin.import_playbook: pg_node.yaml + vars: + host: postgres_secondary + env: "{{ env }}" - - name: Deploy PostgreSQL Secondary Node - ansible.builtin.import_playbook: pg_node.yaml - vars: - host: postgres_secondary - - - name: Run Database Migrations - ansible.builtin.import_playbook: postgres_migrations.yaml - vars: - host: postgres_primary +- name: Run Database Migrations + ansible.builtin.import_playbook: postgres_migrations.yaml + vars: + host: postgres_primary + env: "{{ env }}" diff --git a/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml b/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml index a6c526009..add485060 100644 --- a/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml +++ b/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml @@ -1,26 +1,32 @@ +- name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + +- name: Import rust playbook + ansible.builtin.import_playbook: rust.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + - name: PostgreSQL Migrations Setup hosts: "{{ host }}" tasks: - - name: Import setup playbook - ansible.builtin.import_playbook: setup.yaml - vars: - host: "{{ host }}" + - name: Set config file path + set_fact: + config_file: "ini/config-{{ env }}.ini" - - name: Import rust playbook - ansible.builtin.import_playbook: rust.yaml - vars: - host: "{{ host }}" - -- hosts: "{{ host }}" - vars: - db_name: "{{ lookup('ini', 'db_name', file='ini/config-{{ env }}.ini') }}" - db_user: "{{ lookup('ini', 'db_user', file='ini/config-{{ env }}.ini') }}" - db_password: "{{ lookup('ini', 'db_password', file='ini/config-{{ env }}.ini') }}" - postgres_primary_hostname: "{{ lookup('ini', 'postgres_primary_hostname', file='ini/config-{{ env }}.ini') }}" - git_branch: "{{ lookup('ini', 'git_branch', file='ini/config-{{ env }}.ini') }}" + - name: Set config vars from INI file + set_fact: + db_name: "{{ lookup('ini', 'db_name', file=config_file) }}" + db_user: "{{ lookup('ini', 'db_user', file=config_file) }}" + db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" + postgres_primary_hostname: "{{ lookup('ini', 'postgres_primary_hostname', file=config_file) }}" + git_branch: "{{ lookup('ini', 'git_branch', file=config_file) }}" + no_log: true - tasks: - name: Clone aligned_layer repository git: repo: https://github.com/yetanotherco/aligned_layer.git diff --git a/infra/aggregation_mode/ansible/playbooks/rust.yaml b/infra/aggregation_mode/ansible/playbooks/rust.yaml index c3eac848f..b74c764b4 100644 --- a/infra/aggregation_mode/ansible/playbooks/rust.yaml +++ b/infra/aggregation_mode/ansible/playbooks/rust.yaml @@ -1,7 +1,7 @@ - name: Rust Setup hosts: "{{ host }}" vars: - rust_version: 1.83.0 + rust_version: 1.92.0 tasks: - name: Check if cargo is installed From f5e2995f9b3704e598b168ba87602f15da1743b0 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Wed, 14 Jan 2026 16:51:28 -0300 Subject: [PATCH 04/23] fix load variables correctly --- .../ansible/playbooks/deploy_all.yaml | 38 +++++------ .../ansible/playbooks/gateway.yaml | 68 ++++++++++--------- .../ansible/playbooks/gateway_stack.yaml | 22 +++--- .../ansible/playbooks/grafana_agg_mode.yaml | 33 +++++---- .../ansible/playbooks/metrics_stack.yaml | 22 +++--- .../ansible/playbooks/poller.yaml | 58 +++++++++------- .../playbooks/prometheus_agg_mode.yaml | 28 ++++---- 7 files changed, 143 insertions(+), 126 deletions(-) diff --git a/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml b/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml index 160cfe4f1..d708c6a79 100644 --- a/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml +++ b/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml @@ -1,22 +1,22 @@ -- name: Deploy Full Aggregation Mode Stack - hosts: localhost - gather_facts: no +- name: Deploy PostgreSQL Cluster + ansible.builtin.import_playbook: postgres_cluster.yaml + vars: + env: "{{ env }}" - tasks: - - name: Deploy PostgreSQL Cluster - ansible.builtin.import_playbook: postgres_cluster.yaml +- name: Deploy Gateway and Poller on Primary + ansible.builtin.import_playbook: gateway_stack.yaml + vars: + host: gateway_primary + env: "{{ env }}" - - name: Deploy Gateway and Poller on Primary - ansible.builtin.import_playbook: gateway_stack.yaml - vars: - host: gateway_primary +- name: Deploy Gateway and Poller on Secondary + ansible.builtin.import_playbook: gateway_stack.yaml + vars: + host: gateway_secondary + env: "{{ env }}" - - name: Deploy Gateway and Poller on Secondary - ansible.builtin.import_playbook: gateway_stack.yaml - vars: - host: gateway_secondary - - - name: Deploy Metrics Stack - ansible.builtin.import_playbook: metrics_stack.yaml - vars: - host: metrics +- name: Deploy Metrics Stack + ansible.builtin.import_playbook: metrics_stack.yaml + vars: + host: metrics + env: "{{ env }}" diff --git a/infra/aggregation_mode/ansible/playbooks/gateway.yaml b/infra/aggregation_mode/ansible/playbooks/gateway.yaml index d553d0cdb..775f092a6 100644 --- a/infra/aggregation_mode/ansible/playbooks/gateway.yaml +++ b/infra/aggregation_mode/ansible/playbooks/gateway.yaml @@ -1,40 +1,46 @@ +- name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + +- name: Import rust playbook + ansible.builtin.import_playbook: rust.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + - name: Gateway Setup hosts: "{{ host }}" tasks: - - name: Import setup playbook - ansible.builtin.import_playbook: setup.yaml - vars: - host: "{{ host }}" + - name: Set config file path + set_fact: + config_file: "ini/config-{{ env }}.ini" - - name: Import rust playbook - ansible.builtin.import_playbook: rust.yaml - vars: - host: "{{ host }}" - -- hosts: "{{ host }}" - vars: - git_branch: "{{ lookup('ini', 'git_branch', file='ini/config-{{ env }}.ini') }}" - gateway_port: "{{ lookup('ini', 'gateway_port', file='ini/config-{{ env }}.ini', default='8080') }}" - gateway_tls_enabled: "{{ lookup('ini', 'gateway_tls_enabled', file='ini/config-{{ env }}.ini', default='true') }}" - gateway_tls_port: "{{ lookup('ini', 'gateway_tls_port', file='ini/config-{{ env }}.ini', default='443') }}" - gateway_tls_cert_path: "{{ lookup('ini', 'gateway_tls_cert_path', file='ini/config-{{ env }}.ini') }}" - gateway_tls_key_path: "{{ lookup('ini', 'gateway_tls_key_path', file='ini/config-{{ env }}.ini') }}" - gateway_db_user: "{{ lookup('ini', 'gateway_db_user', file='ini/config-{{ env }}.ini') }}" - gateway_db_password: "{{ lookup('ini', 'gateway_db_password', file='ini/config-{{ env }}.ini') }}" - gateway_db_name: "{{ lookup('ini', 'gateway_db_name', file='ini/config-{{ env }}.ini') }}" - gateway_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file='ini/config-{{ env }}.ini') }}" - gateway_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file='ini/config-{{ env }}.ini') }}" - gateway_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file='ini/config-{{ env }}.ini', default='5432') }}" - gateway_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file='ini/config-{{ env }}.ini') }}" - gateway_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file='ini/config-{{ env }}.ini') }}" - gateway_network: "{{ lookup('ini', 'gateway_network', file='ini/config-{{ env }}.ini') }}" - gateway_max_daily_proofs: "{{ lookup('ini', 'gateway_max_daily_proofs', file='ini/config-{{ env }}.ini', default='100') }}" - gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file='ini/config-{{ env }}.ini', default='9094') }}" - tls_cert_source_path: "{{ lookup('ini', 'tls_cert_source_path', file='ini/config-{{ env }}.ini') }}" - tls_key_source_path: "{{ lookup('ini', 'tls_key_source_path', file='ini/config-{{ env }}.ini') }}" + - name: Set config vars from INI file + set_fact: + git_branch: "{{ lookup('ini', 'git_branch', file=config_file) }}" + gateway_port: "{{ lookup('ini', 'gateway_port', file=config_file, default='8080') }}" + gateway_tls_enabled: "{{ lookup('ini', 'gateway_tls_enabled', file=config_file, default='true') }}" + gateway_tls_port: "{{ lookup('ini', 'gateway_tls_port', file=config_file, default='443') }}" + gateway_tls_cert_path: "{{ lookup('ini', 'gateway_tls_cert_path', file=config_file) }}" + gateway_tls_key_path: "{{ lookup('ini', 'gateway_tls_key_path', file=config_file) }}" + gateway_db_user: "{{ lookup('ini', 'gateway_db_user', file=config_file) }}" + gateway_db_password: "{{ lookup('ini', 'gateway_db_password', file=config_file) }}" + gateway_db_name: "{{ lookup('ini', 'gateway_db_name', file=config_file) }}" + gateway_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file=config_file) }}" + gateway_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file=config_file) }}" + gateway_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file=config_file, default='5432') }}" + gateway_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file=config_file) }}" + gateway_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file=config_file) }}" + gateway_network: "{{ lookup('ini', 'gateway_network', file=config_file) }}" + gateway_max_daily_proofs: "{{ lookup('ini', 'gateway_max_daily_proofs', file=config_file, default='100') }}" + gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file=config_file, default='9094') }}" + tls_cert_source_path: "{{ lookup('ini', 'tls_cert_source_path', file=config_file) }}" + tls_key_source_path: "{{ lookup('ini', 'tls_key_source_path', file=config_file) }}" + no_log: true - tasks: - name: Install required system packages become: true apt: diff --git a/infra/aggregation_mode/ansible/playbooks/gateway_stack.yaml b/infra/aggregation_mode/ansible/playbooks/gateway_stack.yaml index eceba3b91..10ab7cc5b 100644 --- a/infra/aggregation_mode/ansible/playbooks/gateway_stack.yaml +++ b/infra/aggregation_mode/ansible/playbooks/gateway_stack.yaml @@ -1,13 +1,11 @@ -- name: Deploy Gateway and Poller Stack - hosts: "{{ host }}" +- name: Deploy Gateway + ansible.builtin.import_playbook: gateway.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" - tasks: - - name: Deploy Gateway - ansible.builtin.import_playbook: gateway.yaml - vars: - host: "{{ host }}" - - - name: Deploy Poller - ansible.builtin.import_playbook: poller.yaml - vars: - host: "{{ host }}" +- name: Deploy Poller + ansible.builtin.import_playbook: poller.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" diff --git a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml index 388d53ab5..d293077ff 100644 --- a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml @@ -1,23 +1,28 @@ +- name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + - name: Grafana Setup for Aggregation Mode hosts: "{{ host }}" tasks: - - name: Import setup playbook - ansible.builtin.import_playbook: setup.yaml - vars: - host: "{{ host }}" + - name: Set config file path + set_fact: + config_file: "ini/config-{{ env }}.ini" -- hosts: "{{ host }}" - vars: - grafana_prometheus_url: "{{ lookup('ini', 'grafana_prometheus_url', file='ini/config-{{ env }}.ini') }}" - grafana_rpc_url: "{{ lookup('ini', 'grafana_rpc_url', file='ini/config-{{ env }}.ini') }}" - grafana_postgres_host: "{{ lookup('ini', 'grafana_postgres_host', file='ini/config-{{ env }}.ini') }}" - grafana_postgres_port: "{{ lookup('ini', 'grafana_postgres_port', file='ini/config-{{ env }}.ini', default='5432') }}" - grafana_postgres_db: "{{ lookup('ini', 'grafana_postgres_db', file='ini/config-{{ env }}.ini') }}" - grafana_postgres_user: "{{ lookup('ini', 'grafana_postgres_user', file='ini/config-{{ env }}.ini') }}" - grafana_postgres_password: "{{ lookup('ini', 'grafana_postgres_password', file='ini/config-{{ env }}.ini') }}" + - name: Set config vars from INI file + set_fact: + grafana_prometheus_url: "{{ lookup('ini', 'grafana_prometheus_url', file=config_file) }}" + grafana_rpc_url: "{{ lookup('ini', 'grafana_rpc_url', file=config_file) }}" + grafana_postgres_host: "{{ lookup('ini', 'grafana_postgres_host', file=config_file) }}" + grafana_postgres_port: "{{ lookup('ini', 'grafana_postgres_port', file=config_file, default='5432') }}" + grafana_postgres_db: "{{ lookup('ini', 'grafana_postgres_db', file=config_file) }}" + grafana_postgres_user: "{{ lookup('ini', 'grafana_postgres_user', file=config_file) }}" + grafana_postgres_password: "{{ lookup('ini', 'grafana_postgres_password', file=config_file) }}" + no_log: true - tasks: - name: Install required packages become: true apt: diff --git a/infra/aggregation_mode/ansible/playbooks/metrics_stack.yaml b/infra/aggregation_mode/ansible/playbooks/metrics_stack.yaml index 7303a0320..242362595 100644 --- a/infra/aggregation_mode/ansible/playbooks/metrics_stack.yaml +++ b/infra/aggregation_mode/ansible/playbooks/metrics_stack.yaml @@ -1,13 +1,11 @@ -- name: Deploy Metrics Stack - hosts: "{{ host }}" +- name: Deploy Prometheus + ansible.builtin.import_playbook: prometheus_agg_mode.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" - tasks: - - name: Deploy Prometheus - ansible.builtin.import_playbook: prometheus_agg_mode.yaml - vars: - host: "{{ host }}" - - - name: Deploy Grafana - ansible.builtin.import_playbook: grafana_agg_mode.yaml - vars: - host: "{{ host }}" +- name: Deploy Grafana + ansible.builtin.import_playbook: grafana_agg_mode.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" diff --git a/infra/aggregation_mode/ansible/playbooks/poller.yaml b/infra/aggregation_mode/ansible/playbooks/poller.yaml index 26dbb03f1..0c9e3da80 100644 --- a/infra/aggregation_mode/ansible/playbooks/poller.yaml +++ b/infra/aggregation_mode/ansible/playbooks/poller.yaml @@ -1,35 +1,41 @@ +- name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + +- name: Import rust playbook + ansible.builtin.import_playbook: rust.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + - name: Poller Setup hosts: "{{ host }}" tasks: - - name: Import setup playbook - ansible.builtin.import_playbook: setup.yaml - vars: - host: "{{ host }}" + - name: Set config file path + set_fact: + config_file: "ini/config-{{ env }}.ini" - - name: Import rust playbook - ansible.builtin.import_playbook: rust.yaml - vars: - host: "{{ host }}" - -- hosts: "{{ host }}" - vars: - git_branch: "{{ lookup('ini', 'git_branch', file='ini/config-{{ env }}.ini') }}" - poller_db_user: "{{ lookup('ini', 'gateway_db_user', file='ini/config-{{ env }}.ini') }}" - poller_db_password: "{{ lookup('ini', 'gateway_db_password', file='ini/config-{{ env }}.ini') }}" - poller_db_name: "{{ lookup('ini', 'gateway_db_name', file='ini/config-{{ env }}.ini') }}" - poller_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file='ini/config-{{ env }}.ini') }}" - poller_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file='ini/config-{{ env }}.ini') }}" - poller_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file='ini/config-{{ env }}.ini', default='5432') }}" - poller_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file='ini/config-{{ env }}.ini') }}" - poller_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file='ini/config-{{ env }}.ini') }}" - poller_network: "{{ lookup('ini', 'gateway_network', file='ini/config-{{ env }}.ini') }}" - poller_max_daily_proofs: "{{ lookup('ini', 'gateway_max_daily_proofs', file='ini/config-{{ env }}.ini', default='100') }}" - poller_last_block_fetched_filepath: "{{ lookup('ini', 'poller_last_block_fetched_filepath', file='ini/config-{{ env }}.ini') }}" - last_block_fetched_initial_value: "{{ lookup('ini', 'last_block_fetched_initial_value', file='ini/config-{{ env }}.ini', default='0') }}" - poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file='ini/config-{{ env }}.ini', default='9095') }}" + - name: Set config vars from INI file + set_fact: + git_branch: "{{ lookup('ini', 'git_branch', file=config_file) }}" + poller_db_user: "{{ lookup('ini', 'gateway_db_user', file=config_file) }}" + poller_db_password: "{{ lookup('ini', 'gateway_db_password', file=config_file) }}" + poller_db_name: "{{ lookup('ini', 'gateway_db_name', file=config_file) }}" + poller_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file=config_file) }}" + poller_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file=config_file) }}" + poller_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file=config_file, default='5432') }}" + poller_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file=config_file) }}" + poller_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file=config_file) }}" + poller_network: "{{ lookup('ini', 'gateway_network', file=config_file) }}" + poller_max_daily_proofs: "{{ lookup('ini', 'gateway_max_daily_proofs', file=config_file, default='100') }}" + poller_last_block_fetched_filepath: "{{ lookup('ini', 'poller_last_block_fetched_filepath', file=config_file) }}" + last_block_fetched_initial_value: "{{ lookup('ini', 'last_block_fetched_initial_value', file=config_file, default='0') }}" + poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file=config_file, default='9095') }}" + no_log: true - tasks: - name: Install required system packages become: true apt: diff --git a/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml index c7f82c747..1375a1318 100644 --- a/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml @@ -1,21 +1,25 @@ +- name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + - name: Prometheus Setup for Aggregation Mode hosts: "{{ host }}" tasks: - - name: Import setup playbook - ansible.builtin.import_playbook: setup.yaml - vars: - host: "{{ host }}" + - name: Set config file path + set_fact: + config_file: "ini/config-{{ env }}.ini" -- hosts: "{{ host }}" - vars: - prometheus_version: "{{ lookup('ini', 'prometheus_version', file='ini/config-{{ env }}.ini', default='3.6.0') }}" - gateway_primary_hostname: "{{ lookup('ini', 'gateway_primary_hostname', file='ini/config-{{ env }}.ini') }}" - gateway_secondary_hostname: "{{ lookup('ini', 'gateway_secondary_hostname', file='ini/config-{{ env }}.ini') }}" - gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file='ini/config-{{ env }}.ini', default='9094') }}" - poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file='ini/config-{{ env }}.ini', default='9095') }}" + - name: Set config vars from INI file + set_fact: + prometheus_version: "{{ lookup('ini', 'prometheus_version', file=config_file, default='3.6.0') }}" + gateway_primary_hostname: "{{ lookup('ini', 'gateway_primary_hostname', file=config_file) }}" + gateway_secondary_hostname: "{{ lookup('ini', 'gateway_secondary_hostname', file=config_file) }}" + gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file=config_file, default='9094') }}" + poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file=config_file, default='9095') }}" - tasks: - name: Check if Prometheus is installed stat: path: /home/{{ ansible_user }}/prometheus-{{ prometheus_version }}.linux-amd64/prometheus From a7dfd2da17d6864d2c4381afd07a08082b5cc47d Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Wed, 14 Jan 2026 18:27:05 -0300 Subject: [PATCH 05/23] update config ini --- .../ansible/playbooks/ini/config-mainnet.ini | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index 17dacb20f..2695699f0 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -34,10 +34,8 @@ backup_dir=/var/lib/backup # ============================================ gateway_network=mainnet gateway_max_daily_proofs=100 -# TODO: Update with mainnet payment service address -gateway_payment_service_address= -# TODO: Update with mainnet RPC URL -gateway_eth_rpc_url= +gateway_payment_service_address=0xc8631Bc1E60c20db40e474F791126212fA8255F4 +gateway_eth_rpc_url=https://aligned-mainnet-rpc-1.tail665ae.ts.net # Database connection (uses same credentials as postgres) gateway_db_user=autoctl_node @@ -81,7 +79,7 @@ gateway_secondary_hostname=agg-mode-mainnet-gateway-2 # Grafana Configuration grafana_prometheus_url=http://localhost:9090 # TODO: Update with mainnet RPC URL -grafana_rpc_url= +grafana_rpc_url=https://aligned-mainnet-rpc-1.tail665ae.ts.net grafana_postgres_host=agg-mode-mainnet-postgres-1 grafana_postgres_port=5432 grafana_postgres_db=agg_mode From 510465ebe994f17fb764739418eba5763f38c859 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Wed, 14 Jan 2026 20:08:58 -0300 Subject: [PATCH 06/23] fixes --- Makefile | 42 ++++++++++---- infra/aggregation_mode/ansible/README.md | 57 +++++++++++++++---- .../ansible/playbooks/gateway.yaml | 6 ++ .../ansible/playbooks/ini/config-mainnet.ini | 2 +- .../ansible/playbooks/pg_node.yaml | 8 +++ .../ansible/playbooks/poller.yaml | 17 +++++- 6 files changed, 107 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 9dce05b8b..204017bab 100644 --- a/Makefile +++ b/Makefile @@ -1736,41 +1736,61 @@ postgres_status: ## Check PostgreSQL cluster status. Usage: make postgres_status # ------------------------------------------------------------------------------ .PHONY: gateway_deploy -gateway_deploy: ## Deploy Gateway & Poller on both servers. Usage: make gateway_deploy ENV=hoodi +gateway_deploy: ## Deploy Gateway & Poller on both servers. Usage: make gateway_deploy ENV=hoodi [FORCE_REBUILD=true] @if [ -z "$(ENV)" ]; then \ echo "Error: ENV must be set (hoodi or mainnet)"; \ exit 1; \ fi - @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ + @EXTRA_VARS=""; \ + if [ -n "$(FORCE_REBUILD)" ]; then \ + EXTRA_VARS="-e force_rebuild=true"; \ + fi; \ + ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ -e "host=gateway_primary" \ - -e "env=$(ENV)" - @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ + -e "env=$(ENV)" \ + $$EXTRA_VARS + @EXTRA_VARS=""; \ + if [ -n "$(FORCE_REBUILD)" ]; then \ + EXTRA_VARS="-e force_rebuild=true"; \ + fi; \ + ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ -e "host=gateway_secondary" \ - -e "env=$(ENV)" + -e "env=$(ENV)" \ + $$EXTRA_VARS .PHONY: gateway_primary_deploy -gateway_primary_deploy: ## Deploy Gateway & Poller on primary only. Usage: make gateway_primary_deploy ENV=hoodi +gateway_primary_deploy: ## Deploy Gateway & Poller on primary only. Usage: make gateway_primary_deploy ENV=hoodi [FORCE_REBUILD=true] @if [ -z "$(ENV)" ]; then \ echo "Error: ENV must be set (hoodi or mainnet)"; \ exit 1; \ fi - @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ + @EXTRA_VARS=""; \ + if [ -n "$(FORCE_REBUILD)" ]; then \ + EXTRA_VARS="-e force_rebuild=true"; \ + fi; \ + ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ -e "host=gateway_primary" \ - -e "env=$(ENV)" + -e "env=$(ENV)" \ + $$EXTRA_VARS .PHONY: gateway_secondary_deploy -gateway_secondary_deploy: ## Deploy Gateway & Poller on secondary only. Usage: make gateway_secondary_deploy ENV=hoodi +gateway_secondary_deploy: ## Deploy Gateway & Poller on secondary only. Usage: make gateway_secondary_deploy ENV=hoodi [FORCE_REBUILD=true] @if [ -z "$(ENV)" ]; then \ echo "Error: ENV must be set (hoodi or mainnet)"; \ exit 1; \ fi - @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ + @EXTRA_VARS=""; \ + if [ -n "$(FORCE_REBUILD)" ]; then \ + EXTRA_VARS="-e force_rebuild=true"; \ + fi; \ + ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ -e "host=gateway_secondary" \ - -e "env=$(ENV)" + -e "env=$(ENV)" \ + $$EXTRA_VARS # ------------------------------------------------------------------------------ # Metrics Deployment diff --git a/infra/aggregation_mode/ansible/README.md b/infra/aggregation_mode/ansible/README.md index 24b04bbac..06913363e 100644 --- a/infra/aggregation_mode/ansible/README.md +++ b/infra/aggregation_mode/ansible/README.md @@ -246,8 +246,13 @@ make gateway_deploy ENV=hoodi # Or deploy individually make gateway_primary_deploy ENV=hoodi make gateway_secondary_deploy ENV=hoodi + +# Force rebuild (always rebuild binaries from latest code) +make gateway_deploy ENV=hoodi FORCE_REBUILD=true ``` +**Note:** By default, the deployment is idempotent and skips building if the binary already exists. Use `FORCE_REBUILD=true` to always rebuild from the latest code in the repository. + **Verify gateway is running:** ```bash ssh app@agg-mode-hoodi-gateway-1 "sudo systemctl status gateway" @@ -539,22 +544,49 @@ ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ -e "host=gateway_primary" \ -e "env=hoodi" + +# Deploy gateway with forced rebuild +ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ + -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ + -e "host=gateway_primary" \ + -e "env=hoodi" \ + -e "force_rebuild=true" ``` ### Updating Services -**Update gateway code:** +**Update gateway and poller with latest code:** + +The easiest way to update services is using the `FORCE_REBUILD` parameter: + +```bash +# Update both primary and secondary +make gateway_deploy ENV=hoodi FORCE_REBUILD=true + +# Or update individually +make gateway_primary_deploy ENV=hoodi FORCE_REBUILD=true +make gateway_secondary_deploy ENV=hoodi FORCE_REBUILD=true +``` + +This will: +1. Pull latest code from the configured branch (staging for hoodi, main for mainnet) +2. Delete existing binaries +3. Rebuild gateway and poller from source +4. Restart the services + +**Manual update (alternative):** + +If you prefer to update manually: + ```bash +# Gateway ssh app@agg-mode-hoodi-gateway-1 cd ~/repos/gateway/aligned_layer git pull origin staging cargo install --path aggregation_mode/gateway --bin gateway --features tls --locked sudo systemctl restart gateway -``` -**Update poller code:** -```bash -ssh app@agg-mode-hoodi-gateway-1 +# Poller cd ~/repos/poller/aligned_layer git pull origin staging cargo install --path aggregation_mode/payments_poller --bin payments_poller --locked @@ -563,16 +595,21 @@ systemctl --user restart poller ### Redeploy with Latest Code -To redeploy with the latest code from git, simply run the deployment again: +**Idempotent deployment (skip if binary exists):** ```bash make gateway_deploy ENV=hoodi ``` -The playbooks will: -1. Pull latest code from the configured branch -2. Rebuild the binaries -3. Restart the services +This pulls the latest code but skips building if the binary already exists. Use this when you only want to update configuration files. + +**Force rebuild (always rebuild binaries):** + +```bash +make gateway_deploy ENV=hoodi FORCE_REBUILD=true +``` + +This always rebuilds binaries from the latest code, even if they already exist. Use this when you want to deploy code changes. ### Changing Configuration diff --git a/infra/aggregation_mode/ansible/playbooks/gateway.yaml b/infra/aggregation_mode/ansible/playbooks/gateway.yaml index 775f092a6..f06ec16ea 100644 --- a/infra/aggregation_mode/ansible/playbooks/gateway.yaml +++ b/infra/aggregation_mode/ansible/playbooks/gateway.yaml @@ -85,6 +85,12 @@ version: "{{ git_branch }}" update: yes + - name: Remove existing gateway binary (if force rebuild) + file: + path: /home/{{ ansible_user }}/.cargo/bin/gateway + state: absent + when: force_rebuild | default(false) | bool + - name: Build gateway with TLS shell: | export PATH=$HOME/.cargo/bin:$PATH diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index 2695699f0..e302da513 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -59,7 +59,7 @@ gateway_tls_key_path=/home/app/.ssl/key.pem # Poller Service Settings (same for all pollers) poller_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json -last_block_fetched_initial_value=0 +last_block_fetched_initial_value=24235289 # ============================================ # TLS Certificate Management diff --git a/infra/aggregation_mode/ansible/playbooks/pg_node.yaml b/infra/aggregation_mode/ansible/playbooks/pg_node.yaml index 9b394de62..a754a0601 100644 --- a/infra/aggregation_mode/ansible/playbooks/pg_node.yaml +++ b/infra/aggregation_mode/ansible/playbooks/pg_node.yaml @@ -126,6 +126,14 @@ register: is_writable changed_when: false + - name: Set password for autoctl_node user in agg_mode database + become: true + become_user: postgres + shell: | + psql -d {{ db_name }} -c "ALTER USER {{ db_user }} PASSWORD '{{ db_password }}';" + when: is_writable.stdout == 't' + no_log: true + - name: Set password for pgautofailover_replicator user become: true become_user: postgres diff --git a/infra/aggregation_mode/ansible/playbooks/poller.yaml b/infra/aggregation_mode/ansible/playbooks/poller.yaml index 0c9e3da80..ee18f91df 100644 --- a/infra/aggregation_mode/ansible/playbooks/poller.yaml +++ b/infra/aggregation_mode/ansible/playbooks/poller.yaml @@ -55,6 +55,12 @@ version: "{{ git_branch }}" update: yes + - name: Remove existing poller binary (if force rebuild) + file: + path: /home/{{ ansible_user }}/.cargo/bin/payments_poller + state: absent + when: force_rebuild | default(false) | bool + - name: Build poller shell: | export PATH=$HOME/.cargo/bin:$PATH @@ -70,14 +76,19 @@ owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - - name: Create last_block_fetched file + - name: Check if last_block_fetched file exists and get size + stat: + path: "{{ poller_last_block_fetched_filepath }}" + register: last_block_file + + - name: Create or fix last_block_fetched file if empty or missing copy: - content: '{"last_block_fetched":{{ last_block_fetched_initial_value }}}' + content: '{"last_block_fetched": {{ last_block_fetched_initial_value }}}' dest: "{{ poller_last_block_fetched_filepath }}" mode: '0644' owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - force: no + when: not last_block_file.stat.exists or last_block_file.stat.size == 0 - name: Template poller config file template: From 67c8f0fbc050055bd8db2a6f92fce3e01f920dc6 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Thu, 15 Jan 2026 11:46:36 -0300 Subject: [PATCH 07/23] add read only for grafana --- .../ansible/playbooks/pg_monitor.yaml | 21 ++++++++++++++++++ .../ansible/playbooks/pg_node.yaml | 22 +++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml b/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml index 3a7da6aa7..7fc9af380 100644 --- a/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml +++ b/infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml @@ -22,6 +22,8 @@ db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" monitor_pgdata: "{{ lookup('ini', 'monitor_pgdata', file=config_file, default='/var/lib/postgresql/monitor') }}" monitor_port: "{{ lookup('ini', 'monitor_port', file=config_file, default='5432') }}" + grafana_postgres_user: "{{ lookup('ini', 'grafana_postgres_user', file=config_file) }}" + grafana_postgres_password: "{{ lookup('ini', 'grafana_postgres_password', file=config_file) }}" no_log: true - name: Debug vars @@ -113,3 +115,22 @@ shell: | psql -d pg_auto_failover -c "ALTER USER autoctl_node PASSWORD '{{ db_password }}';" no_log: true + + - name: Create Grafana read-only user on monitor + become: true + become_user: postgres + shell: | + psql -d pg_auto_failover << 'EOF' + DO $$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_user WHERE usename = '{{ grafana_postgres_user }}') THEN + CREATE USER {{ grafana_postgres_user }} WITH PASSWORD '{{ grafana_postgres_password }}'; + ELSE + ALTER USER {{ grafana_postgres_user }} WITH PASSWORD '{{ grafana_postgres_password }}'; + END IF; + END + $$; + GRANT CONNECT ON DATABASE pg_auto_failover TO {{ grafana_postgres_user }}; + GRANT pg_read_all_data TO {{ grafana_postgres_user }}; + EOF + no_log: true diff --git a/infra/aggregation_mode/ansible/playbooks/pg_node.yaml b/infra/aggregation_mode/ansible/playbooks/pg_node.yaml index a754a0601..75d85c9b0 100644 --- a/infra/aggregation_mode/ansible/playbooks/pg_node.yaml +++ b/infra/aggregation_mode/ansible/playbooks/pg_node.yaml @@ -22,6 +22,8 @@ node_pgdata: "{{ lookup('ini', 'node_pgdata', file=config_file, default='/var/lib/postgresql/node') }}" node_port: "{{ lookup('ini', 'node_port', file=config_file, default='5432') }}" backup_dir: "{{ lookup('ini', 'backup_dir', file=config_file, default='/var/lib/backup') }}" + grafana_postgres_user: "{{ lookup('ini', 'grafana_postgres_user', file=config_file) }}" + grafana_postgres_password: "{{ lookup('ini', 'grafana_postgres_password', file=config_file) }}" no_log: true - name: Create backup directory @@ -141,3 +143,23 @@ psql -d {{ db_name }} -c "ALTER USER pgautofailover_replicator PASSWORD '{{ db_password }}';" when: is_writable.stdout == 't' no_log: true + + - name: Create Grafana read-only user + become: true + become_user: postgres + shell: | + psql -d {{ db_name }} << 'EOF' + DO $$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_user WHERE usename = '{{ grafana_postgres_user }}') THEN + CREATE USER {{ grafana_postgres_user }} WITH PASSWORD '{{ grafana_postgres_password }}'; + ELSE + ALTER USER {{ grafana_postgres_user }} WITH PASSWORD '{{ grafana_postgres_password }}'; + END IF; + END + $$; + GRANT CONNECT ON DATABASE {{ db_name }} TO {{ grafana_postgres_user }}; + GRANT pg_read_all_data TO {{ grafana_postgres_user }}; + EOF + when: is_writable.stdout == 't' + no_log: true From 882d9e56b07e4f4dca1ebc6f98c9c066f0abe7ae Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Thu, 15 Jan 2026 12:39:45 -0300 Subject: [PATCH 08/23] update ini and set grafana provisioning --- .../ansible/playbooks/grafana_agg_mode.yaml | 24 +++++++++++++++++++ .../ansible/playbooks/ini/config-hoodi.ini | 3 ++- .../ansible/playbooks/ini/config-mainnet.ini | 3 ++- 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml index d293077ff..a0174f3a4 100644 --- a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml @@ -14,6 +14,7 @@ - name: Set config vars from INI file set_fact: + grafana_admin_password: "{{ lookup('ini', 'grafana_admin_password', file=config_file) }}" grafana_prometheus_url: "{{ lookup('ini', 'grafana_prometheus_url', file=config_file) }}" grafana_rpc_url: "{{ lookup('ini', 'grafana_rpc_url', file=config_file) }}" grafana_postgres_host: "{{ lookup('ini', 'grafana_postgres_host', file=config_file) }}" @@ -103,6 +104,29 @@ vars: ansible_ssh_user: "{{ admin_user }}" + - name: Set Grafana admin password + become: true + lineinfile: + path: /etc/grafana/grafana.ini + regexp: '^;?admin_password\s*=' + line: 'admin_password = {{ grafana_admin_password }}' + insertafter: '^\[security\]' + vars: + ansible_ssh_user: "{{ admin_user }}" + no_log: true + + - name: Copy provisioning directory + become: true + copy: + src: ../../../../grafana/provisioning/ + dest: /etc/grafana/provisioning/ + owner: grafana + group: grafana + mode: '0644' + directory_mode: '0755' + vars: + ansible_ssh_user: "{{ admin_user }}" + - name: Enable and start Grafana service become: true systemd_service: diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index aca8d8743..91279b061 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -77,11 +77,12 @@ gateway_primary_hostname=agg-mode-hoodi-gateway-1 gateway_secondary_hostname=agg-mode-hoodi-gateway-2 # Grafana Configuration +grafana_admin_password= grafana_prometheus_url=http://localhost:9090 grafana_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net grafana_postgres_host=agg-mode-hoodi-postgres-1 grafana_postgres_port=5432 grafana_postgres_db=agg_mode -grafana_postgres_user=autoctl_node +grafana_postgres_user=grafana # REQUIRED: Set to same password as db_password grafana_postgres_password= diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index e302da513..e65143f79 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -77,12 +77,13 @@ gateway_primary_hostname=agg-mode-mainnet-gateway-1 gateway_secondary_hostname=agg-mode-mainnet-gateway-2 # Grafana Configuration +grafana_admin_password= grafana_prometheus_url=http://localhost:9090 # TODO: Update with mainnet RPC URL grafana_rpc_url=https://aligned-mainnet-rpc-1.tail665ae.ts.net grafana_postgres_host=agg-mode-mainnet-postgres-1 grafana_postgres_port=5432 grafana_postgres_db=agg_mode -grafana_postgres_user=autoctl_node +grafana_postgres_user=grafana # REQUIRED: Set to same password as db_password grafana_postgres_password= From cbc34bf4809fdaf48ddcbd87867d9c23c8c57784 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Thu, 15 Jan 2026 16:59:09 -0300 Subject: [PATCH 09/23] task sender ansible --- Makefile | 46 ++++ infra/aggregation_mode/ansible/README.md | 194 +++++++++++++++- .../ansible/hoodi-inventory.yaml | 10 + .../ansible/mainnet-inventory.yaml | 10 + .../ansible/playbooks/deploy_all.yaml | 6 + .../ansible/playbooks/ini/config-hoodi.ini | 10 + .../ansible/playbooks/ini/config-mainnet.ini | 10 + .../ansible/playbooks/task_sender.yaml | 211 ++++++++++++++++++ 8 files changed, 493 insertions(+), 4 deletions(-) create mode 100644 infra/aggregation_mode/ansible/playbooks/task_sender.yaml diff --git a/Makefile b/Makefile index 204017bab..e915de8ab 100644 --- a/Makefile +++ b/Makefile @@ -1829,6 +1829,52 @@ grafana_deploy: ## Deploy Grafana only. Usage: make grafana_deploy ENV=hoodi -e "host=metrics" \ -e "env=$(ENV)" +# ------------------------------------------------------------------------------ +# Task Sender Deployment +# ------------------------------------------------------------------------------ + +.PHONY: task_sender_deploy +task_sender_deploy: ## Deploy task sender. Usage: make task_sender_deploy ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/task_sender.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=task_sender" \ + -e "env=$(ENV)" + +.PHONY: task_sender_status +task_sender_status: ## Check task sender status. Usage: make task_sender_status ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @echo "Checking task sender tmux session..." + @ansible -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml task_sender \ + -m shell \ + -a "tmux has-session -t task_sender && echo 'Task sender is running' || echo 'Task sender is not running'" + +.PHONY: task_sender_logs +task_sender_logs: ## View task sender logs. Usage: make task_sender_logs ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @echo "Use: ssh app@agg-mode-$(ENV)-task-sender 'tmux attach -t task_sender'" + @echo "Or: ssh app@agg-mode-$(ENV)-task-sender 'tmux capture-pane -t task_sender -p'" + +.PHONY: task_sender_restart +task_sender_restart: ## Restart task sender. Usage: make task_sender_restart ENV=hoodi + @if [ -z "$(ENV)" ]; then \ + echo "Error: ENV must be set (hoodi or mainnet)"; \ + exit 1; \ + fi + @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/task_sender.yaml \ + -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ + -e "host=task_sender" \ + -e "env=$(ENV)" + # ------------------------------------------------------------------------------ # Full Deployment # ------------------------------------------------------------------------------ diff --git a/infra/aggregation_mode/ansible/README.md b/infra/aggregation_mode/ansible/README.md index 06913363e..bcd921c50 100644 --- a/infra/aggregation_mode/ansible/README.md +++ b/infra/aggregation_mode/ansible/README.md @@ -37,6 +37,11 @@ The Ansible automation deploys a complete aggregation mode stack consisting of: - Grafana for visualization - 90-day retention +5. **Task Sender** (1 server) + - Automated proof submission service + - Runs continuously in tmux session + - Configurable interval and proof files + ## Architecture ``` @@ -65,6 +70,11 @@ The Ansible automation deploys a complete aggregation mode stack consisting of: │ │ └─ Grafana (3000) │ │ │ └────────────────────────┘ │ │ │ +│ ┌────────────────────────┐ │ +│ │ Task Sender │ │ +│ │ (tmux session) │ │ +│ └────────────────────────┘ │ +│ │ └─────────────────────────────────────────────────────────────────┘ ``` @@ -120,6 +130,9 @@ tls_key_source_path=/path/to/your/key.pem # REQUIRED: Same password for Grafana Postgres datasource grafana_postgres_password=your_secure_password_here + +# REQUIRED: Private key for task sender (sends proofs to network) +task_sender_private_key=0xYourPrivateKeyHere ``` **⚠️ CRITICAL**: All three password fields must be set to the same value before deploying! @@ -143,6 +156,9 @@ grafana_postgres_password=your_secure_password_here tls_cert_source_path=/path/to/your/cert.pem tls_key_source_path=/path/to/your/key.pem +# REQUIRED: Private key for task sender +task_sender_private_key=0xYourPrivateKeyHere + # TODO: Update these for mainnet deployment gateway_payment_service_address=0xYourMainnetPaymentServiceAddress gateway_eth_rpc_url=https://your-mainnet-rpc-url @@ -181,6 +197,13 @@ tls_key_source_path= # ← FILL THIS IN # Metrics Configuration grafana_postgres_password= # ← FILL THIS IN (same as db_password) # ... other metrics settings ... + +# Task Sender Configuration +task_sender_interval_hours=1 +task_sender_proof_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof +task_sender_vk_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0_vk.bin +task_sender_private_key= # ← FILL THIS IN +task_sender_network=hoodi ``` The Ansible templates will automatically generate two separate database connection URLs for failover: @@ -204,6 +227,7 @@ This will: 2. Run database migrations 3. Deploy gateway and poller on both servers 4. Deploy Prometheus and Grafana +5. Deploy task sender ### Step-by-Step Deployment @@ -279,6 +303,49 @@ make grafana_deploy ENV=hoodi - Prometheus: `http://:9090` - Grafana: `http://:3000` (default credentials: admin/admin) +#### 4. Deploy Task Sender + +```bash +# Deploy task sender +make task_sender_deploy ENV=hoodi +``` + +The task sender runs in a tmux session and continuously sends proofs to the network at the configured interval (default: 1 hour). + +**Automatic Deposit Check:** + +The deployment automatically: +1. Derives the wallet address from the configured private key +2. Checks if the address has an active subscription on the payment contract +3. If not subscribed or expired, automatically deposits 0.0035 ETH to the payment contract +4. Waits for transaction confirmation before starting the task sender + +**Requirements:** +- The account must have sufficient ETH for: + - Payment deposit: **0.0035 ETH** + - Gas fees: ~**0.001 ETH** (estimated) +- Foundry (cast) will be automatically installed if not present + +**Verify task sender is running:** +```bash +make task_sender_status ENV=hoodi +``` + +**View task sender logs:** +```bash +# Show how to view logs +make task_sender_logs ENV=hoodi + +# Or directly attach to the tmux session +ssh app@agg-mode-hoodi-sender 'tmux attach -t task_sender' +# Press Ctrl+B then D to detach without stopping +``` + +**Restart task sender:** +```bash +make task_sender_restart ENV=hoodi +``` + ## Service Management ### Restart Services @@ -295,6 +362,11 @@ make poller_restart ENV=hoodi HOST=gateway_primary make poller_restart ENV=hoodi HOST=gateway_secondary ``` +**Task Sender:** +```bash +make task_sender_restart ENV=hoodi +``` + ### Check Service Status **PostgreSQL Cluster:** @@ -324,6 +396,13 @@ ssh admin@agg-mode-hoodi-metrics "systemctl --user status prometheus" ssh admin@agg-mode-hoodi-metrics "sudo systemctl status grafana-server" ``` +**Task Sender:** +```bash +make task_sender_status ENV=hoodi +# Or check tmux session directly +ssh app@agg-mode-hoodi-sender "tmux has-session -t task_sender && echo 'Running' || echo 'Not running'" +``` + ### View Logs **Gateway:** @@ -341,6 +420,16 @@ ssh app@agg-mode-hoodi-gateway-1 "journalctl --user -u poller -f" ssh admin@agg-mode-hoodi-postgres-1 "sudo journalctl -u pgautofailover -f" ``` +**Task Sender:** +```bash +# Attach to tmux session to view live logs +ssh app@agg-mode-hoodi-sender 'tmux attach -t task_sender' +# Press Ctrl+B then D to detach + +# Or capture current pane output +ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p' +``` + ## Verification ### PostgreSQL Cluster Health @@ -416,6 +505,23 @@ ssh admin@agg-mode-hoodi-postgres-1 "sudo journalctl -u pgautofailover -f" - Go to Configuration → Data Sources - Verify Prometheus and PostgreSQL datasources are connected +### Task Sender + +1. **Check tmux session is running:** + ```bash + make task_sender_status ENV=hoodi + ``` + +2. **View recent logs:** + ```bash + ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p' + ``` + +3. **Verify proof submissions:** + - Check logs for successful proof submissions + - Look for transaction hashes in the output + - Verify proofs are appearing on the network + ## Troubleshooting ### PostgreSQL Issues @@ -508,6 +614,83 @@ Check Prometheus config: ssh admin@agg-mode-hoodi-metrics "cat ~/config/prometheus.yaml" ``` +### Task Sender Issues + +**Problem: Task sender not running** + +Check if tmux session exists: +```bash +ssh app@agg-mode-hoodi-sender "tmux list-sessions" +``` + +If missing, redeploy: +```bash +make task_sender_deploy ENV=hoodi +``` + +**Problem: Task sender crashes or exits** + +Check logs for errors: +```bash +ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p -S -100' +``` + +Common issues: +- Invalid private key → Check `task_sender_private_key` in `config-{{ env }}.ini` +- Missing proof/vk files → Verify files exist: `task_sender_proof_path`, `task_sender_vk_path` +- Network connectivity → Test RPC: `curl https://aligned-hoodi-rpc-geth.tail665ae.ts.net` +- Insufficient balance → Check account has ETH for gas fees + +**Problem: Proofs not being submitted** + +Check interval configuration: +```bash +ssh app@agg-mode-hoodi-sender "cat ~/repos/sender/aligned_layer/scripts/.agg_mode.task_sender.env" +``` + +Verify `INTERVAL_HOURS` is set correctly (default: 1 hour). Attach to session to see live activity: +```bash +ssh app@agg-mode-hoodi-sender 'tmux attach -t task_sender' +``` + +**Problem: Deployment fails with insufficient balance** + +The automatic deposit check requires the account to have at least **0.0045 ETH** (0.0035 for deposit + ~0.001 for gas). + +Check account balance: +```bash +ssh app@agg-mode-hoodi-sender +export PATH=$HOME/.foundry/bin:$PATH +cast balance --rpc-url +``` + +If balance is insufficient, send ETH to the account and redeploy: +```bash +make task_sender_deploy ENV=hoodi +``` + +**Problem: Automatic deposit fails** + +If the automatic deposit fails during deployment, check the Ansible output for error messages. Common issues: +- Insufficient ETH balance in the account +- RPC connection issues +- Gas price too high + +To manually deposit after fixing the issue: +```bash +ssh app@agg-mode-hoodi-sender +export PATH=$HOME/.cargo/bin:$PATH +agg_mode_cli deposit \ + --network hoodi \ + --rpc-url https://aligned-hoodi-rpc-geth.tail665ae.ts.net \ + --private-key +``` + +Then restart the task sender: +```bash +make task_sender_restart ENV=hoodi +``` + ### General Debugging **Check Tailscale connectivity:** @@ -663,6 +846,7 @@ infra/aggregation_mode/ansible/ ├── poller.yaml # Poller deployment ├── prometheus_agg_mode.yaml # Prometheus deployment ├── grafana_agg_mode.yaml # Grafana deployment + ├── task_sender.yaml # Task sender deployment ├── postgres_cluster.yaml # Postgres orchestration ├── gateway_stack.yaml # Gateway + poller orchestration ├── metrics_stack.yaml # Metrics orchestration @@ -673,13 +857,15 @@ infra/aggregation_mode/ansible/ 1. **Passwords**: Config files are tracked in git with empty password fields. Fill in passwords locally. Use `git update-index --assume-unchanged config-*.ini` after filling passwords to prevent accidentally committing them. -2. **TLS Certificates**: Keep private keys secure. The playbooks set appropriate permissions (0600). +2. **Private Keys**: The `task_sender_private_key` field must be filled with a valid Ethereum private key. Never commit this value to git. The playbook sets appropriate permissions (0600) on the environment file. + +3. **TLS Certificates**: Keep private keys secure. The playbooks set appropriate permissions (0600). -3. **SSH Access**: All servers are only accessible via Tailscale VPN (100.64.0.0/10). +4. **SSH Access**: All servers are only accessible via Tailscale VPN (100.64.0.0/10). -4. **PostgreSQL**: Uses scram-sha-256 password authentication, not trust mode. +5. **PostgreSQL**: Uses scram-sha-256 password authentication, not trust mode. -5. **Firewall**: UFW is configured on all servers with deny-by-default policy. +6. **Firewall**: UFW is configured on all servers with deny-by-default policy. ## Support diff --git a/infra/aggregation_mode/ansible/hoodi-inventory.yaml b/infra/aggregation_mode/ansible/hoodi-inventory.yaml index 4e2f1c91b..f08c3dd5d 100644 --- a/infra/aggregation_mode/ansible/hoodi-inventory.yaml +++ b/infra/aggregation_mode/ansible/hoodi-inventory.yaml @@ -65,9 +65,19 @@ metrics: ansible_user: admin ansible_python_interpreter: /usr/bin/python3 +# Task Sender +task_sender: + hosts: + agg-mode-hoodi-sender: + ansible_host: agg-mode-hoodi-sender + admin_user: admin + ansible_user: app + ansible_python_interpreter: /usr/bin/python3 + # All aggregation mode servers aggregation_mode: children: postgres_cluster: gateway_cluster: metrics: + task_sender: diff --git a/infra/aggregation_mode/ansible/mainnet-inventory.yaml b/infra/aggregation_mode/ansible/mainnet-inventory.yaml index 489cc9d8d..48ca062c8 100644 --- a/infra/aggregation_mode/ansible/mainnet-inventory.yaml +++ b/infra/aggregation_mode/ansible/mainnet-inventory.yaml @@ -65,9 +65,19 @@ metrics: ansible_user: admin ansible_python_interpreter: /usr/bin/python3 +# Task Sender +task_sender: + hosts: + agg-mode-mainnet-sender: + ansible_host: agg-mode-mainnet-sender + admin_user: admin + ansible_user: app + ansible_python_interpreter: /usr/bin/python3 + # All aggregation mode servers aggregation_mode: children: postgres_cluster: gateway_cluster: metrics: + task_sender: diff --git a/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml b/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml index d708c6a79..51e23a8a9 100644 --- a/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml +++ b/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml @@ -20,3 +20,9 @@ vars: host: metrics env: "{{ env }}" + +- name: Deploy Task Sender + ansible.builtin.import_playbook: task_sender.yaml + vars: + host: task_sender + env: "{{ env }}" diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index 91279b061..d7d07726e 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -86,3 +86,13 @@ grafana_postgres_db=agg_mode grafana_postgres_user=grafana # REQUIRED: Set to same password as db_password grafana_postgres_password= + +# ============================================ +# Task Sender Configuration +# ============================================ +task_sender_interval_hours=1 +task_sender_proof_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof +task_sender_vk_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0_vk.bin +# REQUIRED: Set private key for sending proofs +task_sender_private_key= +task_sender_network=hoodi diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index e65143f79..99c0c3bd3 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -87,3 +87,13 @@ grafana_postgres_db=agg_mode grafana_postgres_user=grafana # REQUIRED: Set to same password as db_password grafana_postgres_password= + +# ============================================ +# Task Sender Configuration +# ============================================ +task_sender_interval_hours=1 +task_sender_proof_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof +task_sender_vk_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0_vk.bin +# REQUIRED: Set private key for sending proofs +task_sender_private_key= +task_sender_network=mainnet diff --git a/infra/aggregation_mode/ansible/playbooks/task_sender.yaml b/infra/aggregation_mode/ansible/playbooks/task_sender.yaml new file mode 100644 index 000000000..f7fa523d0 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/task_sender.yaml @@ -0,0 +1,211 @@ +- name: Import setup playbook + ansible.builtin.import_playbook: setup.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + +- name: Import rust playbook + ansible.builtin.import_playbook: rust.yaml + vars: + host: "{{ host }}" + env: "{{ env }}" + +- name: Task Sender Setup + hosts: "{{ host }}" + + tasks: + - name: Set config file path + set_fact: + config_file: "ini/config-{{ env }}.ini" + + - name: Set config vars from INI file + set_fact: + git_branch: "{{ lookup('ini', 'git_branch', file=config_file) }}" + task_sender_interval_hours: "{{ lookup('ini', 'task_sender_interval_hours', file=config_file, default='1') }}" + task_sender_proof_path: "{{ lookup('ini', 'task_sender_proof_path', file=config_file) }}" + task_sender_vk_path: "{{ lookup('ini', 'task_sender_vk_path', file=config_file) }}" + task_sender_private_key: "{{ lookup('ini', 'task_sender_private_key', file=config_file) }}" + task_sender_network: "{{ lookup('ini', 'task_sender_network', file=config_file) }}" + gateway_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file=config_file) }}" + gateway_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file=config_file) }}" + no_log: true + + - name: Install tmux + become: true + apt: + pkg: + - tmux + state: latest + update_cache: true + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Create sender directory + file: + path: /home/{{ ansible_user }}/repos/sender + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Clone aligned_layer repository + git: + repo: https://github.com/yetanotherco/aligned_layer.git + dest: /home/{{ ansible_user }}/repos/sender/aligned_layer + version: "{{ git_branch }}" + update: yes + force: yes + + - name: Build CLI + shell: | + export PATH=$HOME/.cargo/bin:$PATH + cd /home/{{ ansible_user }}/repos/sender/aligned_layer + make agg_mode_install_cli + args: + creates: /home/{{ ansible_user }}/.cargo/bin/agg_mode_cli + + - name: Check if foundry (cast) is installed + shell: command -v cast + register: cast_installed + failed_when: false + changed_when: false + + - name: Install foundry (for cast tool) + shell: | + curl -L https://foundry.paradigm.xyz | bash + export PATH=$HOME/.foundry/bin:$PATH + foundryup + when: cast_installed.rc != 0 + + - name: Derive wallet address from private key + shell: | + export PATH=$HOME/.foundry/bin:$PATH + cast wallet address {{ task_sender_private_key }} + register: wallet_address_output + no_log: true + changed_when: false + + - name: Set wallet address variable + set_fact: + wallet_address: "{{ wallet_address_output.stdout | trim }}" + no_log: true + + - name: Check subscription status on payment contract + shell: | + export PATH=$HOME/.foundry/bin:$PATH + cast call {{ gateway_payment_service_address }} \ + "subscribedAddresses(address)(uint256)" \ + {{ wallet_address }} \ + --rpc-url {{ gateway_eth_rpc_url }} + register: subscription_expiration_output + changed_when: false + + - name: Get current timestamp + shell: date +%s + register: current_timestamp + changed_when: false + + - name: Set subscription status variables + set_fact: + subscription_expiration: "{{ subscription_expiration_output.stdout | regex_replace('\\s*\\[.*\\]', '') | trim }}" + current_time: "{{ current_timestamp.stdout | trim }}" + + - name: Display subscription status + debug: + msg: | + Wallet address: {{ wallet_address }} + Subscription expiration timestamp: {{ subscription_expiration }} + Current timestamp: {{ current_time }} + Is subscribed: {{ (subscription_expiration | int) > (current_time | int) }} + + - name: Deposit to payment contract if not subscribed + shell: | + export PATH=$HOME/.cargo/bin:$PATH + agg_mode_cli deposit \ + --network {{ task_sender_network }} \ + --rpc-url {{ gateway_eth_rpc_url }} \ + --private-key {{ task_sender_private_key }} + when: (subscription_expiration | int) <= (current_time | int) + no_log: true + register: deposit_result + + - name: Display deposit result + debug: + msg: "Deposit successful. Transaction receipt: {{ deposit_result.stdout }}" + when: (subscription_expiration | int) <= (current_time | int) and deposit_result is defined + + - name: Wait for deposit transaction to confirm + pause: + seconds: 10 + when: (subscription_expiration | int) <= (current_time | int) + + - name: Create scripts directory + file: + path: /home/{{ ansible_user }}/repos/sender/aligned_layer/scripts + state: directory + mode: '0755' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + + - name: Template task sender env file + template: + src: config-files/agg_mode.task_sender.env.j2 + dest: /home/{{ ansible_user }}/repos/sender/aligned_layer/scripts/.agg_mode.task_sender.env + mode: '0600' + owner: "{{ ansible_user }}" + group: "{{ ansible_user }}" + no_log: true + + - name: Check if tmux session exists + shell: tmux has-session -t task_sender 2>/dev/null + register: tmux_session_exists + failed_when: false + changed_when: false + + - name: Kill existing tmux session + shell: tmux kill-session -t task_sender + when: tmux_session_exists.rc == 0 + + - name: Start task sender in tmux session + shell: | + cd /home/{{ ansible_user }}/repos/sender/aligned_layer + tmux new-session -d -s task_sender 'bash -c "export PATH=$HOME/.cargo/bin:$PATH && make agg_mode_task_sender_start 2>&1 | tee /tmp/task_sender.log; exec bash"' + register: tmux_start + changed_when: true + + - name: Wait for task sender to initialize + pause: + seconds: 5 + + - name: Check if tmux session is still running + shell: tmux has-session -t task_sender 2>/dev/null + register: verify_tmux + failed_when: false + changed_when: false + + - name: Capture tmux pane content if session exists + shell: tmux capture-pane -t task_sender -p + register: tmux_output + when: verify_tmux.rc == 0 + failed_when: false + + - name: Display tmux output + debug: + msg: "Tmux session output: {{ tmux_output.stdout }}" + when: verify_tmux.rc == 0 and tmux_output is defined + + - name: Check error log if session failed + shell: cat /tmp/task_sender.log 2>/dev/null || echo "No log file found" + register: error_log + when: verify_tmux.rc != 0 + failed_when: false + + - name: Display error log if session failed + debug: + msg: "Task sender failed. Log: {{ error_log.stdout }}" + when: verify_tmux.rc != 0 + + - name: Fail if tmux session is not running + fail: + msg: "Task sender tmux session failed to start. Check the output above for details." + when: verify_tmux.rc != 0 From 315f24f4831b0346821de314a0dcd2abd6bd8b80 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Thu, 15 Jan 2026 18:14:46 -0300 Subject: [PATCH 10/23] missing template --- .../templates/config-files/agg_mode.task_sender.env.j2 | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/config-files/agg_mode.task_sender.env.j2 diff --git a/infra/aggregation_mode/ansible/playbooks/templates/config-files/agg_mode.task_sender.env.j2 b/infra/aggregation_mode/ansible/playbooks/templates/config-files/agg_mode.task_sender.env.j2 new file mode 100644 index 000000000..2ceab4fe0 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/config-files/agg_mode.task_sender.env.j2 @@ -0,0 +1,5 @@ +INTERVAL_HOURS={{ task_sender_interval_hours }} +PROOF_PATH={{ task_sender_proof_path }} +VK_PATH={{ task_sender_vk_path }} +PRIVATE_KEY={{ task_sender_private_key }} +NETWORK={{ task_sender_network }} From 40889c396660701c79b63ef7d8a6dab093e199ec Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Fri, 16 Jan 2026 11:57:55 -0300 Subject: [PATCH 11/23] improve variables and README.md --- infra/aggregation_mode/ansible/README.md | 121 +++--------------- .../ansible/playbooks/gateway.yaml | 6 +- .../ansible/playbooks/ini/config-hoodi.ini | 38 +++--- .../ansible/playbooks/ini/config-mainnet.ini | 39 +++--- .../ansible/playbooks/poller.yaml | 6 +- 5 files changed, 61 insertions(+), 149 deletions(-) diff --git a/infra/aggregation_mode/ansible/README.md b/infra/aggregation_mode/ansible/README.md index bcd921c50..76005c6ee 100644 --- a/infra/aggregation_mode/ansible/README.md +++ b/infra/aggregation_mode/ansible/README.md @@ -106,112 +106,37 @@ All servers are provisioned via Terraform and connected via Tailscale VPN. They ## Initial Setup -All configuration is consolidated into environment-specific files with predefined values. You only need to fill in sensitive values (passwords, certificate paths). +All configuration is consolidated into environment-specific files with predefined values. You only need to fill in sensitive values at the top of each config file. -### 1. Configure Hoodi Environment +### Configure Environment -Edit `playbooks/ini/config-hoodi.ini`: +Edit the config file for your environment: +- **Hoodi**: `playbooks/ini/config-hoodi.ini` +- **Mainnet**: `playbooks/ini/config-mainnet.ini` -All non-sensitive values are already pre-filled. You only need to set: +All non-sensitive values are already pre-filled. Fill in the required values at the top of the file: ```ini -[DEFAULT] -# ... (all values pre-filled) ... - -# REQUIRED: Set a strong password before deploying +# ============================================ +# REQUIRED: Sensitive Values (fill these in) +# ============================================ +# Database password (used by postgres, gateway, and poller) db_password=your_secure_password_here -# REQUIRED: Same password for gateway/poller database access -gateway_db_password=your_secure_password_here - -# REQUIRED: Provide local paths to your TLS certificate files -tls_cert_source_path=/path/to/your/cert.pem -tls_key_source_path=/path/to/your/key.pem - -# REQUIRED: Same password for Grafana Postgres datasource +# Grafana read-only database user password grafana_postgres_password=your_secure_password_here -# REQUIRED: Private key for task sender (sends proofs to network) -task_sender_private_key=0xYourPrivateKeyHere -``` - -**⚠️ CRITICAL**: All three password fields must be set to the same value before deploying! - -### 2. Configure Mainnet Environment (if needed) - -Edit `playbooks/ini/config-mainnet.ini`: - -Similar to Hoodi, fill in the required values: - -```ini -[DEFAULT] -# ... (most values pre-filled) ... - -# REQUIRED: Set passwords (same as above) -db_password=your_secure_password_here -gateway_db_password=your_secure_password_here -grafana_postgres_password=your_secure_password_here - -# REQUIRED: TLS certificate paths +# TLS certificates (local paths to copy from) tls_cert_source_path=/path/to/your/cert.pem tls_key_source_path=/path/to/your/key.pem -# REQUIRED: Private key for task sender -task_sender_private_key=0xYourPrivateKeyHere +# Grafana admin password +grafana_admin_password=your_grafana_admin_password -# TODO: Update these for mainnet deployment -gateway_payment_service_address=0xYourMainnetPaymentServiceAddress -gateway_eth_rpc_url=https://your-mainnet-rpc-url -grafana_rpc_url=https://your-mainnet-rpc-url +# Task sender private key (for sending proofs) +task_sender_private_key=0xYourPrivateKeyHere ``` -### Configuration File Structure - -The consolidated config files contain all settings organized by component: - -```ini -# config-hoodi.ini structure: -[DEFAULT] -environment=hoodi -git_branch=staging - -# PostgreSQL Configuration -postgres_monitor_hostname=agg-mode-hoodi-postgres-monitor -postgres_primary_hostname=agg-mode-hoodi-postgres-1 -postgres_secondary_hostname=agg-mode-hoodi-postgres-2 -db_name=agg_mode -db_user=autoctl_node -db_password= # ← FILL THIS IN - -# Gateway & Poller Configuration -gateway_network=Hoodi -gateway_payment_service_address=0x7222E0183cE1A96619d0c883e9bfc6b76D4e780e -gateway_eth_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net -gateway_db_password= # ← FILL THIS IN (same as db_password) -# ... other gateway settings ... - -# TLS Certificate Management -tls_cert_source_path= # ← FILL THIS IN -tls_key_source_path= # ← FILL THIS IN - -# Metrics Configuration -grafana_postgres_password= # ← FILL THIS IN (same as db_password) -# ... other metrics settings ... - -# Task Sender Configuration -task_sender_interval_hours=1 -task_sender_proof_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof -task_sender_vk_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0_vk.bin -task_sender_private_key= # ← FILL THIS IN -task_sender_network=hoodi -``` - -The Ansible templates will automatically generate two separate database connection URLs for failover: -- `postgres://autoctl_node:password@agg-mode-hoodi-postgres-1:5432/agg_mode` -- `postgres://autoctl_node:password@agg-mode-hoodi-postgres-2:5432/agg_mode` - -The sqlx driver will try them in order for automatic failover - ## Deployment ### Full Stack Deployment @@ -540,10 +465,7 @@ ssh admin@agg-mode-hoodi-postgres-1 "sudo journalctl -u pgautofailover -n 100" **Problem: Password authentication fails** -Verify password is set correctly in your environment config file (`config-hoodi.ini` or `config-mainnet.ini`). All three password fields must match: -- `db_password` -- `gateway_db_password` -- `grafana_postgres_password` +Verify `db_password` is set correctly in your environment config file (`config-hoodi.ini` or `config-mainnet.ini`). Check pg_hba.conf: ```bash @@ -566,7 +488,7 @@ ssh app@agg-mode-hoodi-gateway-1 "sudo journalctl -u gateway -n 100" Common issues: - Missing TLS certificates → Check paths in `config-{{ env }}.ini` (tls_cert_source_path, tls_key_source_path) -- Database connection failed → Verify password in `config-{{ env }}.ini` (gateway_db_password) +- Database connection failed → Verify `db_password` in `config-{{ env }}.ini` - Port 443 already in use → Check with `sudo lsof -i :443` **Problem: TLS certificate errors** @@ -806,10 +728,9 @@ This always rebuilds binaries from the latest code, even if they already exist. ### Rotating Passwords -1. Update all three password fields in your environment config file (`config-hoodi.ini` or `config-mainnet.ini`): - - `db_password` - - `gateway_db_password` - - `grafana_postgres_password` +1. Update password fields in your environment config file (`config-hoodi.ini` or `config-mainnet.ini`): + - `db_password` (used by postgres, gateway, and poller) + - `grafana_postgres_password` (separate read-only user) 2. Run password update on PostgreSQL: ```bash ssh admin@agg-mode-hoodi-postgres-monitor "sudo -u postgres psql -d pg_auto_failover -c \"ALTER USER autoctl_node PASSWORD 'new_password'\"" diff --git a/infra/aggregation_mode/ansible/playbooks/gateway.yaml b/infra/aggregation_mode/ansible/playbooks/gateway.yaml index f06ec16ea..98f73e418 100644 --- a/infra/aggregation_mode/ansible/playbooks/gateway.yaml +++ b/infra/aggregation_mode/ansible/playbooks/gateway.yaml @@ -26,9 +26,9 @@ gateway_tls_port: "{{ lookup('ini', 'gateway_tls_port', file=config_file, default='443') }}" gateway_tls_cert_path: "{{ lookup('ini', 'gateway_tls_cert_path', file=config_file) }}" gateway_tls_key_path: "{{ lookup('ini', 'gateway_tls_key_path', file=config_file) }}" - gateway_db_user: "{{ lookup('ini', 'gateway_db_user', file=config_file) }}" - gateway_db_password: "{{ lookup('ini', 'gateway_db_password', file=config_file) }}" - gateway_db_name: "{{ lookup('ini', 'gateway_db_name', file=config_file) }}" + gateway_db_user: "{{ lookup('ini', 'db_user', file=config_file) }}" + gateway_db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" + gateway_db_name: "{{ lookup('ini', 'db_name', file=config_file) }}" gateway_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file=config_file) }}" gateway_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file=config_file) }}" gateway_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file=config_file, default='5432') }}" diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index d7d07726e..aad6296c9 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -3,9 +3,25 @@ # Hoodi Environment Configuration # ============================================ # This file contains all configuration for the Hoodi environment. -# Only sensitive values (passwords, certificate paths) need to be filled in. +# ============================================ +# REQUIRED: Sensitive Values (fill these in) +# ============================================ +# Database password (used by postgres, gateway, and poller) +db_password= +# Grafana read-only database user password +grafana_postgres_password= +# TLS certificates (local paths to copy from) +tls_cert_source_path= +tls_key_source_path= +# Grafana admin password +grafana_admin_password= +# Task sender private key (for sending proofs) +task_sender_private_key= + +# ============================================ # Environment +# ============================================ environment=hoodi git_branch=staging @@ -17,8 +33,6 @@ postgres_primary_hostname=agg-mode-hoodi-postgres-1 postgres_secondary_hostname=agg-mode-hoodi-postgres-2 db_name=agg_mode db_user=autoctl_node -# REQUIRED: Set this password before deploying postgres -db_password= # PostgreSQL Monitor monitor_pgdata=/var/lib/postgresql/monitor @@ -37,11 +51,6 @@ gateway_max_daily_proofs=100 gateway_payment_service_address=0x7222E0183cE1A96619d0c883e9bfc6b76D4e780e gateway_eth_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net -# Database connection (uses same credentials as postgres) -gateway_db_user=autoctl_node -# REQUIRED: Set to same password as db_password -gateway_db_password= -gateway_db_name=agg_mode gateway_postgres_primary=agg-mode-hoodi-postgres-1 gateway_postgres_secondary=agg-mode-hoodi-postgres-2 gateway_postgres_port=5432 @@ -61,14 +70,6 @@ gateway_tls_key_path=/home/app/.ssl/key.pem poller_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json last_block_fetched_initial_value=0 -# ============================================ -# TLS Certificate Management -# ============================================ -# REQUIRED: Provide paths to existing certificates on your local machine -# These will be copied to the gateway servers -tls_cert_source_path= -tls_key_source_path= - # ============================================ # Metrics Configuration # ============================================ @@ -77,15 +78,12 @@ gateway_primary_hostname=agg-mode-hoodi-gateway-1 gateway_secondary_hostname=agg-mode-hoodi-gateway-2 # Grafana Configuration -grafana_admin_password= grafana_prometheus_url=http://localhost:9090 grafana_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net grafana_postgres_host=agg-mode-hoodi-postgres-1 grafana_postgres_port=5432 grafana_postgres_db=agg_mode grafana_postgres_user=grafana -# REQUIRED: Set to same password as db_password -grafana_postgres_password= # ============================================ # Task Sender Configuration @@ -93,6 +91,4 @@ grafana_postgres_password= task_sender_interval_hours=1 task_sender_proof_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof task_sender_vk_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0_vk.bin -# REQUIRED: Set private key for sending proofs -task_sender_private_key= task_sender_network=hoodi diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index 99c0c3bd3..587bd187f 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -3,9 +3,25 @@ # Mainnet Environment Configuration # ============================================ # This file contains all configuration for the Mainnet environment. -# Only sensitive values (passwords, certificate paths) need to be filled in. +# ============================================ +# REQUIRED: Sensitive Values (fill these in) +# ============================================ +# Database password (used by postgres, gateway, and poller) +db_password= +# Grafana read-only database user password +grafana_postgres_password= +# TLS certificates (local paths to copy from) +tls_cert_source_path= +tls_key_source_path= +# Grafana admin password +grafana_admin_password= +# Task sender private key (for sending proofs) +task_sender_private_key= + +# ============================================ # Environment +# ============================================ environment=mainnet git_branch=staging @@ -17,8 +33,6 @@ postgres_primary_hostname=agg-mode-mainnet-postgres-1 postgres_secondary_hostname=agg-mode-mainnet-postgres-2 db_name=agg_mode db_user=autoctl_node -# REQUIRED: Set this password before deploying postgres -db_password= # PostgreSQL Monitor monitor_pgdata=/var/lib/postgresql/monitor @@ -37,11 +51,6 @@ gateway_max_daily_proofs=100 gateway_payment_service_address=0xc8631Bc1E60c20db40e474F791126212fA8255F4 gateway_eth_rpc_url=https://aligned-mainnet-rpc-1.tail665ae.ts.net -# Database connection (uses same credentials as postgres) -gateway_db_user=autoctl_node -# REQUIRED: Set to same password as db_password -gateway_db_password= -gateway_db_name=agg_mode gateway_postgres_primary=agg-mode-mainnet-postgres-1 gateway_postgres_secondary=agg-mode-mainnet-postgres-2 gateway_postgres_port=5432 @@ -61,14 +70,6 @@ gateway_tls_key_path=/home/app/.ssl/key.pem poller_last_block_fetched_filepath=/home/app/config/proof-aggregator.last_block_fetched.json last_block_fetched_initial_value=24235289 -# ============================================ -# TLS Certificate Management -# ============================================ -# REQUIRED: Provide paths to existing certificates on your local machine -# These will be copied to the gateway servers -tls_cert_source_path= -tls_key_source_path= - # ============================================ # Metrics Configuration # ============================================ @@ -77,16 +78,12 @@ gateway_primary_hostname=agg-mode-mainnet-gateway-1 gateway_secondary_hostname=agg-mode-mainnet-gateway-2 # Grafana Configuration -grafana_admin_password= grafana_prometheus_url=http://localhost:9090 -# TODO: Update with mainnet RPC URL grafana_rpc_url=https://aligned-mainnet-rpc-1.tail665ae.ts.net grafana_postgres_host=agg-mode-mainnet-postgres-1 grafana_postgres_port=5432 grafana_postgres_db=agg_mode grafana_postgres_user=grafana -# REQUIRED: Set to same password as db_password -grafana_postgres_password= # ============================================ # Task Sender Configuration @@ -94,6 +91,4 @@ grafana_postgres_password= task_sender_interval_hours=1 task_sender_proof_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0.proof task_sender_vk_path=scripts/test_files/sp1/sp1_fibonacci_5_0_0_vk.bin -# REQUIRED: Set private key for sending proofs -task_sender_private_key= task_sender_network=mainnet diff --git a/infra/aggregation_mode/ansible/playbooks/poller.yaml b/infra/aggregation_mode/ansible/playbooks/poller.yaml index ee18f91df..ee0e05953 100644 --- a/infra/aggregation_mode/ansible/playbooks/poller.yaml +++ b/infra/aggregation_mode/ansible/playbooks/poller.yaml @@ -21,9 +21,9 @@ - name: Set config vars from INI file set_fact: git_branch: "{{ lookup('ini', 'git_branch', file=config_file) }}" - poller_db_user: "{{ lookup('ini', 'gateway_db_user', file=config_file) }}" - poller_db_password: "{{ lookup('ini', 'gateway_db_password', file=config_file) }}" - poller_db_name: "{{ lookup('ini', 'gateway_db_name', file=config_file) }}" + poller_db_user: "{{ lookup('ini', 'db_user', file=config_file) }}" + poller_db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" + poller_db_name: "{{ lookup('ini', 'db_name', file=config_file) }}" poller_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file=config_file) }}" poller_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file=config_file) }}" poller_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file=config_file, default='5432') }}" From ae3c344e9d6625a6cc44ef5b056d65495f63a692 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Fri, 16 Jan 2026 12:55:44 -0300 Subject: [PATCH 12/23] add commands for both networks --- infra/aggregation_mode/ansible/README.md | 351 ++++++++++++++++++++--- 1 file changed, 316 insertions(+), 35 deletions(-) diff --git a/infra/aggregation_mode/ansible/README.md b/infra/aggregation_mode/ansible/README.md index 76005c6ee..5e73acfcd 100644 --- a/infra/aggregation_mode/ansible/README.md +++ b/infra/aggregation_mode/ansible/README.md @@ -144,7 +144,11 @@ task_sender_private_key=0xYourPrivateKeyHere To deploy everything in one command: ```bash +# For Hoodi make agg_mode_deploy_all ENV=hoodi + +# For Mainnet +make agg_mode_deploy_all ENV=mainnet ``` This will: @@ -161,8 +165,11 @@ For more control, deploy each component separately: #### 1. Deploy PostgreSQL Cluster ```bash -# Deploy complete postgres cluster with password authentication +# For Hoodi make postgres_deploy ENV=hoodi + +# For Mainnet +make postgres_deploy ENV=mainnet ``` This will: @@ -174,7 +181,11 @@ This will: **Verify cluster status:** ```bash +# For Hoodi make postgres_status ENV=hoodi + +# For Mainnet +make postgres_status ENV=mainnet ``` Expected output: @@ -189,15 +200,17 @@ node_2 | 3 | 100.x.x.x:5432 | 1: 0/... | read-only | seconda #### 2. Deploy Gateway & Poller ```bash -# Deploy on both servers +# For Hoodi make gateway_deploy ENV=hoodi - -# Or deploy individually make gateway_primary_deploy ENV=hoodi make gateway_secondary_deploy ENV=hoodi - -# Force rebuild (always rebuild binaries from latest code) make gateway_deploy ENV=hoodi FORCE_REBUILD=true + +# For Mainnet +make gateway_deploy ENV=mainnet +make gateway_primary_deploy ENV=mainnet +make gateway_secondary_deploy ENV=mainnet +make gateway_deploy ENV=mainnet FORCE_REBUILD=true ``` **Note:** By default, the deployment is idempotent and skips building if the binary already exists. Use `FORCE_REBUILD=true` to always rebuild from the latest code in the repository. @@ -210,18 +223,21 @@ ssh app@agg-mode-hoodi-gateway-1 "systemctl --user status poller" **Test endpoint:** ```bash -curl -k https://agg-mode-hoodi-gateway-1/health +curl -k https://agg-mode-hoodi-gateway-1/ ``` #### 3. Deploy Metrics Stack ```bash -# Deploy both Prometheus and Grafana +# For Hoodi make metrics_deploy ENV=hoodi - -# Or deploy individually make prometheus_deploy ENV=hoodi make grafana_deploy ENV=hoodi + +# For Mainnet +make metrics_deploy ENV=mainnet +make prometheus_deploy ENV=mainnet +make grafana_deploy ENV=mainnet ``` **Access dashboards:** @@ -231,8 +247,11 @@ make grafana_deploy ENV=hoodi #### 4. Deploy Task Sender ```bash -# Deploy task sender +# For Hoodi make task_sender_deploy ENV=hoodi + +# For Mainnet +make task_sender_deploy ENV=mainnet ``` The task sender runs in a tmux session and continuously sends proofs to the network at the configured interval (default: 1 hour). @@ -253,22 +272,33 @@ The deployment automatically: **Verify task sender is running:** ```bash +# For Hoodi make task_sender_status ENV=hoodi + +# For Mainnet +make task_sender_status ENV=mainnet ``` **View task sender logs:** ```bash -# Show how to view logs +# For Hoodi make task_sender_logs ENV=hoodi - -# Or directly attach to the tmux session ssh app@agg-mode-hoodi-sender 'tmux attach -t task_sender' + +# For Mainnet +make task_sender_logs ENV=mainnet +ssh app@agg-mode-mainnet-sender 'tmux attach -t task_sender' + # Press Ctrl+B then D to detach without stopping ``` **Restart task sender:** ```bash +# For Hoodi make task_sender_restart ENV=hoodi + +# For Mainnet +make task_sender_restart ENV=mainnet ``` ## Service Management @@ -277,82 +307,137 @@ make task_sender_restart ENV=hoodi **Gateway:** ```bash +# For Hoodi make gateway_restart ENV=hoodi HOST=gateway_primary make gateway_restart ENV=hoodi HOST=gateway_secondary + +# For Mainnet +make gateway_restart ENV=mainnet HOST=gateway_primary +make gateway_restart ENV=mainnet HOST=gateway_secondary ``` **Poller:** ```bash +# For Hoodi make poller_restart ENV=hoodi HOST=gateway_primary make poller_restart ENV=hoodi HOST=gateway_secondary + +# For Mainnet +make poller_restart ENV=mainnet HOST=gateway_primary +make poller_restart ENV=mainnet HOST=gateway_secondary ``` **Task Sender:** ```bash +# For Hoodi make task_sender_restart ENV=hoodi + +# For Mainnet +make task_sender_restart ENV=mainnet ``` ### Check Service Status **PostgreSQL Cluster:** ```bash +# For Hoodi make postgres_status ENV=hoodi + +# For Mainnet +make postgres_status ENV=mainnet ``` **Gateway:** ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "sudo systemctl status gateway" ssh app@agg-mode-hoodi-gateway-1 "sudo journalctl -u gateway -n 50" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "sudo systemctl status gateway" +ssh app@agg-mode-mainnet-gateway-1 "sudo journalctl -u gateway -n 50" ``` **Poller:** ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "systemctl --user status poller" ssh app@agg-mode-hoodi-gateway-1 "journalctl --user -u poller -n 50" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "systemctl --user status poller" +ssh app@agg-mode-mainnet-gateway-1 "journalctl --user -u poller -n 50" ``` **Prometheus:** ```bash +# For Hoodi ssh admin@agg-mode-hoodi-metrics "systemctl --user status prometheus" + +# For Mainnet +ssh admin@agg-mode-mainnet-metrics "systemctl --user status prometheus" ``` **Grafana:** ```bash +# For Hoodi ssh admin@agg-mode-hoodi-metrics "sudo systemctl status grafana-server" + +# For Mainnet +ssh admin@agg-mode-mainnet-metrics "sudo systemctl status grafana-server" ``` **Task Sender:** ```bash +# For Hoodi make task_sender_status ENV=hoodi -# Or check tmux session directly ssh app@agg-mode-hoodi-sender "tmux has-session -t task_sender && echo 'Running' || echo 'Not running'" + +# For Mainnet +make task_sender_status ENV=mainnet +ssh app@agg-mode-mainnet-sender "tmux has-session -t task_sender && echo 'Running' || echo 'Not running'" ``` ### View Logs **Gateway:** ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "sudo journalctl -u gateway -f" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "sudo journalctl -u gateway -f" ``` **Poller:** ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "journalctl --user -u poller -f" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "journalctl --user -u poller -f" ``` **PostgreSQL:** ```bash +# For Hoodi ssh admin@agg-mode-hoodi-postgres-1 "sudo journalctl -u pgautofailover -f" + +# For Mainnet +ssh admin@agg-mode-mainnet-postgres-1 "sudo journalctl -u pgautofailover -f" ``` **Task Sender:** ```bash -# Attach to tmux session to view live logs +# For Hoodi ssh app@agg-mode-hoodi-sender 'tmux attach -t task_sender' -# Press Ctrl+B then D to detach - -# Or capture current pane output ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p' + +# For Mainnet +ssh app@agg-mode-mainnet-sender 'tmux attach -t task_sender' +ssh app@agg-mode-mainnet-sender 'tmux capture-pane -t task_sender -p' + +# Press Ctrl+B then D to detach ``` ## Verification @@ -361,62 +446,98 @@ ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p' 1. **Check cluster state:** ```bash + # For Hoodi make postgres_status ENV=hoodi + + # For Mainnet + make postgres_status ENV=mainnet ``` 2. **Test password authentication:** ```bash + # For Hoodi ssh admin@agg-mode-hoodi-postgres-1 "PGPASSWORD='your_password' psql -U autoctl_node -h localhost -d agg_mode -c 'SELECT 1'" + # For Mainnet + ssh admin@agg-mode-mainnet-postgres-1 "PGPASSWORD='your_password' psql -U autoctl_node -h localhost -d agg_mode -c 'SELECT 1'" ``` 3. **Verify replication:** ```bash + # For Hoodi ssh admin@agg-mode-hoodi-postgres-1 "sudo -u postgres psql -d agg_mode -c 'SELECT * FROM pg_stat_replication'" + + # For Mainnet + ssh admin@agg-mode-mainnet-postgres-1 "sudo -u postgres psql -d agg_mode -c 'SELECT * FROM pg_stat_replication'" ``` 4. **Test failover (optional):** ```bash - # Stop primary + # For Hoodi ssh admin@agg-mode-hoodi-postgres-1 "sudo systemctl stop pgautofailover" - # Wait 30 seconds, check status make postgres_status ENV=hoodi # Secondary should now be primary - - # Restart original primary ssh admin@agg-mode-hoodi-postgres-1 "sudo systemctl start pgautofailover" + + # For Mainnet + ssh admin@agg-mode-mainnet-postgres-1 "sudo systemctl stop pgautofailover" + # Wait 30 seconds, check status + make postgres_status ENV=mainnet + # Secondary should now be primary + ssh admin@agg-mode-mainnet-postgres-1 "sudo systemctl start pgautofailover" ``` ### Gateway Health 1. **Check HTTP health endpoint:** ```bash - curl -k https://agg-mode-hoodi-gateway-1/health + # For Hoodi + curl -k https://agg-mode-hoodi-gateway-1/ + + # For Mainnet + curl -k https://agg-mode-mainnet-gateway-1/ ``` 2. **Check metrics:** ```bash + # For Hoodi curl http://agg-mode-hoodi-gateway-1:9094/metrics + + # For Mainnet + curl http://agg-mode-mainnet-gateway-1:9094/metrics ``` 3. **Verify database connectivity:** ```bash + # For Hoodi ssh app@agg-mode-hoodi-gateway-1 PGPASSWORD='your_password' psql -U autoctl_node -h agg-mode-hoodi-postgres-1 -d agg_mode -c "SELECT 1" + + # For Mainnet + ssh app@agg-mode-mainnet-gateway-1 + PGPASSWORD='your_password' psql -U autoctl_node -h agg-mode-mainnet-postgres-1 -d agg_mode -c "SELECT 1" ``` ### Poller Health 1. **Check last processed block:** ```bash + # For Hoodi ssh app@agg-mode-hoodi-gateway-1 "cat ~/config/proof-aggregator.last_block_fetched.json" + + # For Mainnet + ssh app@agg-mode-mainnet-gateway-1 "cat ~/config/proof-aggregator.last_block_fetched.json" ``` The block number should increase over time. 2. **Check metrics:** ```bash + # For Hoodi curl http://agg-mode-hoodi-gateway-1:9095/metrics + + # For Mainnet + curl http://agg-mode-mainnet-gateway-1:9095/metrics ``` ### Metrics Stack @@ -434,12 +555,20 @@ ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p' 1. **Check tmux session is running:** ```bash + # For Hoodi make task_sender_status ENV=hoodi + + # For Mainnet + make task_sender_status ENV=mainnet ``` 2. **View recent logs:** ```bash + # For Hoodi ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p' + + # For Mainnet + ssh app@agg-mode-mainnet-sender 'tmux capture-pane -t task_sender -p' ``` 3. **Verify proof submissions:** @@ -455,12 +584,20 @@ ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p' Check monitor logs: ```bash +# For Hoodi ssh admin@agg-mode-hoodi-postgres-monitor "sudo journalctl -u pgautofailover -n 100" + +# For Mainnet +ssh admin@agg-mode-mainnet-postgres-monitor "sudo journalctl -u pgautofailover -n 100" ``` Check node logs: ```bash +# For Hoodi ssh admin@agg-mode-hoodi-postgres-1 "sudo journalctl -u pgautofailover -n 100" + +# For Mainnet +ssh admin@agg-mode-mainnet-postgres-1 "sudo journalctl -u pgautofailover -n 100" ``` **Problem: Password authentication fails** @@ -469,7 +606,11 @@ Verify `db_password` is set correctly in your environment config file (`config-h Check pg_hba.conf: ```bash +# For Hoodi ssh admin@agg-mode-hoodi-postgres-1 "sudo -u postgres cat /var/lib/postgresql/node/pg_hba.conf" + +# For Mainnet +ssh admin@agg-mode-mainnet-postgres-1 "sudo -u postgres cat /var/lib/postgresql/node/pg_hba.conf" ``` Should contain: @@ -483,7 +624,11 @@ host all all 100.64.0.0/10 scram-sha-256 Check logs for errors: ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "sudo journalctl -u gateway -n 100" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "sudo journalctl -u gateway -n 100" ``` Common issues: @@ -495,12 +640,20 @@ Common issues: Verify certificates exist: ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "ls -la ~/.ssl/" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "ls -la ~/.ssl/" ``` Check certificate validity: ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "openssl x509 -in ~/.ssl/cert.pem -text -noout" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "openssl x509 -in ~/.ssl/cert.pem -text -noout" ``` ### Poller Issues @@ -509,12 +662,20 @@ ssh app@agg-mode-hoodi-gateway-1 "openssl x509 -in ~/.ssl/cert.pem -text -noout" Check logs: ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "journalctl --user -u poller -n 100" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "journalctl --user -u poller -n 100" ``` Verify RPC connectivity: ```bash +# For Hoodi ssh app@agg-mode-hoodi-gateway-1 "curl -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}' https://aligned-hoodi-rpc-geth.tail665ae.ts.net" + +# For Mainnet +ssh app@agg-mode-mainnet-gateway-1 "curl -X POST -H 'Content-Type: application/json' --data '{\"jsonrpc\":\"2.0\",\"method\":\"eth_blockNumber\",\"params\":[],\"id\":1}' https://aligned-mainnet-rpc-1.tail665ae.ts.net" ``` ### Metrics Issues @@ -523,17 +684,29 @@ ssh app@agg-mode-hoodi-gateway-1 "curl -X POST -H 'Content-Type: application/jso Check Prometheus logs: ```bash +# For Hoodi ssh admin@agg-mode-hoodi-metrics "journalctl --user -u prometheus -n 100" + +# For Mainnet +ssh admin@agg-mode-mainnet-metrics "journalctl --user -u prometheus -n 100" ``` Verify targets are reachable from metrics server: ```bash +# For Hoodi ssh admin@agg-mode-hoodi-metrics "curl http://agg-mode-hoodi-gateway-1:9094/metrics" + +# For Mainnet +ssh admin@agg-mode-mainnet-metrics "curl http://agg-mode-mainnet-gateway-1:9094/metrics" ``` Check Prometheus config: ```bash +# For Hoodi ssh admin@agg-mode-hoodi-metrics "cat ~/config/prometheus.yaml" + +# For Mainnet +ssh admin@agg-mode-mainnet-metrics "cat ~/config/prometheus.yaml" ``` ### Task Sender Issues @@ -542,37 +715,57 @@ ssh admin@agg-mode-hoodi-metrics "cat ~/config/prometheus.yaml" Check if tmux session exists: ```bash +# For Hoodi ssh app@agg-mode-hoodi-sender "tmux list-sessions" + +# For Mainnet +ssh app@agg-mode-mainnet-sender "tmux list-sessions" ``` If missing, redeploy: ```bash +# For Hoodi make task_sender_deploy ENV=hoodi + +# For Mainnet +make task_sender_deploy ENV=mainnet ``` **Problem: Task sender crashes or exits** Check logs for errors: ```bash +# For Hoodi ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p -S -100' + +# For Mainnet +ssh app@agg-mode-mainnet-sender 'tmux capture-pane -t task_sender -p -S -100' ``` Common issues: - Invalid private key → Check `task_sender_private_key` in `config-{{ env }}.ini` - Missing proof/vk files → Verify files exist: `task_sender_proof_path`, `task_sender_vk_path` -- Network connectivity → Test RPC: `curl https://aligned-hoodi-rpc-geth.tail665ae.ts.net` +- Network connectivity → Test RPC: `curl https://aligned-hoodi-rpc-geth.tail665ae.ts.net` (Hoodi) or `curl https://aligned-mainnet-rpc-1.tail665ae.ts.net` (Mainnet) - Insufficient balance → Check account has ETH for gas fees **Problem: Proofs not being submitted** Check interval configuration: ```bash +# For Hoodi ssh app@agg-mode-hoodi-sender "cat ~/repos/sender/aligned_layer/scripts/.agg_mode.task_sender.env" + +# For Mainnet +ssh app@agg-mode-mainnet-sender "cat ~/repos/sender/aligned_layer/scripts/.agg_mode.task_sender.env" ``` Verify `INTERVAL_HOURS` is set correctly (default: 1 hour). Attach to session to see live activity: ```bash +# For Hoodi ssh app@agg-mode-hoodi-sender 'tmux attach -t task_sender' + +# For Mainnet +ssh app@agg-mode-mainnet-sender 'tmux attach -t task_sender' ``` **Problem: Deployment fails with insufficient balance** @@ -581,14 +774,24 @@ The automatic deposit check requires the account to have at least **0.0045 ETH** Check account balance: ```bash +# For Hoodi ssh app@agg-mode-hoodi-sender export PATH=$HOME/.foundry/bin:$PATH -cast balance --rpc-url +cast balance --rpc-url https://aligned-hoodi-rpc-geth.tail665ae.ts.net + +# For Mainnet +ssh app@agg-mode-mainnet-sender +export PATH=$HOME/.foundry/bin:$PATH +cast balance --rpc-url https://aligned-mainnet-rpc-1.tail665ae.ts.net ``` If balance is insufficient, send ETH to the account and redeploy: ```bash +# For Hoodi make task_sender_deploy ENV=hoodi + +# For Mainnet +make task_sender_deploy ENV=mainnet ``` **Problem: Automatic deposit fails** @@ -602,15 +805,27 @@ To manually deposit after fixing the issue: ```bash ssh app@agg-mode-hoodi-sender export PATH=$HOME/.cargo/bin:$PATH + +# For Hoodi agg_mode_cli deposit \ --network hoodi \ --rpc-url https://aligned-hoodi-rpc-geth.tail665ae.ts.net \ --private-key + +# For Mainnet +agg_mode_cli deposit \ + --network mainnet \ + --rpc-url https://aligned-mainnet-rpc-1.tail665ae.ts.net \ + --private-key ``` Then restart the task sender: ```bash +# For Hoodi make task_sender_restart ENV=hoodi + +# For Mainnet +make task_sender_restart ENV=mainnet ``` ### General Debugging @@ -622,13 +837,22 @@ tailscale status **Test SSH access to servers:** ```bash +# For Hoodi ssh admin@agg-mode-hoodi-postgres-monitor "echo 'Connection successful'" ssh app@agg-mode-hoodi-gateway-1 "echo 'Connection successful'" + +# For Mainnet +ssh admin@agg-mode-mainnet-postgres-monitor "echo 'Connection successful'" +ssh app@agg-mode-mainnet-gateway-1 "echo 'Connection successful'" ``` **Verify Ansible inventory:** ```bash +# For Hoodi ansible-inventory -i infra/aggregation_mode/ansible/hoodi-inventory.yaml --list + +# For Mainnet +ansible-inventory -i infra/aggregation_mode/ansible/mainnet-inventory.yaml --list ``` ## Advanced Usage @@ -638,24 +862,43 @@ ansible-inventory -i infra/aggregation_mode/ansible/hoodi-inventory.yaml --list You can run any playbook directly with ansible-playbook: ```bash -# Deploy only postgres monitor +# Deploy only postgres monitor (Hoodi) ansible-playbook infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml \ -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ -e "host=postgres_monitor" \ -e "env=hoodi" -# Deploy only gateway (no poller) +# Deploy only postgres monitor (Mainnet) +ansible-playbook infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml \ + -i infra/aggregation_mode/ansible/mainnet-inventory.yaml \ + -e "host=postgres_monitor" \ + -e "env=mainnet" + +# Deploy only gateway (no poller) - Hoodi ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ -e "host=gateway_primary" \ -e "env=hoodi" -# Deploy gateway with forced rebuild +# Deploy only gateway (no poller) - Mainnet +ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ + -i infra/aggregation_mode/ansible/mainnet-inventory.yaml \ + -e "host=gateway_primary" \ + -e "env=mainnet" + +# Deploy gateway with forced rebuild (Hoodi) ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ -e "host=gateway_primary" \ -e "env=hoodi" \ -e "force_rebuild=true" + +# Deploy gateway with forced rebuild (Mainnet) +ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ + -i infra/aggregation_mode/ansible/mainnet-inventory.yaml \ + -e "host=gateway_primary" \ + -e "env=mainnet" \ + -e "force_rebuild=true" ``` ### Updating Services @@ -665,12 +908,15 @@ ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ The easiest way to update services is using the `FORCE_REBUILD` parameter: ```bash -# Update both primary and secondary +# For Hoodi make gateway_deploy ENV=hoodi FORCE_REBUILD=true - -# Or update individually make gateway_primary_deploy ENV=hoodi FORCE_REBUILD=true make gateway_secondary_deploy ENV=hoodi FORCE_REBUILD=true + +# For Mainnet +make gateway_deploy ENV=mainnet FORCE_REBUILD=true +make gateway_primary_deploy ENV=mainnet FORCE_REBUILD=true +make gateway_secondary_deploy ENV=mainnet FORCE_REBUILD=true ``` This will: @@ -684,14 +930,29 @@ This will: If you prefer to update manually: ```bash -# Gateway +# Gateway (Hoodi) ssh app@agg-mode-hoodi-gateway-1 cd ~/repos/gateway/aligned_layer git pull origin staging cargo install --path aggregation_mode/gateway --bin gateway --features tls --locked sudo systemctl restart gateway -# Poller +# Gateway (Mainnet) +ssh app@agg-mode-mainnet-gateway-1 +cd ~/repos/gateway/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/gateway --bin gateway --features tls --locked +sudo systemctl restart gateway + +# Poller (Hoodi) +ssh app@agg-mode-hoodi-gateway-1 +cd ~/repos/poller/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/payments_poller --bin payments_poller --locked +systemctl --user restart poller + +# Poller (Mainnet) +ssh app@agg-mode-mainnet-gateway-1 cd ~/repos/poller/aligned_layer git pull origin staging cargo install --path aggregation_mode/payments_poller --bin payments_poller --locked @@ -703,7 +964,11 @@ systemctl --user restart poller **Idempotent deployment (skip if binary exists):** ```bash +# For Hoodi make gateway_deploy ENV=hoodi + +# For Mainnet +make gateway_deploy ENV=mainnet ``` This pulls the latest code but skips building if the binary already exists. Use this when you only want to update configuration files. @@ -711,7 +976,11 @@ This pulls the latest code but skips building if the binary already exists. Use **Force rebuild (always rebuild binaries):** ```bash +# For Hoodi make gateway_deploy ENV=hoodi FORCE_REBUILD=true + +# For Mainnet +make gateway_deploy ENV=mainnet FORCE_REBUILD=true ``` This always rebuilds binaries from the latest code, even if they already exist. Use this when you want to deploy code changes. @@ -721,9 +990,13 @@ This always rebuilds binaries from the latest code, even if they already exist. 1. Update INI files in `playbooks/ini/` 2. Redeploy the affected service: ```bash + # For Hoodi make gateway_deploy ENV=hoodi - # or make postgres_deploy ENV=hoodi + + # For Mainnet + make gateway_deploy ENV=mainnet + make postgres_deploy ENV=mainnet ``` ### Rotating Passwords @@ -733,12 +1006,20 @@ This always rebuilds binaries from the latest code, even if they already exist. - `grafana_postgres_password` (separate read-only user) 2. Run password update on PostgreSQL: ```bash + # For Hoodi ssh admin@agg-mode-hoodi-postgres-monitor "sudo -u postgres psql -d pg_auto_failover -c \"ALTER USER autoctl_node PASSWORD 'new_password'\"" + # For Mainnet + ssh admin@agg-mode-mainnet-postgres-monitor "sudo -u postgres psql -d pg_auto_failover -c \"ALTER USER autoctl_node PASSWORD 'new_password'\"" ``` 3. Redeploy gateway and metrics: ```bash + # For Hoodi make gateway_deploy ENV=hoodi make metrics_deploy ENV=hoodi + + # For Mainnet + make gateway_deploy ENV=mainnet + make metrics_deploy ENV=mainnet ``` ## File Structure From 3bda57e3215c3563bf1d1ecd54e22c27bcf93c34 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Fri, 16 Jan 2026 13:01:51 -0300 Subject: [PATCH 13/23] remove restart section --- Makefile | 33 --------------- infra/aggregation_mode/ansible/README.md | 51 ------------------------ 2 files changed, 84 deletions(-) diff --git a/Makefile b/Makefile index e915de8ab..619fbd6c4 100644 --- a/Makefile +++ b/Makefile @@ -1864,17 +1864,6 @@ task_sender_logs: ## View task sender logs. Usage: make task_sender_logs ENV=hoo @echo "Use: ssh app@agg-mode-$(ENV)-task-sender 'tmux attach -t task_sender'" @echo "Or: ssh app@agg-mode-$(ENV)-task-sender 'tmux capture-pane -t task_sender -p'" -.PHONY: task_sender_restart -task_sender_restart: ## Restart task sender. Usage: make task_sender_restart ENV=hoodi - @if [ -z "$(ENV)" ]; then \ - echo "Error: ENV must be set (hoodi or mainnet)"; \ - exit 1; \ - fi - @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/task_sender.yaml \ - -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -e "host=task_sender" \ - -e "env=$(ENV)" - # ------------------------------------------------------------------------------ # Full Deployment # ------------------------------------------------------------------------------ @@ -1888,25 +1877,3 @@ agg_mode_deploy_all: ## Deploy entire aggregation mode stack. Usage: make agg_mo @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/deploy_all.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ -e "env=$(ENV)" - -# ------------------------------------------------------------------------------ -# Service Management -# ------------------------------------------------------------------------------ - -.PHONY: gateway_restart -gateway_restart: ## Restart gateway service. Usage: make gateway_restart ENV=hoodi HOST=gateway_primary - @if [ -z "$(ENV)" ] || [ -z "$(HOST)" ]; then \ - echo "Error: ENV and HOST must be set"; \ - exit 1; \ - fi - @ansible $(HOST) -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -m shell -a "sudo systemctl restart gateway" --become - -.PHONY: poller_restart -poller_restart: ## Restart poller service. Usage: make poller_restart ENV=hoodi HOST=gateway_primary - @if [ -z "$(ENV)" ] || [ -z "$(HOST)" ]; then \ - echo "Error: ENV and HOST must be set"; \ - exit 1; \ - fi - @ansible $(HOST) -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -m shell -a "systemctl --user restart poller" diff --git a/infra/aggregation_mode/ansible/README.md b/infra/aggregation_mode/ansible/README.md index 5e73acfcd..b478e8b26 100644 --- a/infra/aggregation_mode/ansible/README.md +++ b/infra/aggregation_mode/ansible/README.md @@ -292,50 +292,8 @@ ssh app@agg-mode-mainnet-sender 'tmux attach -t task_sender' # Press Ctrl+B then D to detach without stopping ``` -**Restart task sender:** -```bash -# For Hoodi -make task_sender_restart ENV=hoodi - -# For Mainnet -make task_sender_restart ENV=mainnet -``` - ## Service Management -### Restart Services - -**Gateway:** -```bash -# For Hoodi -make gateway_restart ENV=hoodi HOST=gateway_primary -make gateway_restart ENV=hoodi HOST=gateway_secondary - -# For Mainnet -make gateway_restart ENV=mainnet HOST=gateway_primary -make gateway_restart ENV=mainnet HOST=gateway_secondary -``` - -**Poller:** -```bash -# For Hoodi -make poller_restart ENV=hoodi HOST=gateway_primary -make poller_restart ENV=hoodi HOST=gateway_secondary - -# For Mainnet -make poller_restart ENV=mainnet HOST=gateway_primary -make poller_restart ENV=mainnet HOST=gateway_secondary -``` - -**Task Sender:** -```bash -# For Hoodi -make task_sender_restart ENV=hoodi - -# For Mainnet -make task_sender_restart ENV=mainnet -``` - ### Check Service Status **PostgreSQL Cluster:** @@ -819,15 +777,6 @@ agg_mode_cli deposit \ --private-key ``` -Then restart the task sender: -```bash -# For Hoodi -make task_sender_restart ENV=hoodi - -# For Mainnet -make task_sender_restart ENV=mainnet -``` - ### General Debugging **Check Tailscale connectivity:** From 421e35ca2476f142356698b61e66fe22557cd0e4 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Fri, 16 Jan 2026 13:18:20 -0300 Subject: [PATCH 14/23] redeploy section and remove verification section --- infra/aggregation_mode/ansible/README.md | 370 +++++++++-------------- 1 file changed, 149 insertions(+), 221 deletions(-) diff --git a/infra/aggregation_mode/ansible/README.md b/infra/aggregation_mode/ansible/README.md index b478e8b26..45be3904a 100644 --- a/infra/aggregation_mode/ansible/README.md +++ b/infra/aggregation_mode/ansible/README.md @@ -9,8 +9,8 @@ This directory contains Ansible playbooks and configuration for automating the d - [Prerequisites](#prerequisites) - [Initial Setup](#initial-setup) - [Deployment](#deployment) +- [Redeployment](#redeployment) - [Service Management](#service-management) -- [Verification](#verification) - [Troubleshooting](#troubleshooting) - [Advanced Usage](#advanced-usage) @@ -292,6 +292,154 @@ ssh app@agg-mode-mainnet-sender 'tmux attach -t task_sender' # Press Ctrl+B then D to detach without stopping ``` +## Redeployment + +### Idempotent Deployment + +Idempotent deployment skips building if the binary already exists. Use this when you only want to update configuration files. + +```bash +# For Hoodi +make gateway_deploy ENV=hoodi + +# For Mainnet +make gateway_deploy ENV=mainnet +``` + +### Force Rebuild + +Force rebuild always rebuilds binaries from the latest code, even if they already exist. Use this when you want to deploy code changes. + +```bash +# For Hoodi +make gateway_deploy ENV=hoodi FORCE_REBUILD=true +make gateway_primary_deploy ENV=hoodi FORCE_REBUILD=true +make gateway_secondary_deploy ENV=hoodi FORCE_REBUILD=true + +# For Mainnet +make gateway_deploy ENV=mainnet FORCE_REBUILD=true +make gateway_primary_deploy ENV=mainnet FORCE_REBUILD=true +make gateway_secondary_deploy ENV=mainnet FORCE_REBUILD=true +``` + +This will: +1. Pull latest code from the configured branch (staging for hoodi, main for mainnet) +2. Delete existing binaries +3. Rebuild gateway and poller from source + +### Migrations + +To run database migrations: + +```bash +# For Hoodi +make postgres_migrations ENV=hoodi + +# For Mainnet +make postgres_migrations ENV=mainnet +``` + +### Task Sender + +To redeploy the task sender: + +```bash +# For Hoodi +make task_sender_deploy ENV=hoodi + +# For Mainnet +make task_sender_deploy ENV=mainnet +``` + +### Metrics Stack + +To redeploy the metrics stack (Prometheus and Grafana): + +```bash +# For Hoodi +make metrics_deploy ENV=hoodi +make prometheus_deploy ENV=hoodi +make grafana_deploy ENV=hoodi + +# For Mainnet +make metrics_deploy ENV=mainnet +make prometheus_deploy ENV=mainnet +make grafana_deploy ENV=mainnet +``` + +### Manual Update + +If you prefer to update manually: + +**Gateway:** +```bash +# Hoodi +ssh app@agg-mode-hoodi-gateway-1 +cd ~/repos/gateway/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/gateway --bin gateway --features tls --locked + +# Mainnet +ssh app@agg-mode-mainnet-gateway-1 +cd ~/repos/gateway/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/gateway --bin gateway --features tls --locked +``` + +**Poller:** +```bash +# Hoodi +ssh app@agg-mode-hoodi-gateway-1 +cd ~/repos/poller/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/payments_poller --bin payments_poller --locked + +# Mainnet +ssh app@agg-mode-mainnet-gateway-1 +cd ~/repos/poller/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/payments_poller --bin payments_poller --locked +``` + +**Task Sender:** +```bash +# Hoodi +ssh app@agg-mode-hoodi-sender +cd ~/repos/sender/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/cli --bin agg_mode_cli --locked + +# Mainnet +ssh app@agg-mode-mainnet-sender +cd ~/repos/sender/aligned_layer +git pull origin staging +cargo install --path aggregation_mode/cli --bin agg_mode_cli --locked +``` + +**Prometheus:** +```bash +# Hoodi +ssh admin@agg-mode-hoodi-metrics +# Update prometheus.yaml configuration manually +systemctl --user restart prometheus + +# Mainnet +ssh admin@agg-mode-mainnet-metrics +# Update prometheus.yaml configuration manually +systemctl --user restart prometheus +``` + +**Grafana:** +```bash +# Hoodi +ssh admin@agg-mode-hoodi-metrics +sudo systemctl restart grafana-server + +# Mainnet +ssh admin@agg-mode-mainnet-metrics +sudo systemctl restart grafana-server +``` + ## Service Management ### Check Service Status @@ -398,142 +546,6 @@ ssh app@agg-mode-mainnet-sender 'tmux capture-pane -t task_sender -p' # Press Ctrl+B then D to detach ``` -## Verification - -### PostgreSQL Cluster Health - -1. **Check cluster state:** - ```bash - # For Hoodi - make postgres_status ENV=hoodi - - # For Mainnet - make postgres_status ENV=mainnet - ``` - -2. **Test password authentication:** - ```bash - # For Hoodi - ssh admin@agg-mode-hoodi-postgres-1 "PGPASSWORD='your_password' psql -U autoctl_node -h localhost -d agg_mode -c 'SELECT 1'" - # For Mainnet - ssh admin@agg-mode-mainnet-postgres-1 "PGPASSWORD='your_password' psql -U autoctl_node -h localhost -d agg_mode -c 'SELECT 1'" - ``` - -3. **Verify replication:** - ```bash - # For Hoodi - ssh admin@agg-mode-hoodi-postgres-1 "sudo -u postgres psql -d agg_mode -c 'SELECT * FROM pg_stat_replication'" - - # For Mainnet - ssh admin@agg-mode-mainnet-postgres-1 "sudo -u postgres psql -d agg_mode -c 'SELECT * FROM pg_stat_replication'" - ``` - -4. **Test failover (optional):** - ```bash - # For Hoodi - ssh admin@agg-mode-hoodi-postgres-1 "sudo systemctl stop pgautofailover" - # Wait 30 seconds, check status - make postgres_status ENV=hoodi - # Secondary should now be primary - ssh admin@agg-mode-hoodi-postgres-1 "sudo systemctl start pgautofailover" - - # For Mainnet - ssh admin@agg-mode-mainnet-postgres-1 "sudo systemctl stop pgautofailover" - # Wait 30 seconds, check status - make postgres_status ENV=mainnet - # Secondary should now be primary - ssh admin@agg-mode-mainnet-postgres-1 "sudo systemctl start pgautofailover" - ``` - -### Gateway Health - -1. **Check HTTP health endpoint:** - ```bash - # For Hoodi - curl -k https://agg-mode-hoodi-gateway-1/ - - # For Mainnet - curl -k https://agg-mode-mainnet-gateway-1/ - ``` - -2. **Check metrics:** - ```bash - # For Hoodi - curl http://agg-mode-hoodi-gateway-1:9094/metrics - - # For Mainnet - curl http://agg-mode-mainnet-gateway-1:9094/metrics - ``` - -3. **Verify database connectivity:** - ```bash - # For Hoodi - ssh app@agg-mode-hoodi-gateway-1 - PGPASSWORD='your_password' psql -U autoctl_node -h agg-mode-hoodi-postgres-1 -d agg_mode -c "SELECT 1" - - # For Mainnet - ssh app@agg-mode-mainnet-gateway-1 - PGPASSWORD='your_password' psql -U autoctl_node -h agg-mode-mainnet-postgres-1 -d agg_mode -c "SELECT 1" - ``` - -### Poller Health - -1. **Check last processed block:** - ```bash - # For Hoodi - ssh app@agg-mode-hoodi-gateway-1 "cat ~/config/proof-aggregator.last_block_fetched.json" - - # For Mainnet - ssh app@agg-mode-mainnet-gateway-1 "cat ~/config/proof-aggregator.last_block_fetched.json" - ``` - - The block number should increase over time. - -2. **Check metrics:** - ```bash - # For Hoodi - curl http://agg-mode-hoodi-gateway-1:9095/metrics - - # For Mainnet - curl http://agg-mode-mainnet-gateway-1:9095/metrics - ``` - -### Metrics Stack - -1. **Prometheus targets:** - - Navigate to `http://:9090/targets` - - All targets should show as "UP" - -2. **Grafana datasources:** - - Navigate to `http://:3000` - - Go to Configuration → Data Sources - - Verify Prometheus and PostgreSQL datasources are connected - -### Task Sender - -1. **Check tmux session is running:** - ```bash - # For Hoodi - make task_sender_status ENV=hoodi - - # For Mainnet - make task_sender_status ENV=mainnet - ``` - -2. **View recent logs:** - ```bash - # For Hoodi - ssh app@agg-mode-hoodi-sender 'tmux capture-pane -t task_sender -p' - - # For Mainnet - ssh app@agg-mode-mainnet-sender 'tmux capture-pane -t task_sender -p' - ``` - -3. **Verify proof submissions:** - - Check logs for successful proof submissions - - Look for transaction hashes in the output - - Verify proofs are appearing on the network - ## Troubleshooting ### PostgreSQL Issues @@ -850,90 +862,6 @@ ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ -e "force_rebuild=true" ``` -### Updating Services - -**Update gateway and poller with latest code:** - -The easiest way to update services is using the `FORCE_REBUILD` parameter: - -```bash -# For Hoodi -make gateway_deploy ENV=hoodi FORCE_REBUILD=true -make gateway_primary_deploy ENV=hoodi FORCE_REBUILD=true -make gateway_secondary_deploy ENV=hoodi FORCE_REBUILD=true - -# For Mainnet -make gateway_deploy ENV=mainnet FORCE_REBUILD=true -make gateway_primary_deploy ENV=mainnet FORCE_REBUILD=true -make gateway_secondary_deploy ENV=mainnet FORCE_REBUILD=true -``` - -This will: -1. Pull latest code from the configured branch (staging for hoodi, main for mainnet) -2. Delete existing binaries -3. Rebuild gateway and poller from source -4. Restart the services - -**Manual update (alternative):** - -If you prefer to update manually: - -```bash -# Gateway (Hoodi) -ssh app@agg-mode-hoodi-gateway-1 -cd ~/repos/gateway/aligned_layer -git pull origin staging -cargo install --path aggregation_mode/gateway --bin gateway --features tls --locked -sudo systemctl restart gateway - -# Gateway (Mainnet) -ssh app@agg-mode-mainnet-gateway-1 -cd ~/repos/gateway/aligned_layer -git pull origin staging -cargo install --path aggregation_mode/gateway --bin gateway --features tls --locked -sudo systemctl restart gateway - -# Poller (Hoodi) -ssh app@agg-mode-hoodi-gateway-1 -cd ~/repos/poller/aligned_layer -git pull origin staging -cargo install --path aggregation_mode/payments_poller --bin payments_poller --locked -systemctl --user restart poller - -# Poller (Mainnet) -ssh app@agg-mode-mainnet-gateway-1 -cd ~/repos/poller/aligned_layer -git pull origin staging -cargo install --path aggregation_mode/payments_poller --bin payments_poller --locked -systemctl --user restart poller -``` - -### Redeploy with Latest Code - -**Idempotent deployment (skip if binary exists):** - -```bash -# For Hoodi -make gateway_deploy ENV=hoodi - -# For Mainnet -make gateway_deploy ENV=mainnet -``` - -This pulls the latest code but skips building if the binary already exists. Use this when you only want to update configuration files. - -**Force rebuild (always rebuild binaries):** - -```bash -# For Hoodi -make gateway_deploy ENV=hoodi FORCE_REBUILD=true - -# For Mainnet -make gateway_deploy ENV=mainnet FORCE_REBUILD=true -``` - -This always rebuilds binaries from the latest code, even if they already exist. Use this when you want to deploy code changes. - ### Changing Configuration 1. Update INI files in `playbooks/ini/` From 0010afe13270fa216ef7c09e8597e5f514f0a903 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Fri, 16 Jan 2026 13:20:24 -0300 Subject: [PATCH 15/23] update makefile --- Makefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 619fbd6c4..a41e24008 100644 --- a/Makefile +++ b/Makefile @@ -1667,9 +1667,7 @@ AGG_MODE_ANSIBLE_DIR = infra/aggregation_mode/ansible AGG_MODE_PLAYBOOKS_DIR = $(AGG_MODE_ANSIBLE_DIR)/playbooks AGG_MODE_INI_DIR = $(AGG_MODE_PLAYBOOKS_DIR)/ini -# ------------------------------------------------------------------------------ -# Setup: Create INI configuration files -# ------------------------------------------------------------------------------ +# TODO: Check and add targets to install gateway, poller and cli binaries locally # ------------------------------------------------------------------------------ # PostgreSQL Cluster Deployment From 5ef90c7f993b67cc4ecdd3ada3df0ef855456d64 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Fri, 16 Jan 2026 20:05:31 -0300 Subject: [PATCH 16/23] improve prometheus deploy --- grafana/provisioning/datasources/datasource.yaml | 2 +- .../aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml | 3 +++ .../aggregation_mode/ansible/playbooks/ini/config-hoodi.ini | 4 ++++ .../ansible/playbooks/ini/config-mainnet.ini | 4 ++++ .../ansible/playbooks/prometheus_agg_mode.yaml | 4 ++-- .../ansible/playbooks/templates/grafana/grafana_env.j2 | 5 +++++ .../templates/prometheus/prometheus_agg_mode.yaml.j2 | 4 ++-- 7 files changed, 21 insertions(+), 5 deletions(-) diff --git a/grafana/provisioning/datasources/datasource.yaml b/grafana/provisioning/datasources/datasource.yaml index 4afd2c4a7..32137e4a3 100644 --- a/grafana/provisioning/datasources/datasource.yaml +++ b/grafana/provisioning/datasources/datasource.yaml @@ -63,7 +63,7 @@ datasources: database: "${MONITOR_DB_DB}" user: "${MONITOR_DB_USER}" secureJsonData: - password: "" + password: ${MONITOR_DB_PASSWORD} basicAuth: false isDefault: false editable: true diff --git a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml index a0174f3a4..019916511 100644 --- a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml @@ -22,6 +22,9 @@ grafana_postgres_db: "{{ lookup('ini', 'grafana_postgres_db', file=config_file) }}" grafana_postgres_user: "{{ lookup('ini', 'grafana_postgres_user', file=config_file) }}" grafana_postgres_password: "{{ lookup('ini', 'grafana_postgres_password', file=config_file) }}" + grafana_monitor_host: "{{ lookup('ini', 'grafana_monitor_host', file=config_file) }}" + grafana_monitor_port: "{{ lookup('ini', 'grafana_monitor_port', file=config_file, default='5432') }}" + grafana_monitor_db: "{{ lookup('ini', 'grafana_monitor_db', file=config_file, default='pg_auto_failover') }}" no_log: true - name: Install required packages diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index aad6296c9..7e76643ac 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -84,6 +84,10 @@ grafana_postgres_host=agg-mode-hoodi-postgres-1 grafana_postgres_port=5432 grafana_postgres_db=agg_mode grafana_postgres_user=grafana +# Monitor datasource (uses same user/password as postgres datasource) +grafana_monitor_host=agg-mode-hoodi-postgres-monitor +grafana_monitor_port=5432 +grafana_monitor_db=pg_auto_failover # ============================================ # Task Sender Configuration diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index 587bd187f..9d42109d0 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -84,6 +84,10 @@ grafana_postgres_host=agg-mode-mainnet-postgres-1 grafana_postgres_port=5432 grafana_postgres_db=agg_mode grafana_postgres_user=grafana +# Monitor datasource (uses same user/password as postgres datasource) +grafana_monitor_host=agg-mode-mainnet-postgres-monitor +grafana_monitor_port=5432 +grafana_monitor_db=pg_auto_failover # ============================================ # Task Sender Configuration diff --git a/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml index 1375a1318..739f91b27 100644 --- a/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml @@ -73,10 +73,10 @@ owner: "{{ ansible_user }}" group: "{{ ansible_user }}" - - name: Enable and start Prometheus service + - name: Enable and restart Prometheus service systemd_service: name: prometheus - state: started + state: restarted enabled: true scope: user daemon_reload: true diff --git a/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 b/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 index f1a809492..87944f109 100644 --- a/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 +++ b/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 @@ -7,3 +7,8 @@ POSTGRES_PORT={{ grafana_postgres_port }} POSTGRES_DB={{ grafana_postgres_db }} POSTGRES_USER={{ grafana_postgres_user }} POSTGRES_PASSWORD={{ grafana_postgres_password }} +MONITOR_DB_HOST={{ grafana_monitor_host }} +MONITOR_DB_PORT={{ grafana_monitor_port }} +MONITOR_DB_DB={{ grafana_monitor_db }} +MONITOR_DB_USER={{ grafana_postgres_user }} +MONITOR_DB_PASSWORD={{ grafana_postgres_password }} diff --git a/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 index 2886c16fc..968cc3790 100644 --- a/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 +++ b/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 @@ -7,7 +7,7 @@ scrape_configs: static_configs: - targets: ["{{ gateway_primary_hostname }}:8080"] labels: - service: "gateway" + service: "gateway-http" instance: "primary" - job_name: "gateway-secondary-http" @@ -15,7 +15,7 @@ scrape_configs: static_configs: - targets: ["{{ gateway_secondary_hostname }}:8080"] labels: - service: "gateway" + service: "gateway-http" instance: "secondary" - job_name: "gateway-primary" From 13b178396015bbb9cdf66d3fdbbc2a8cb63c7f32 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Mon, 19 Jan 2026 11:50:20 -0300 Subject: [PATCH 17/23] rename primary/secondary to 1/2 --- infra/aggregation_mode/ansible/README.md | 34 +++++++++--------- .../ansible/hoodi-inventory.yaml | 24 ++++++------- .../ansible/mainnet-inventory.yaml | 24 ++++++------- .../ansible/playbooks/deploy_all.yaml | 8 ++--- .../ansible/playbooks/gateway.yaml | 4 +-- .../ansible/playbooks/grafana_agg_mode.yaml | 3 ++ .../ansible/playbooks/ini/config-hoodi.ini | 14 ++++---- .../ansible/playbooks/ini/config-mainnet.ini | 14 ++++---- .../ansible/playbooks/poller.yaml | 4 +-- .../ansible/playbooks/postgres_cluster.yaml | 10 +++--- .../playbooks/postgres_migrations.yaml | 4 +-- .../playbooks/prometheus_agg_mode.yaml | 4 +-- .../config-agg-mode-gateway.yaml.j2 | 4 +-- .../config-agg-mode-poller.yaml.j2 | 4 +-- .../templates/grafana/grafana_env.j2 | 1 + .../prometheus/prometheus_agg_mode.yaml.j2 | 36 +++++++++---------- 16 files changed, 100 insertions(+), 92 deletions(-) diff --git a/infra/aggregation_mode/ansible/README.md b/infra/aggregation_mode/ansible/README.md index 45be3904a..e50085a47 100644 --- a/infra/aggregation_mode/ansible/README.md +++ b/infra/aggregation_mode/ansible/README.md @@ -20,7 +20,7 @@ The Ansible automation deploys a complete aggregation mode stack consisting of: 1. **PostgreSQL Auto-Failover Cluster** (3 servers) - 1 Monitor node (EC2) - - 2 Data nodes (Primary + Secondary) with automatic failover (Scaleway Elastic Metal) + - 2 Data nodes with automatic failover (Scaleway Elastic Metal) - Password authentication with scram-sha-256 2. **Gateway Service** (2 servers) @@ -51,7 +51,7 @@ The Ansible automation deploys a complete aggregation mode stack consisting of: ├─────────────────────────────────────────────────────────────────┤ │ │ │ ┌──────────────┐ ┌──────────────┐ ┌──────────────┐ │ -│ │ PG Monitor │ │ PG Primary │ │ PG Secondary │ │ +│ │ PG Monitor │ │ PG Node 1 │ │ PG Node 2 │ │ │ │ (EC2) │ │ (Scaleway) │ │ (Scaleway) │ │ │ └──────┬───────┘ └──────┬───────┘ └──────┬───────┘ │ │ │ │ │ │ @@ -59,7 +59,7 @@ The Ansible automation deploys a complete aggregation mode stack consisting of: │ pg_auto_failover │ │ │ │ ┌────────────────────────┐ ┌────────────────────────┐ │ -│ │ Gateway Primary │ │ Gateway Secondary │ │ +│ │ Gateway 1 │ │ Gateway 2 │ │ │ │ ├─ Gateway (8080+443)│ │ ├─ Gateway (8080+443) │ │ │ │ └─ Poller │ │ └─ Poller │ │ │ └────────────────────────┘ └────────────────────────┘ │ @@ -152,7 +152,7 @@ make agg_mode_deploy_all ENV=mainnet ``` This will: -1. Deploy PostgreSQL cluster (monitor, primary, secondary) +1. Deploy PostgreSQL cluster (monitor, node 1, node 2) 2. Run database migrations 3. Deploy gateway and poller on both servers 4. Deploy Prometheus and Grafana @@ -175,7 +175,7 @@ make postgres_deploy ENV=mainnet This will: - Deploy monitor with scram-sha-256 auth - Set password for autoctl_node user -- Deploy primary and secondary nodes +- Deploy node 1 and node 2 - Configure replication with password auth - Run database migrations @@ -202,14 +202,14 @@ node_2 | 3 | 100.x.x.x:5432 | 1: 0/... | read-only | seconda ```bash # For Hoodi make gateway_deploy ENV=hoodi -make gateway_primary_deploy ENV=hoodi -make gateway_secondary_deploy ENV=hoodi +make gateway_1_deploy ENV=hoodi +make gateway_2_deploy ENV=hoodi make gateway_deploy ENV=hoodi FORCE_REBUILD=true # For Mainnet make gateway_deploy ENV=mainnet -make gateway_primary_deploy ENV=mainnet -make gateway_secondary_deploy ENV=mainnet +make gateway_1_deploy ENV=mainnet +make gateway_2_deploy ENV=mainnet make gateway_deploy ENV=mainnet FORCE_REBUILD=true ``` @@ -313,13 +313,13 @@ Force rebuild always rebuilds binaries from the latest code, even if they alread ```bash # For Hoodi make gateway_deploy ENV=hoodi FORCE_REBUILD=true -make gateway_primary_deploy ENV=hoodi FORCE_REBUILD=true -make gateway_secondary_deploy ENV=hoodi FORCE_REBUILD=true +make gateway_1_deploy ENV=hoodi FORCE_REBUILD=true +make gateway_2_deploy ENV=hoodi FORCE_REBUILD=true # For Mainnet make gateway_deploy ENV=mainnet FORCE_REBUILD=true -make gateway_primary_deploy ENV=mainnet FORCE_REBUILD=true -make gateway_secondary_deploy ENV=mainnet FORCE_REBUILD=true +make gateway_1_deploy ENV=mainnet FORCE_REBUILD=true +make gateway_2_deploy ENV=mainnet FORCE_REBUILD=true ``` This will: @@ -838,26 +838,26 @@ ansible-playbook infra/aggregation_mode/ansible/playbooks/pg_monitor.yaml \ # Deploy only gateway (no poller) - Hoodi ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ - -e "host=gateway_primary" \ + -e "host=gateway_1" \ -e "env=hoodi" # Deploy only gateway (no poller) - Mainnet ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ -i infra/aggregation_mode/ansible/mainnet-inventory.yaml \ - -e "host=gateway_primary" \ + -e "host=gateway_1" \ -e "env=mainnet" # Deploy gateway with forced rebuild (Hoodi) ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ -i infra/aggregation_mode/ansible/hoodi-inventory.yaml \ - -e "host=gateway_primary" \ + -e "host=gateway_1" \ -e "env=hoodi" \ -e "force_rebuild=true" # Deploy gateway with forced rebuild (Mainnet) ansible-playbook infra/aggregation_mode/ansible/playbooks/gateway.yaml \ -i infra/aggregation_mode/ansible/mainnet-inventory.yaml \ - -e "host=gateway_primary" \ + -e "host=gateway_1" \ -e "env=mainnet" \ -e "force_rebuild=true" ``` diff --git a/infra/aggregation_mode/ansible/hoodi-inventory.yaml b/infra/aggregation_mode/ansible/hoodi-inventory.yaml index f08c3dd5d..cb97ad198 100644 --- a/infra/aggregation_mode/ansible/hoodi-inventory.yaml +++ b/infra/aggregation_mode/ansible/hoodi-inventory.yaml @@ -7,8 +7,8 @@ postgres_monitor: ansible_user: admin ansible_python_interpreter: /usr/bin/python3 -# PostgreSQL Primary -postgres_primary: +# PostgreSQL Node 1 +postgres_1: hosts: agg-mode-hoodi-postgres-1: ansible_host: agg-mode-hoodi-postgres-1 @@ -16,8 +16,8 @@ postgres_primary: ansible_user: admin ansible_python_interpreter: /usr/bin/python3 -# PostgreSQL Secondary -postgres_secondary: +# PostgreSQL Node 2 +postgres_2: hosts: agg-mode-hoodi-postgres-2: ansible_host: agg-mode-hoodi-postgres-2 @@ -29,11 +29,11 @@ postgres_secondary: postgres_cluster: children: postgres_monitor: - postgres_primary: - postgres_secondary: + postgres_1: + postgres_2: -# Gateway Primary -gateway_primary: +# Gateway Node 1 +gateway_1: hosts: agg-mode-hoodi-gateway-1: ansible_host: agg-mode-hoodi-gateway-1 @@ -41,8 +41,8 @@ gateway_primary: ansible_user: app ansible_python_interpreter: /usr/bin/python3 -# Gateway Secondary -gateway_secondary: +# Gateway Node 2 +gateway_2: hosts: agg-mode-hoodi-gateway-2: ansible_host: agg-mode-hoodi-gateway-2 @@ -53,8 +53,8 @@ gateway_secondary: # Gateway Cluster (all gateway nodes) gateway_cluster: children: - gateway_primary: - gateway_secondary: + gateway_1: + gateway_2: # Metrics Server metrics: diff --git a/infra/aggregation_mode/ansible/mainnet-inventory.yaml b/infra/aggregation_mode/ansible/mainnet-inventory.yaml index 48ca062c8..2c6709380 100644 --- a/infra/aggregation_mode/ansible/mainnet-inventory.yaml +++ b/infra/aggregation_mode/ansible/mainnet-inventory.yaml @@ -7,8 +7,8 @@ postgres_monitor: ansible_user: admin ansible_python_interpreter: /usr/bin/python3 -# PostgreSQL Primary -postgres_primary: +# PostgreSQL Node 1 +postgres_1: hosts: agg-mode-mainnet-postgres-1: ansible_host: agg-mode-mainnet-postgres-1 @@ -16,8 +16,8 @@ postgres_primary: ansible_user: admin ansible_python_interpreter: /usr/bin/python3 -# PostgreSQL Secondary -postgres_secondary: +# PostgreSQL Node 2 +postgres_2: hosts: agg-mode-mainnet-postgres-2: ansible_host: agg-mode-mainnet-postgres-2 @@ -29,11 +29,11 @@ postgres_secondary: postgres_cluster: children: postgres_monitor: - postgres_primary: - postgres_secondary: + postgres_1: + postgres_2: -# Gateway Primary -gateway_primary: +# Gateway Node 1 +gateway_1: hosts: agg-mode-mainnet-gateway-1: ansible_host: agg-mode-mainnet-gateway-1 @@ -41,8 +41,8 @@ gateway_primary: ansible_user: app ansible_python_interpreter: /usr/bin/python3 -# Gateway Secondary -gateway_secondary: +# Gateway Node 2 +gateway_2: hosts: agg-mode-mainnet-gateway-2: ansible_host: agg-mode-mainnet-gateway-2 @@ -53,8 +53,8 @@ gateway_secondary: # Gateway Cluster (all gateway nodes) gateway_cluster: children: - gateway_primary: - gateway_secondary: + gateway_1: + gateway_2: # Metrics Server metrics: diff --git a/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml b/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml index 51e23a8a9..53a93ddeb 100644 --- a/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml +++ b/infra/aggregation_mode/ansible/playbooks/deploy_all.yaml @@ -3,16 +3,16 @@ vars: env: "{{ env }}" -- name: Deploy Gateway and Poller on Primary +- name: Deploy Gateway and Poller on Node 1 ansible.builtin.import_playbook: gateway_stack.yaml vars: - host: gateway_primary + host: gateway_1 env: "{{ env }}" -- name: Deploy Gateway and Poller on Secondary +- name: Deploy Gateway and Poller on Node 2 ansible.builtin.import_playbook: gateway_stack.yaml vars: - host: gateway_secondary + host: gateway_2 env: "{{ env }}" - name: Deploy Metrics Stack diff --git a/infra/aggregation_mode/ansible/playbooks/gateway.yaml b/infra/aggregation_mode/ansible/playbooks/gateway.yaml index 98f73e418..65661af9d 100644 --- a/infra/aggregation_mode/ansible/playbooks/gateway.yaml +++ b/infra/aggregation_mode/ansible/playbooks/gateway.yaml @@ -29,8 +29,8 @@ gateway_db_user: "{{ lookup('ini', 'db_user', file=config_file) }}" gateway_db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" gateway_db_name: "{{ lookup('ini', 'db_name', file=config_file) }}" - gateway_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file=config_file) }}" - gateway_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file=config_file) }}" + gateway_postgres_1: "{{ lookup('ini', 'gateway_postgres_1', file=config_file) }}" + gateway_postgres_2: "{{ lookup('ini', 'gateway_postgres_2', file=config_file) }}" gateway_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file=config_file, default='5432') }}" gateway_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file=config_file) }}" gateway_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file=config_file) }}" diff --git a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml index 019916511..523a7cce1 100644 --- a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml @@ -25,6 +25,9 @@ grafana_monitor_host: "{{ lookup('ini', 'grafana_monitor_host', file=config_file) }}" grafana_monitor_port: "{{ lookup('ini', 'grafana_monitor_port', file=config_file, default='5432') }}" grafana_monitor_db: "{{ lookup('ini', 'grafana_monitor_db', file=config_file, default='pg_auto_failover') }}" + gateway_1_hostname: "{{ lookup('ini', 'gateway_1_hostname', file=config_file) }}" + gateway_2_hostname: "{{ lookup('ini', 'gateway_2_hostname', file=config_file) }}" + grafana_gateway_url: "{{ lookup('ini', 'grafana_gateway_url', file=config_file) }}" no_log: true - name: Install required packages diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index 7e76643ac..8bfa18d0e 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -29,8 +29,8 @@ git_branch=staging # PostgreSQL Configuration # ============================================ postgres_monitor_hostname=agg-mode-hoodi-postgres-monitor -postgres_primary_hostname=agg-mode-hoodi-postgres-1 -postgres_secondary_hostname=agg-mode-hoodi-postgres-2 +postgres_1_hostname=agg-mode-hoodi-postgres-1 +postgres_2_hostname=agg-mode-hoodi-postgres-2 db_name=agg_mode db_user=autoctl_node @@ -51,8 +51,8 @@ gateway_max_daily_proofs=100 gateway_payment_service_address=0x7222E0183cE1A96619d0c883e9bfc6b76D4e780e gateway_eth_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net -gateway_postgres_primary=agg-mode-hoodi-postgres-1 -gateway_postgres_secondary=agg-mode-hoodi-postgres-2 +gateway_postgres_1=agg-mode-hoodi-postgres-1 +gateway_postgres_2=agg-mode-hoodi-postgres-2 gateway_postgres_port=5432 # Metrics ports @@ -74,8 +74,8 @@ last_block_fetched_initial_value=0 # Metrics Configuration # ============================================ prometheus_version=3.6.0 -gateway_primary_hostname=agg-mode-hoodi-gateway-1 -gateway_secondary_hostname=agg-mode-hoodi-gateway-2 +gateway_1_hostname=agg-mode-hoodi-gateway-1 +gateway_2_hostname=agg-mode-hoodi-gateway-2 # Grafana Configuration grafana_prometheus_url=http://localhost:9090 @@ -88,6 +88,8 @@ grafana_postgres_user=grafana grafana_monitor_host=agg-mode-hoodi-postgres-monitor grafana_monitor_port=5432 grafana_monitor_db=pg_auto_failover +# Public gateway URL +grafana_gateway_url=https://hoodi.gateway.alignedlayer.com # ============================================ # Task Sender Configuration diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index 9d42109d0..94abf89a3 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -29,8 +29,8 @@ git_branch=staging # PostgreSQL Configuration # ============================================ postgres_monitor_hostname=agg-mode-mainnet-postgres-monitor -postgres_primary_hostname=agg-mode-mainnet-postgres-1 -postgres_secondary_hostname=agg-mode-mainnet-postgres-2 +postgres_1_hostname=agg-mode-mainnet-postgres-1 +postgres_2_hostname=agg-mode-mainnet-postgres-2 db_name=agg_mode db_user=autoctl_node @@ -51,8 +51,8 @@ gateway_max_daily_proofs=100 gateway_payment_service_address=0xc8631Bc1E60c20db40e474F791126212fA8255F4 gateway_eth_rpc_url=https://aligned-mainnet-rpc-1.tail665ae.ts.net -gateway_postgres_primary=agg-mode-mainnet-postgres-1 -gateway_postgres_secondary=agg-mode-mainnet-postgres-2 +gateway_postgres_1=agg-mode-mainnet-postgres-1 +gateway_postgres_2=agg-mode-mainnet-postgres-2 gateway_postgres_port=5432 # Metrics ports @@ -74,8 +74,8 @@ last_block_fetched_initial_value=24235289 # Metrics Configuration # ============================================ prometheus_version=3.6.0 -gateway_primary_hostname=agg-mode-mainnet-gateway-1 -gateway_secondary_hostname=agg-mode-mainnet-gateway-2 +gateway_1_hostname=agg-mode-mainnet-gateway-1 +gateway_2_hostname=agg-mode-mainnet-gateway-2 # Grafana Configuration grafana_prometheus_url=http://localhost:9090 @@ -88,6 +88,8 @@ grafana_postgres_user=grafana grafana_monitor_host=agg-mode-mainnet-postgres-monitor grafana_monitor_port=5432 grafana_monitor_db=pg_auto_failover +# Public gateway URL +grafana_gateway_url=https://mainnet.gateway.alignedlayer.com # ============================================ # Task Sender Configuration diff --git a/infra/aggregation_mode/ansible/playbooks/poller.yaml b/infra/aggregation_mode/ansible/playbooks/poller.yaml index ee0e05953..f673a5533 100644 --- a/infra/aggregation_mode/ansible/playbooks/poller.yaml +++ b/infra/aggregation_mode/ansible/playbooks/poller.yaml @@ -24,8 +24,8 @@ poller_db_user: "{{ lookup('ini', 'db_user', file=config_file) }}" poller_db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" poller_db_name: "{{ lookup('ini', 'db_name', file=config_file) }}" - poller_postgres_primary: "{{ lookup('ini', 'gateway_postgres_primary', file=config_file) }}" - poller_postgres_secondary: "{{ lookup('ini', 'gateway_postgres_secondary', file=config_file) }}" + poller_postgres_1: "{{ lookup('ini', 'gateway_postgres_1', file=config_file) }}" + poller_postgres_2: "{{ lookup('ini', 'gateway_postgres_2', file=config_file) }}" poller_postgres_port: "{{ lookup('ini', 'gateway_postgres_port', file=config_file, default='5432') }}" poller_eth_rpc_url: "{{ lookup('ini', 'gateway_eth_rpc_url', file=config_file) }}" poller_payment_service_address: "{{ lookup('ini', 'gateway_payment_service_address', file=config_file) }}" diff --git a/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml b/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml index 9cc845212..a39345162 100644 --- a/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml +++ b/infra/aggregation_mode/ansible/playbooks/postgres_cluster.yaml @@ -4,20 +4,20 @@ host: postgres_monitor env: "{{ env }}" -- name: Deploy PostgreSQL Primary Node +- name: Deploy PostgreSQL Node 1 ansible.builtin.import_playbook: pg_node.yaml vars: - host: postgres_primary + host: postgres_1 env: "{{ env }}" -- name: Deploy PostgreSQL Secondary Node +- name: Deploy PostgreSQL Node 2 ansible.builtin.import_playbook: pg_node.yaml vars: - host: postgres_secondary + host: postgres_2 env: "{{ env }}" - name: Run Database Migrations ansible.builtin.import_playbook: postgres_migrations.yaml vars: - host: postgres_primary + host: postgres_1 env: "{{ env }}" diff --git a/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml b/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml index add485060..f3b8560cb 100644 --- a/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml +++ b/infra/aggregation_mode/ansible/playbooks/postgres_migrations.yaml @@ -23,7 +23,7 @@ db_name: "{{ lookup('ini', 'db_name', file=config_file) }}" db_user: "{{ lookup('ini', 'db_user', file=config_file) }}" db_password: "{{ lookup('ini', 'db_password', file=config_file) }}" - postgres_primary_hostname: "{{ lookup('ini', 'postgres_primary_hostname', file=config_file) }}" + postgres_1_hostname: "{{ lookup('ini', 'postgres_1_hostname', file=config_file) }}" git_branch: "{{ lookup('ini', 'git_branch', file=config_file) }}" no_log: true @@ -37,7 +37,7 @@ - name: Run database migrations shell: | export PATH=$HOME/.cargo/bin:$PATH - cargo run --manifest-path /home/{{ ansible_user }}/repos/migrations/aligned_layer/aggregation_mode/Cargo.toml --release --bin migrate -- "postgres://{{ db_user }}:{{ db_password }}@{{ postgres_primary_hostname }}:5432/{{ db_name }}" + cargo run --manifest-path /home/{{ ansible_user }}/repos/migrations/aligned_layer/aggregation_mode/Cargo.toml --release --bin migrate -- "postgres://{{ db_user }}:{{ db_password }}@{{ postgres_1_hostname }}:5432/{{ db_name }}" register: migration_result no_log: true diff --git a/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml index 739f91b27..2d94ab92a 100644 --- a/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/prometheus_agg_mode.yaml @@ -15,8 +15,8 @@ - name: Set config vars from INI file set_fact: prometheus_version: "{{ lookup('ini', 'prometheus_version', file=config_file, default='3.6.0') }}" - gateway_primary_hostname: "{{ lookup('ini', 'gateway_primary_hostname', file=config_file) }}" - gateway_secondary_hostname: "{{ lookup('ini', 'gateway_secondary_hostname', file=config_file) }}" + gateway_1_hostname: "{{ lookup('ini', 'gateway_1_hostname', file=config_file) }}" + gateway_2_hostname: "{{ lookup('ini', 'gateway_2_hostname', file=config_file) }}" gateway_metrics_port: "{{ lookup('ini', 'gateway_metrics_port', file=config_file, default='9094') }}" poller_metrics_port: "{{ lookup('ini', 'poller_metrics_port', file=config_file, default='9095') }}" diff --git a/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-gateway.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-gateway.yaml.j2 index fab21b85b..69a0d98f5 100644 --- a/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-gateway.yaml.j2 +++ b/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-gateway.yaml.j2 @@ -6,8 +6,8 @@ tls_key_path: "{{ gateway_tls_key_path }}" tls_port: {{ gateway_tls_port }} {% endif %} db_connection_urls: - - "postgres://{{ gateway_db_user }}:{{ gateway_db_password }}@{{ gateway_postgres_primary }}:{{ gateway_postgres_port }}/{{ gateway_db_name }}" - - "postgres://{{ gateway_db_user }}:{{ gateway_db_password }}@{{ gateway_postgres_secondary }}:{{ gateway_postgres_port }}/{{ gateway_db_name }}" + - "postgres://{{ gateway_db_user }}:{{ gateway_db_password }}@{{ gateway_postgres_1 }}:{{ gateway_postgres_port }}/{{ gateway_db_name }}" + - "postgres://{{ gateway_db_user }}:{{ gateway_db_password }}@{{ gateway_postgres_2 }}:{{ gateway_postgres_port }}/{{ gateway_db_name }}" eth_rpc_url: "{{ gateway_eth_rpc_url }}" payment_service_address: "{{ gateway_payment_service_address }}" network: "{{ gateway_network }}" diff --git a/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-poller.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-poller.yaml.j2 index 8f8114287..e67c6b8fe 100644 --- a/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-poller.yaml.j2 +++ b/infra/aggregation_mode/ansible/playbooks/templates/config-files/config-agg-mode-poller.yaml.j2 @@ -1,6 +1,6 @@ db_connection_urls: - - "postgres://{{ poller_db_user }}:{{ poller_db_password }}@{{ poller_postgres_primary }}:{{ poller_postgres_port }}/{{ poller_db_name }}" - - "postgres://{{ poller_db_user }}:{{ poller_db_password }}@{{ poller_postgres_secondary }}:{{ poller_postgres_port }}/{{ poller_db_name }}" + - "postgres://{{ poller_db_user }}:{{ poller_db_password }}@{{ poller_postgres_1 }}:{{ poller_postgres_port }}/{{ poller_db_name }}" + - "postgres://{{ poller_db_user }}:{{ poller_db_password }}@{{ poller_postgres_2 }}:{{ poller_postgres_port }}/{{ poller_db_name }}" eth_rpc_url: "{{ poller_eth_rpc_url }}" payment_service_address: "{{ poller_payment_service_address }}" network: "{{ poller_network }}" diff --git a/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 b/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 index 87944f109..8eef46a11 100644 --- a/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 +++ b/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 @@ -12,3 +12,4 @@ MONITOR_DB_PORT={{ grafana_monitor_port }} MONITOR_DB_DB={{ grafana_monitor_db }} MONITOR_DB_USER={{ grafana_postgres_user }} MONITOR_DB_PASSWORD={{ grafana_postgres_password }} +GATEWAY_URL={{ grafana_gateway_url }} diff --git a/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 index 968cc3790..c9c88aa44 100644 --- a/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 +++ b/infra/aggregation_mode/ansible/playbooks/templates/prometheus/prometheus_agg_mode.yaml.j2 @@ -2,50 +2,50 @@ global: scrape_interval: 15s scrape_configs: -- job_name: "gateway-primary-http" +- job_name: "gateway-1-http" scrape_interval: 60s static_configs: - - targets: ["{{ gateway_primary_hostname }}:8080"] + - targets: ["{{ gateway_1_hostname }}:8080"] labels: service: "gateway-http" - instance: "primary" + instance: "1" -- job_name: "gateway-secondary-http" +- job_name: "gateway-2-http" scrape_interval: 60s static_configs: - - targets: ["{{ gateway_secondary_hostname }}:8080"] + - targets: ["{{ gateway_2_hostname }}:8080"] labels: service: "gateway-http" - instance: "secondary" + instance: "2" -- job_name: "gateway-primary" +- job_name: "gateway-1" scrape_interval: 15s static_configs: - - targets: ["{{ gateway_primary_hostname }}:{{ gateway_metrics_port }}"] + - targets: ["{{ gateway_1_hostname }}:{{ gateway_metrics_port }}"] labels: service: "gateway" - instance: "primary" + instance: "1" -- job_name: "gateway-secondary" +- job_name: "gateway-2" scrape_interval: 15s static_configs: - - targets: ["{{ gateway_secondary_hostname }}:{{ gateway_metrics_port }}"] + - targets: ["{{ gateway_2_hostname }}:{{ gateway_metrics_port }}"] labels: service: "gateway" - instance: "secondary" + instance: "2" -- job_name: "poller-primary" +- job_name: "poller-1" scrape_interval: 15s static_configs: - - targets: ["{{ gateway_primary_hostname }}:{{ poller_metrics_port }}"] + - targets: ["{{ gateway_1_hostname }}:{{ poller_metrics_port }}"] labels: service: "poller" - instance: "primary" + instance: "1" -- job_name: "poller-secondary" +- job_name: "poller-2" scrape_interval: 15s static_configs: - - targets: ["{{ gateway_secondary_hostname }}:{{ poller_metrics_port }}"] + - targets: ["{{ gateway_2_hostname }}:{{ poller_metrics_port }}"] labels: service: "poller" - instance: "secondary" + instance: "2" From 0187c6825fdac8671e9568b8393e836887e8e266 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Mon, 19 Jan 2026 12:49:36 -0300 Subject: [PATCH 18/23] update grafana dashboard --- .../aligned/aggregation_mode_gateway.json | 5777 +++++++++-------- .../provisioning/datasources/datasource.yaml | 16 +- 2 files changed, 2938 insertions(+), 2855 deletions(-) diff --git a/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json b/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json index 6a57ce960..87907ef93 100644 --- a/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json +++ b/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json @@ -18,3235 +18,3307 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 8, + "id": 4, "links": [], - "liveNow": false, "panels": [ { - "collapsed": true, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { - "h": 1, + "h": 2, "w": 24, "x": 0, "y": 0 }, - "id": 9, - "panels": [ + "id": 51, + "links": [ { - "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - }, - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 0, - "y": 1 - }, - "id": 41, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN pg_is_in_recovery() = false THEN 1\n ELSE 0\n END AS up;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 + "targetBlank": true, + "title": "Explorer", + "url": "https://hood.explorer.alignedlayer.com" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n

ETHEREUM

\n
", + "mode": "html" + }, + "pluginVersion": "12.3.1", + "title": "", + "type": "text" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 } - } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "last" ], - "title": "Monitor Status", - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - }, - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "filters": [], + "format": "table", + "global_query_id": "", + "json_options": { + "root_is_not_array": true + }, + "method": "POST", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"\n| project \"block_number\"=\"number\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_blockNumber\",\n \"params\": [],\n \"id\": 1\n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 6, - "y": 1 - }, - "id": 42, + ], + "method": "POST" + } + } + ], + "title": "Current Block", + "transformations": [ + { + "id": "convertFieldType", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" + "conversions": [ + { + "destinationType": "number", + "targetField": "A" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT node.nodename, health\nFROM pgautofailover.node\nWHERE node.nodename = 'node1';", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 + { + "color": "orange", + "value": 5 + }, + { + "color": "red", + "value": 10 + }, + { + "color": "yellow", + "value": 20 } - } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 2 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Node 1 Status", - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - }, - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "filters": [], + "format": "table", + "global_query_id": "", + "json_options": { + "root_is_not_array": true + }, + "method": "POST", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_gasPrice\",\n \"params\": [],\n \"id\": 1\n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 12, - "y": 1 - }, - "id": 43, + ], + "method": "POST" + } + } + ], + "title": "Current Gas Price (GWEI)", + "transformations": [ + { + "id": "convertFieldType", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT node.nodename, health\nFROM pgautofailover.node\nWHERE node.nodename = 'node2';", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 + "conversions": [ + { + "destinationType": "number", + "targetField": "A" } - } - ], - "title": "Node 2 Status", - "type": "stat" + ], + "fields": {} + } }, { - "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "id": "calculateField", + "options": { + "alias": "GWEI", + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "A" + } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "operator": "*", + "right": { + "fixed": "1e-9" } }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 6, - "x": 18, - "y": 1 - }, - "id": 44, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "mode": "binary", + "reduce": { + "reducer": "sum" }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT\n CASE\n WHEN node.nodename = 'node1' THEN 1\n WHEN node.nodename = 'node2' THEN 2\n ELSE NULL\n END AS primary_db\nFROM pgautofailover.node\nWHERE node.reportedstate = 'primary'\nLIMIT 1;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } - } - ], - "title": "Selected Primary DB", - "type": "stat" - }, + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 46, + "links": [ { - "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" + "targetBlank": true, + "title": "Explorer", + "url": "https://hood.explorer.alignedlayer.com" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n

SYSTEM STATUS

\n
", + "mode": "html" + }, + "pluginVersion": "12.3.1", + "title": "", + "type": "text" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "http" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 6 - }, - "id": 40, - "options": { - "cellHeight": "sm", "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false + "reducers": [] }, - "showHeader": true, - "sortBy": [ - { - "desc": false, - "displayName": "reportedpgisrunning" - } - ] + "inspect": false }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "datasource": { - "type": "postgres", - "uid": "pgaf_monitor_host" + "options": { + "200": { + "color": "green", + "index": 0, + "text": "UP" + }, + "null": { + "color": "red", + "index": 1, + "text": "DOWN" + } }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT *\nFROM pgautofailover.node;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } + "type": "value" } ], - "title": "Complete monitor state", - "type": "table" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 0, + "y": 10 + }, + "id": 10, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.1", + "targets": [ { - "datasource": { - "type": "postgres", - "uid": "postgres" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "left", - "cellOptions": { - "type": "auto" - }, - "inspect": false - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 6 - }, - "id": 45, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.1.10", - "targets": [ + "columns": [ { - "datasource": { - "type": "postgres", - "uid": "postgres" - }, - "editorMode": "code", - "format": "table", - "rawQuery": true, - "rawSql": "SELECT version, description\nFROM _sqlx_migrations\nORDER BY version DESC LIMIT 1;", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - } + "selector": "$__response.statusCode", + "text": "status", + "type": "number" } ], - "title": "Schema Version", - "type": "table" + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "http" + }, + "filters": [], + "format": "as-is", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "", + "source": "url", + "type": "json", + "uql": "parse-json\n | extend status = $__response.statusCode", + "url": "https://hoodi.gateway.alignedlayer.com", + "url_options": { + "body_content_type": "text/plain", + "body_type": "raw", + "data": "", + "method": "GET" + } } ], - "title": "Database", - "type": "row" + "title": "Public Gateway Status (TODO)", + "type": "table" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 + "datasource": { + "type": "prometheus", + "uid": "prometheus" }, - "id": 11, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" }, - "thresholdsStyle": { - "mode": "off" + "1": { + "color": "green", + "index": 0, + "text": "UP" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 50 - }, - "id": 3, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "type": "value" } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "editorMode": "code", - "expr": "floor(\n increase(api_response_code{job=\"gateway-http\",statuscode=~\"4..\"}[10y])\n)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 5, + "y": 10 + }, + "id": 47, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "User error response count", - "type": "timeseries" + "fields": "", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "editorMode": "code", + "expr": "up{service=\"gateway\"}", + "instant": false, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Gateway Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" }, - "thresholdsStyle": { - "mode": "off" + "1": { + "color": "green", + "index": 0, + "text": "UP" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 50 - }, - "id": 2, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "type": "value" } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "editorMode": "code", - "expr": "floor(\n increase(api_response_code{job=\"gateway-http\",statuscode=~\"5..\"}[10y])\n)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 9, + "x": 14, + "y": 10 + }, + "id": 48, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Server Error response count", - "type": "timeseries" + "fields": "", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "editorMode": "code", + "expr": "up{service=\"poller\"}", + "instant": false, + "interval": "", + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Poller Status", + "type": "stat" + }, + { + "datasource": { + "type": "postgres", + "uid": "pgaf_monitor_host" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + { + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" }, - "thresholdsStyle": { - "mode": "off" + "1": { + "color": "green", + "index": 0, + "text": "UP" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 50 + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 3, + "x": 0, + "y": 17 + }, + "id": 41, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "pgaf_monitor_host" }, - "id": 1, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT\n CASE\n WHEN pg_is_in_recovery() = false THEN 1\n ELSE 0\n END AS up;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Postgres Monitor", + "type": "stat" + }, + { + "datasource": { + "type": "postgres", + "uid": "pgaf_monitor_host" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "hidden", + "axisSoftMax": 1.5, + "axisSoftMin": 0, + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ + "mappings": [ { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "options": { + "0": { + "color": "red", + "index": 1, + "text": "DOWN" + }, + "1": { + "color": "green", + "index": 0, + "text": "UP" + } }, - "editorMode": "code", - "expr": "floor(\n increase(api_response_code{job=\"gateway-http\",statuscode=~\"2..\",endpoint!=\"/metrics\"}[$__range])\n)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" + "type": "value" } ], - "title": "Success response count", - "type": "timeseries" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 5, + "x": 3, + "y": 17 + }, + "id": 43, + "options": { + "barRadius": 0, + "barWidth": 0.9, + "colorByField": "health", + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 0 + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "http" + "type": "postgres", + "uid": "pgaf_monitor_host" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "200": { - "color": "green", - "index": 0, - "text": "UP" - }, - "null": { - "color": "red", - "index": 1, - "text": "DOWN" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT UPPER(node.nodename), health\nFROM pgautofailover.node\nORDER BY node.nodename;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 6, - "x": 0, - "y": 58 + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Postgres Nodes Status", + "type": "barchart" + }, + { + "datasource": { + "type": "postgres", + "uid": "pgaf_monitor_host" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "id": 10, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" }, - "textMode": "auto" + "footer": { + "reducers": [] + }, + "inspect": false }, - "pluginVersion": "10.1.10", - "targets": [ - { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "http" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "", - "source": "url", - "type": "json", - "url": "http://host.docker.internal:8089", - "url_options": { - "data": "", - "method": "GET" + { + "color": "red", + "value": 80 } - } - ], - "title": "Gateway Status", - "type": "stat" + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 7, + "x": 8, + "y": 17 + }, + "id": 44, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "postgres", + "uid": "pgaf_monitor_host" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT node.nodename, node.nodehost\nFROM pgautofailover.node\nWHERE node.reportedstate = 'primary'\nLIMIT 1;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "type": "groupBy" } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 6, - "y": 58 + ], + "limit": 50 + } + } + ], + "title": "Selected Primary DB", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "align": "left", + "cellOptions": { + "type": "auto" }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "footer": { + "reducers": [] + }, + "inspect": false }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "editorMode": "code", - "expr": "histogram_quantile(0.50, sum by (le) (rate(time_elapsed_db_query_bucket{query=\"sp1-post\"}[10y])))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "DB posts latency (p50)", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "version" + }, + "properties": [ + { + "id": "custom.width", + "value": 134 + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 9, + "x": 15, + "y": 17 + }, + "id": 45, + "options": { + "cellHeight": "sm", + "showHeader": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "postgres", + "uid": "postgres" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT version, description\nFROM _sqlx_migrations\nORDER BY version DESC LIMIT 1;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + "type": "groupBy" } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 9, - "x": 15, - "y": 58 + ], + "limit": 50 + } + } + ], + "title": "Schema Version", + "type": "table" + }, + { + "datasource": { + "type": "postgres", + "uid": "pgaf_monitor_host" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "id": 5, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "footer": { + "reducers": [] + }, + "inspect": false }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "editorMode": "code", - "expr": "sum(rate(time_elapsed_db_query_count[10y]))", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "DB posts throughput", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 16, + "x": 8, + "y": 20 + }, + "id": 40, + "options": { + "cellHeight": "sm", + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "reportedpgisrunning" + } + ] + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "pgaf_monitor_host" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT nodename, nodehost, nodeport, goalstate, reportedstate, health\nFROM pgautofailover.node;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } ], - "title": "Gateway", - "type": "row" + "title": "Postgres Nodes Info", + "type": "table" }, { - "collapsed": true, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { - "h": 1, + "h": 2, "w": 24, "x": 0, - "y": 2 + "y": 24 }, - "id": 8, - "panels": [ + "id": 49, + "links": [ { - "datasource": { - "type": "postgres", - "uid": "postgres" + "targetBlank": true, + "title": "Explorer", + "url": "https://hood.explorer.alignedlayer.com" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n

GATEWAY

\n
", + "mode": "html" + }, + "pluginVersion": "12.3.1", + "title": "", + "type": "text" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "custom": { - "align": "auto", - "cellOptions": { - "type": "auto" - }, - "inspect": false + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + { + "color": "red", + "value": 80 } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 7, - "options": { - "cellHeight": "sm", - "footer": { - "countRows": false, - "fields": "", - "reducer": [ - "sum" - ], - "show": false - }, - "showHeader": true - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "postgres", - "uid": "postgres" - }, - "editorMode": "builder", - "format": "table", - "rawSql": "SELECT * FROM payment_events LIMIT 50 ", - "refId": "A", - "sql": { - "columns": [ - { - "parameters": [ - { - "name": "*", - "type": "functionParameter" - } - ], - "type": "function" - } - ], - "groupBy": [ - { - "property": { - "type": "string" - }, - "type": "groupBy" - } - ], - "limit": 50 - }, - "table": "payment_events" - } - ], - "title": "Payment events (placeholder)", - "type": "table" + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 26 + }, + "id": 3, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 12, - "y": 43 - }, - "id": 6, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "editorMode": "code", + "expr": "floor(\n increase(api_response_code{job=\"gateway-http\",statuscode=~\"4..\"}[$__range])\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "User Errors Response Count (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "editorMode": "code", - "expr": "last_processed_block{}", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Last Processed Block", - "type": "stat" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 26 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 5, - "x": 19, - "y": 43 - }, - "id": 15, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "editorMode": "code", + "expr": "floor(\n increase(api_response_code{job=\"gateway-http\",statuscode=~\"5..\"}[$__range])\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Server Errors Response Count (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "editorMode": "code", - "expr": "active_subscriptions{}", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Active Subscriptions", - "type": "stat" + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 26 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { "type": "prometheus", "uid": "prometheus" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "0": { - "color": "red", - "index": 1, - "text": "DOWN" - }, - "1": { - "color": "green", - "index": 0, - "text": "UP" - } - }, - "type": "value" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 7, - "x": 0, - "y": 49 - }, - "id": 39, - "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "prometheus" - }, - "editorMode": "code", - "expr": "up{bot=\"poller\"}", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Status", - "type": "stat" + "editorMode": "code", + "expr": "floor(\n increase(api_response_code{service=\"gateway-http\",statuscode=~\"2..\",endpoint!=\"/metrics\", endpoint!=\"/\"}[$__range])\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" } ], - "title": "Poller", - "type": "row" + "title": "Success Responses Count (2xx)", + "type": "timeseries" }, { - "collapsed": true, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { - "h": 1, + "h": 2, "w": 24, "x": 0, - "y": 3 + "y": 34 }, - "id": 12, - "panels": [], - "title": "Proof Aggregator", - "type": "row" + "id": 50, + "links": [ + { + "targetBlank": true, + "title": "Explorer", + "url": "https://hood.explorer.alignedlayer.com" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n

SUBSCRIPTIONS

\n
", + "mode": "html" + }, + "pluginVersion": "12.3.1", + "title": "", + "type": "text" }, { - "collapsed": true, + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, + "h": 6, + "w": 5, "x": 0, - "y": 4 + "y": 36 + }, + "id": 15, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "id": 13, - "panels": [ + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 53 - }, - "id": 21, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" + "type": "prometheus", + "uid": "prometheus" }, - "pluginVersion": "10.1.10", - "targets": [ - { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "root_is_not_array": true - }, - "method": "POST", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"\n| project \"block_number\"=\"number\"", - "url": "", - "url_options": { - "body_content_type": "application/json", - "body_type": "raw", - "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_blockNumber\",\n \"params\": [],\n \"id\": 1\n}", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } - } - ], - "title": "Current Block", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "number", - "targetField": "value" - } - ], - "fields": {} + "editorMode": "code", + "expr": "active_subscriptions{}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Active Subscriptions", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 } - } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 5, + "y": 36 + }, + "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "last_processed_block{}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Last Processed Block", + "type": "stat" + }, + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "footer": { + "reducers": [] + }, + "inspect": false }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + { + "color": "red", + "value": 80 } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "address" }, - "overrides": [] + "properties": [ + { + "id": "custom.width", + "value": 387 + } + ] }, - "gridPos": { - "h": 6, - "w": 8, - "x": 0, - "y": 59 + { + "matcher": { + "id": "byName", + "options": "started_at" + }, + "properties": [ + { + "id": "custom.width", + "value": 144 + } + ] }, - "id": 22, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false + { + "matcher": { + "id": "byName", + "options": "Address" }, - "textMode": "auto" + "properties": [ + { + "id": "custom.width", + "value": 378 + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "frameIndex": 0, + "showHeader": true, + "sortBy": [ + { + "desc": false, + "displayName": "valid_until" + } + ] + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "dataset": "agg_mode", + "datasource": { + "type": "postgres", + "uid": "postgres" }, - "pluginVersion": "10.1.10", - "targets": [ - { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "root_is_not_array": true - }, - "method": "POST", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "", - "url": "", - "url_options": { - "body_content_type": "application/json", - "body_type": "raw", - "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_gasPrice\",\n \"params\": [],\n \"id\": 1\n}", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT \n address as \"Address\", \n to_char(to_timestamp(started_at), 'YYYY-MM-DD HH24:MI:SS') as \"Started At\",\n to_char(to_timestamp(valid_until), 'YYYY-MM-DD HH24:MI:SS') as \"Valid Until\",\n tx_hash as \"TX Hash\"\nFROM payment_events\nORDER BY \"valid_until\" DESC\n;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" } - } - ], - "title": "Current Gas Price (wei)", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "number", - "targetField": "value" - } - ], - "fields": {} + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" } - } - ], - "type": "stat" + ], + "limit": 50 + } } ], - "title": "Ethereum", - "type": "row" + "title": "Payment events (placeholder)", + "type": "table" }, { - "collapsed": true, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, "gridPos": { - "h": 1, + "h": 2, "w": 24, "x": 0, - "y": 5 + "y": 42 }, - "id": 14, - "panels": [ + "id": 52, + "links": [ { - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "noValue": "0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 14, - "x": 5, - "y": 60 - }, - "id": 32, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + "targetBlank": true, + "title": "Explorer", + "url": "https://hood.explorer.alignedlayer.com" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n

PAYMENTS CONTRACT

\n
", + "mode": "html" + }, + "pluginVersion": "12.3.1", + "title": "", + "type": "text" + }, + { + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 11, + "x": 0, + "y": 44 + }, + "id": 32, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

${payments_contract}

", + "mode": "html" + }, + "pluginVersion": "12.3.1", + "title": "Address", + "type": "text" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "columnar": false, - "root_is_not_array": true + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "body_content_type": "application/json", - "body_type": "raw", - "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_call\",\n \"params\": [\n {\n \"to\": \"0xd84c0fEfEaEf69A51836DBf607E031a9033a4ed7\",\n \"data\": \"0x18160ddd\"\n },\n \"latest\"\n ],\n \"id\": 1\n }", - "method": "POST" - } + "type": "regex" } ], - "title": "Address", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "number", - "targetField": "value" - } - ], - "fields": {} + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 } - } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 11, + "y": 44 + }, + "id": 37, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "string" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 63 - }, - "id": 34, + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${payments_contract}\", \n \"data\": \"0xa4ece875\" \n }, \n \"latest\" ], \n \"id\": 1 \n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "method": "POST" + } + } + ], + "title": "Subscription Limit", + "transformations": [ + { + "id": "convertFieldType", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" + "conversions": [ + { + "destinationType": "number", + "targetField": "A" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0x922D6956C99E12DFeB3224DEA977D0939758A1Fe\", \"data\": \"0x07f56ad4\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } + "type": "regex" } ], - "title": "Cost in Gwei", - "transformations": [ - { - "id": "convertFieldType", - "options": {} - }, - { - "id": "calculateField", - "options": { - "binary": { - "left": "value", - "operator": "/", - "reducer": "sum", - "right": "1000000000" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 } - } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 15, + "y": 44 + }, + "id": 35, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "string" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 6, - "y": 63 - }, - "id": 33, + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${payments_contract}\", \n \"data\": \"0x93588bee\" \n }, \"latest\" ], \n \"id\": 1 \n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "method": "POST" + } + } + ], + "title": "Active Subscriptions", + "transformations": [ + { + "id": "convertFieldType", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" + "conversions": [ + { + "destinationType": "number", + "targetField": "A" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0x922D6956C99E12DFeB3224DEA977D0939758A1Fe\", \"data\": \"0x86925476\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } + "type": "regex" } ], - "title": "Expiration time in days", - "transformations": [ - { - "id": "convertFieldType", - "options": {} - }, - { - "id": "calculateField", - "options": { - "binary": { - "left": "value", - "operator": "/", - "reducer": "sum", - "right": "86400" - }, - "mode": "binary", - "reduce": { - "include": [ - "value" - ], - "reducer": "lastNotNull" - }, - "replaceFields": true + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 } - } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 5, + "x": 19, + "y": 44 + }, + "id": 34, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" + "filters": [], + "format": "table", + "global_query_id": "", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"${payments_contract}\", \"data\": \"0x07f56ad4\" }, \"latest\" ], \"id\": 1 }", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "method": "POST" + } + } + ], + "title": "Subscription Cost (Ether)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "value" + } + ], + "fields": {} + } + }, + { + "id": "calculateField", + "options": { + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "value" } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] }, - "unit": "string" + "operator": "*", + "right": { + "fixed": "1e-18" + } }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 12, - "y": 63 - }, - "id": 37, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false + "mode": "binary", + "reduce": { + "reducer": "sum" }, - "textMode": "auto" + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0x922D6956C99E12DFeB3224DEA977D0939758A1Fe\", \"data\": \"0xa4ece875\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } + "type": "regex" } ], - "title": "Subscription Limit", - "transformations": [ - { - "id": "convertFieldType", - "options": {} - } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 11, + "x": 0, + "y": 47 + }, + "id": 36, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "string" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 18, - "y": 63 - }, - "id": 38, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${payments_contract}\", \n \"data\": \"0x2c42a9dd\" \n }, \n \"latest\" \n ], \n \"id\": 1 \n }", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "method": "POST" + } + } + ], + "title": "Funds recipient", + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0x922D6956C99E12DFeB3224DEA977D0939758A1Fe\", \"data\": \"0x051c7de3\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } + "type": "regex" } ], - "title": "Max Subscription Time Ahead (in days)", - "transformations": [ - { - "id": "convertFieldType", - "options": {} - }, - { - "id": "calculateField", - "options": { - "binary": { - "left": "value", - "operator": "/", - "reducer": "sum", - "right": "86400" - }, - "mode": "binary", - "reduce": { - "reducer": "sum" - }, - "replaceFields": true + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 } - } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 11, + "y": 47 + }, + "id": 33, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${payments_contract}\", \n \"data\": \"0x86925476\" \n }, \n \"latest\" \n ],\n\"id\": 1 }", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "method": "POST" + } + } + ], + "title": "Expiration time in days", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "A" + } + ], + "fields": {} + } + }, + { + "id": "calculateField", + "options": { + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "A" } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] }, - "unit": "string" + "operator": "/", + "right": { + "fixed": "86400" + } }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 6, - "x": 0, - "y": 69 - }, - "id": 35, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "mode": "binary", + "reduce": { + "include": [ + "value" ], - "fields": "/.*/", - "values": false + "reducer": "lastNotNull" }, - "textMode": "auto" + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0x922D6956C99E12DFeB3224DEA977D0939758A1Fe\", \"data\": \"0x93588bee\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } + "type": "regex" } ], - "title": "Active Subscriptions", - "transformations": [ - { - "id": "convertFieldType", - "options": {} - } + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 15, + "y": 47 + }, + "id": 38, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${payments_contract}\", \n \"data\": \"0x051c7de3\" \n }, \n \"latest\" \n ], \n \"id\": 1 \n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "method": "POST" + } + } + ], + "title": "Max Subscription Time Ahead (in days)", + "transformations": [ + { + "id": "convertFieldType", + "options": { + "conversions": [ + { + "destinationType": "number", + "targetField": "A" + } + ], + "fields": {} + } + }, + { + "id": "calculateField", + "options": { + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "A" } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] }, - "unit": "string" + "operator": "/", + "right": { + "fixed": "86400" + } }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 9, - "x": 6, - "y": 69 - }, - "id": 36, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false + "mode": "binary", + "reduce": { + "reducer": "sum" }, - "textMode": "auto" + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 24, + "x": 0, + "y": 50 + }, + "id": 53, + "links": [ + { + "targetBlank": true, + "title": "Explorer", + "url": "https://hood.explorer.alignedlayer.com" + } + ], + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "
\n

PROOF AGGREGATOR CONTRACT

\n
", + "mode": "html" + }, + "pluginVersion": "12.3.1", + "title": "", + "type": "text" + }, + { + "description": "", + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 11, + "x": 0, + "y": 52 + }, + "id": 31, + "options": { + "code": { + "language": "plaintext", + "showLineNumbers": false, + "showMiniMap": false + }, + "content": "

${proof_aggregator_contract}

", + "mode": "html" + }, + "pluginVersion": "12.3.1", + "title": "Address", + "type": "text" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0x922D6956C99E12DFeB3224DEA977D0939758A1Fe\", \"data\": \"0x2c42a9dd\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } + "type": "regex" } ], - "title": "Funds recipient", - "transformations": [], - "type": "stat" - } - ], - "title": "Payments contract", - "type": "row" - }, - { - "collapsed": true, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 + "h": 3, + "w": 12, + "x": 11, + "y": 52 + }, + "id": 26, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, - "id": 16, - "panels": [ + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${proof_aggregator_contract}\",\n \"data\": \"0x8da5cb5b\" \n }, \n \"latest\" \n ], \n \"id\": 1 \n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 0, - "y": 55 - }, - "id": 24, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" + ], + "method": "POST" + } + } + ], + "title": "Owner", + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "columnar": false, - "root_is_not_array": true + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "body_content_type": "application/json", - "body_type": "raw", - "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_call\",\n \"params\": [\n {\n \"to\": \"0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc\",\n \"data\": \"0x972e58ba\"\n },\n \"latest\"\n ],\n \"id\": 1\n }", - "method": "POST" - } + "type": "regex" } ], - "title": "RISC0 ID", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "number", - "targetField": "value" - } - ], - "fields": {} + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 } - } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 11, + "x": 0, + "y": 55 + }, + "id": 29, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${proof_aggregator_contract}\", \n \"data\": \"0x4c46688c\" \n }, \n \"latest\" \n ], \n \"id\": 1 \n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" + } + ], + "method": "POST" + } + } + ], + "title": "Proof Aggregator", + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 4, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] + { + "color": "red", + "value": 0.001 }, - "unit": "string" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 7, - "y": 55 - }, - "id": 27, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + { + "color": "orange", + "value": 0.01 + }, + { + "color": "yellow", + "value": 0.1 }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc\", \"data\": \"0x616af4f7\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" + { + "color": "green", + "value": 1 } - } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 12, + "x": 11, + "y": 55 + }, + "id": 30, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "RISC0 Verifier", - "transformations": [], - "type": "stat" + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "noValue": "0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc", - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_getBalance\",\n \"params\": [\n \"${proof_aggregator_wallet}\",\n \"latest\"\n ],\n \"id\": 1\n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" } - }, - "overrides": [] - }, - "gridPos": { - "h": 3, - "w": 8, - "x": 16, - "y": 55 - }, - "id": 31, + ], + "method": "POST" + } + } + ], + "title": "Proof Aggregator Balance in ETH", + "transformations": [ + { + "id": "convertFieldType", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "columnar": false, - "root_is_not_array": true - }, - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "body_content_type": "application/json", - "body_type": "raw", - "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_call\",\n \"params\": [\n {\n \"to\": \"0xd84c0fEfEaEf69A51836DBf607E031a9033a4ed7\",\n \"data\": \"0x18160ddd\"\n },\n \"latest\"\n ],\n \"id\": 1\n }", - "method": "POST" - } - } - ], - "title": "Address", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "number", - "targetField": "value" - } - ], - "fields": {} + "conversions": [ + { + "destinationType": "number", + "targetField": "A" } - } - ], - "type": "stat" + ], + "fields": {} + } }, { - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "id": "calculateField", + "options": { + "binary": { + "left": { + "matcher": { + "id": "byName", + "options": "A" + } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] + "operator": "*", + "right": { + "fixed": "1e-18" } }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 0, - "y": 61 - }, - "id": 25, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "mode": "binary", + "reduce": { + "include": [ + "value" ], - "fields": "", - "values": false + "reducer": "sum" }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "filters": [], - "format": "table", - "global_query_id": "", - "json_options": { - "columnar": false, - "root_is_not_array": true - }, - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "url": "", - "url_options": { - "body_content_type": "application/json", - "body_type": "raw", - "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_call\",\n \"params\": [\n {\n \"to\": \"0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc\",\n \"data\": \"0x6eecb4c9\"\n },\n \"latest\"\n ],\n \"id\": 1\n }", - "method": "POST" - } - } - ], - "title": "SP1 ID", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "number", - "targetField": "value" - } - ], - "fields": {} + "replaceFields": true + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 } - } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 58 + }, + "id": 25, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "type": "stat" + "fields": "", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "string" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 7, - "y": 61 - }, - "id": 28, + "filters": [], + "format": "table", + "global_query_id": "", + "json_options": { + "columnar": false, + "root_is_not_array": true + }, + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_call\",\n \"params\": [\n {\n \"to\": \"${proof_aggregator_contract}\",\n \"data\": \"0x6eecb4c9\"\n },\n \"latest\"\n ],\n \"id\": 1\n }", + "method": "POST" + } + } + ], + "title": "SP1 ID", + "transformations": [ + { + "id": "convertFieldType", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" + "conversions": [ + { + "destinationType": "number", + "targetField": "A" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc\", \"data\": \"0x294e3ccb\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } + "type": "regex" } ], - "title": "SP1 Verifier", - "transformations": [], - "type": "stat" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 11, + "x": 3, + "y": 58 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "string" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 7, - "x": 0, - "y": 67 - }, - "id": 26, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc\", \"data\": \"0x8da5cb5b\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${proof_aggregator_contract}\", \n \"data\": \"0x294e3ccb\" \n }, \n \"latest\" ],\n \"id\": 1 \n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" } - } + ], + "method": "POST" + } + } + ], + "title": "SP1 Verifier Contract Address", + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 61 + }, + "id": 24, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Owner", - "transformations": [], - "type": "stat" + "fields": "", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "string" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 7, - "y": 67 - }, - "id": 29, + "filters": [], + "format": "table", + "global_query_id": "", + "json_options": { + "columnar": false, + "root_is_not_array": true + }, + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_call\",\n \"params\": [\n {\n \"to\": \"${proof_aggregator_contract}\",\n \"data\": \"0x972e58ba\"\n },\n \"latest\"\n ],\n \"id\": 1\n }", + "method": "POST" + } + } + ], + "title": "RISC0 ID", + "transformations": [ + { + "id": "convertFieldType", "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" + "conversions": [ + { + "destinationType": "number", + "targetField": "A" + } + ], + "fields": {} + } + } + ], + "type": "stat" + }, + { + "datasource": { + "type": "yesoreyeram-infinity-datasource", + "uid": "ethereum_rpc" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "10.1.10", - "targets": [ + "mappings": [ { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" + "options": { + "pattern": "^(.{2})0+(.{40})$", + "result": { + "index": 0, + "text": "$1$2" + } }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{ \"jsonrpc\": \"2.0\", \"method\": \"eth_call\", \"params\": [ { \"to\": \"0xcbEAF3BDe82155F56486Fb5a1072cb8baAf547cc\", \"data\": \"0x4c46688c\" }, \"latest\" ], \"id\": 1 }", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } + "type": "regex" } ], - "title": "Proof Aggregator", - "transformations": [], - "type": "stat" + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + }, + "unit": "string" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 11, + "x": 3, + "y": 61 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "/.*/", + "values": false }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ { + "columns": [], "datasource": { "type": "yesoreyeram-infinity-datasource", "uid": "ethereum_rpc" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - { - "options": { - "pattern": "^(.{2})0+(.{40})$", - "result": { - "index": 0, - "text": "$1$2" - } - }, - "type": "regex" - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "string" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 8, - "x": 15, - "y": 67 - }, - "id": 30, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "/.*/", - "values": false - }, - "textMode": "auto" - }, - "pluginVersion": "10.1.10", - "targets": [ - { - "columns": [], - "datasource": { - "type": "yesoreyeram-infinity-datasource", - "uid": "ethereum_rpc" - }, - "filters": [], - "format": "table", - "global_query_id": "", - "refId": "A", - "root_selector": "$.result", - "source": "url", - "type": "json", - "uql": "parse-json\n| scope \"result\"", - "url": "", - "url_options": { - "body_content_type": "text/plain", - "body_type": "raw", - "data": "{\n \"jsonrpc\": \"2.0\",\n \"method\": \"eth_getBalance\",\n \"params\": [\n \"0xa0ee7a142d267c1f36714e4a8f75612f20a79720\",\n \"latest\"\n ],\n \"id\": 1\n}", - "headers": [ - { - "key": "Content-Type", - "value": "application/json" - } - ], - "method": "POST" - } - } - ], - "title": "Proof Aggregator Balance in ETH", - "transformations": [ - { - "id": "convertFieldType", - "options": { - "conversions": [ - { - "destinationType": "number", - "targetField": "value" - } - ], - "fields": {} - } - }, - { - "id": "calculateField", - "options": { - "binary": { - "left": "value", - "operator": "/", - "reducer": "sum", - "right": "1000000000000000000" - }, - "mode": "binary", - "reduce": { - "include": [ - "value" - ], - "reducer": "sum" - }, - "replaceFields": true + "filters": [], + "format": "table", + "global_query_id": "", + "parser": "backend", + "refId": "A", + "root_selector": "$.result", + "source": "url", + "type": "json", + "uql": "parse-json\n| scope \"result\"", + "url": "", + "url_options": { + "body_content_type": "application/json", + "body_type": "raw", + "data": "{ \n \"jsonrpc\": \"2.0\", \n \"method\": \"eth_call\", \n \"params\": [ \n { \n \"to\": \"${proof_aggregator_contract}\", \n \"data\": \"0x616af4f7\" \n },\n \"latest\" \n ], \n \"id\": 1 \n}", + "headers": [ + { + "key": "Content-Type", + "value": "application/json" } - } - ], - "type": "stat" + ], + "method": "POST" + } } ], - "title": "Proof Aggregator contract", - "type": "row" + "title": "RISC0 Verifier Contractt Address", + "type": "stat" } ], + "preload": false, "refresh": "", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 42, "tags": [], "templating": { - "list": [] + "list": [ + { + "current": { + "text": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64", + "value": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64" + }, + "name": "payments_contract", + "options": [ + { + "selected": true, + "text": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64", + "value": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64" + } + ], + "query": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64", + "type": "custom" + }, + { + "current": { + "text": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59", + "value": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59" + }, + "name": "proof_aggregator_contract", + "options": [ + { + "selected": true, + "text": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59", + "value": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59" + } + ], + "query": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59", + "type": "custom" + }, + { + "current": { + "text": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc", + "value": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc" + }, + "name": "proof_aggregator_wallet", + "options": [ + { + "selected": true, + "text": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc", + "value": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc" + } + ], + "query": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc", + "type": "custom" + } + ] }, "time": { "from": "now-15m", @@ -3256,6 +3328,5 @@ "timezone": "", "title": "Aggregation Mode", "uid": "a66a5480-6a60-4b87-9d29-4f0f446edafd", - "version": 1, - "weekStart": "" -} + "version": 3 +} \ No newline at end of file diff --git a/grafana/provisioning/datasources/datasource.yaml b/grafana/provisioning/datasources/datasource.yaml index 32137e4a3..c69329b83 100644 --- a/grafana/provisioning/datasources/datasource.yaml +++ b/grafana/provisioning/datasources/datasource.yaml @@ -43,8 +43,20 @@ datasources: editable: true basicAuth: false - # Note: We use this data source for health check calls. The one above is not useful for non-Ethereum calls - # because the RPC URL is prepended to the one indicated in the panel. + - name: "Gateway" + type: "yesoreyeram-infinity-datasource" + typeName: "Infinity" + uid: "gateway" + access: "proxy" + url: "${GATEWAY_URL}" + isDefault: false + jsonData: + global_queries: [ ] + readOnly: false + editable: true + basicAuth: false + + # Note: We use this data source for general HTTP calls. - name: "HTTP" type: "yesoreyeram-infinity-datasource" uid: "http" From 3872bc627dfc3447e0bf954c51e7c85c71b72291 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Mon, 19 Jan 2026 14:17:24 -0300 Subject: [PATCH 19/23] set variables for grafana queries --- .../aligned/aggregation_mode_gateway.json | 53 +------ .../ansible/playbooks/grafana_agg_mode.yaml | 33 +++++ .../ansible/playbooks/ini/config-hoodi.ini | 4 + .../ansible/playbooks/ini/config-mainnet.ini | 4 + .../scripts/grafana_set_variables.sh | 137 ++++++++++++++++++ 5 files changed, 180 insertions(+), 51 deletions(-) create mode 100644 infra/aggregation_mode/ansible/playbooks/scripts/grafana_set_variables.sh diff --git a/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json b/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json index 87907ef93..b890171a0 100644 --- a/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json +++ b/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json @@ -3269,56 +3269,7 @@ "schemaVersion": 42, "tags": [], "templating": { - "list": [ - { - "current": { - "text": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64", - "value": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64" - }, - "name": "payments_contract", - "options": [ - { - "selected": true, - "text": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64", - "value": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64" - } - ], - "query": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64", - "type": "custom" - }, - { - "current": { - "text": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59", - "value": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59" - }, - "name": "proof_aggregator_contract", - "options": [ - { - "selected": true, - "text": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59", - "value": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59" - } - ], - "query": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59", - "type": "custom" - }, - { - "current": { - "text": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc", - "value": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc" - }, - "name": "proof_aggregator_wallet", - "options": [ - { - "selected": true, - "text": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc", - "value": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc" - } - ], - "query": "0x7EB3B63A4F3e7810Cc9bbc346749E2491Da4D7Cc", - "type": "custom" - } - ] + "list": [] }, "time": { "from": "now-15m", @@ -3328,5 +3279,5 @@ "timezone": "", "title": "Aggregation Mode", "uid": "a66a5480-6a60-4b87-9d29-4f0f446edafd", - "version": 3 + "version": 4 } \ No newline at end of file diff --git a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml index 523a7cce1..4e72d66da 100644 --- a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml @@ -28,6 +28,9 @@ gateway_1_hostname: "{{ lookup('ini', 'gateway_1_hostname', file=config_file) }}" gateway_2_hostname: "{{ lookup('ini', 'gateway_2_hostname', file=config_file) }}" grafana_gateway_url: "{{ lookup('ini', 'grafana_gateway_url', file=config_file) }}" + grafana_var_payments_contract: "{{ lookup('ini', 'grafana_var_payments_contract', file=config_file) }}" + grafana_var_proof_aggregator_contract: "{{ lookup('ini', 'grafana_var_proof_aggregator_contract', file=config_file) }}" + grafana_var_proof_aggregator_wallet: "{{ lookup('ini', 'grafana_var_proof_aggregator_wallet', file=config_file) }}" no_log: true - name: Install required packages @@ -38,6 +41,7 @@ - software-properties-common - wget - gnupg + - jq state: latest update_cache: true vars: @@ -142,3 +146,32 @@ daemon_reload: true vars: ansible_ssh_user: "{{ admin_user }}" + + - name: Wait for Grafana to be ready + uri: + url: "http://localhost:3000/api/health" + status_code: 200 + register: grafana_health + until: grafana_health.status == 200 + retries: 10 + delay: 5 + + - name: Copy dashboard variables script + become: true + copy: + src: scripts/grafana_set_variables.sh + dest: /tmp/grafana_set_variables.sh + mode: '0755' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Set dashboard variables via API + shell: | + /tmp/grafana_set_variables.sh \ + "http://localhost:3000" \ + "{{ grafana_admin_password }}" \ + "Aggregation Mode" \ + "payments_contract={{ grafana_var_payments_contract }}" \ + "proof_aggregator_contract={{ grafana_var_proof_aggregator_contract }}" \ + "proof_aggregator_wallet={{ grafana_var_proof_aggregator_wallet }}" + no_log: false diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index 8bfa18d0e..56dc784f0 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -90,6 +90,10 @@ grafana_monitor_port=5432 grafana_monitor_db=pg_auto_failover # Public gateway URL grafana_gateway_url=https://hoodi.gateway.alignedlayer.com +# Dashboard variables (contract addresses for dashboards) +grafana_var_payments_contract=0x7222E0183cE1A96619d0c883e9bfc6b76D4e780e +grafana_var_proof_aggregator_contract=0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59 +grafana_var_proof_aggregator_wallet=0x7eb3b63a4f3e7810cc9bbc346749e2491da4d7cc # ============================================ # Task Sender Configuration diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index 94abf89a3..b577b7c47 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -90,6 +90,10 @@ grafana_monitor_port=5432 grafana_monitor_db=pg_auto_failover # Public gateway URL grafana_gateway_url=https://mainnet.gateway.alignedlayer.com +# Dashboard variables (contract addresses for dashboards) +grafana_var_payments_contract=0xc8631Bc1E60c20db40e474F791126212fA8255F4 +grafana_var_proof_aggregator_contract=0xD0696d3eEebffcAB2D1b358805efAA005A9A8BC0 +grafana_var_proof_aggregator_wallet=0x57628fe929016C30463473349DcDd592ea27C8b3 # ============================================ # Task Sender Configuration diff --git a/infra/aggregation_mode/ansible/playbooks/scripts/grafana_set_variables.sh b/infra/aggregation_mode/ansible/playbooks/scripts/grafana_set_variables.sh new file mode 100644 index 000000000..8e96ed57c --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/scripts/grafana_set_variables.sh @@ -0,0 +1,137 @@ +#!/bin/bash +# grafana_set_variables.sh +# Sets dashboard variables in a Grafana dashboard via the Grafana API. +# +# Usage: +# ./grafana_set_variables.sh [key=value ...] +# +# The third argument can be either: +# - A dashboard UID (e.g., "a66a5480-6a60-4b87-9d29-4f0f446edafd") +# - A dashboard title to search for (e.g., "Aggregation Mode") +# +# Example: +# ./grafana_set_variables.sh "http://localhost:3000" "admin123" "a66a5480-6a60-4b87-9d29-4f0f446edafd" \ +# "payments_contract=0x1234..." \ +# "proof_aggregator_contract=0x5678..." \ +# "proof_aggregator_wallet=0xabcd..." + +set -e + +GRAFANA_URL="${1}" +ADMIN_PASSWORD="${2}" +DASHBOARD_ID="${3}" +shift 3 + +if [ -z "$GRAFANA_URL" ] || [ -z "$ADMIN_PASSWORD" ] || [ -z "$DASHBOARD_ID" ]; then + echo "Usage: $0 [key=value ...]" + exit 1 +fi + +AUTH="admin:${ADMIN_PASSWORD}" + +# Check if DASHBOARD_ID looks like a UID (contains dashes in UUID format) +if [[ "$DASHBOARD_ID" =~ ^[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}$ ]]; then + # It's a UID, use it directly + DASHBOARD_UID="$DASHBOARD_ID" + echo "Using dashboard UID: ${DASHBOARD_UID}" +else + # List all dashboards and find by title + DASHBOARD_TITLE="$DASHBOARD_ID" + SEARCH_RESULT=$(curl -s -u "${AUTH}" "${GRAFANA_URL}/api/search?type=dash-db") + + # Find dashboard UID by matching title + DASHBOARD_UID=$(echo "${SEARCH_RESULT}" | jq -r --arg title "$DASHBOARD_TITLE" '.[] | select(.title == $title) | .uid' | head -1) + + if [ -z "$DASHBOARD_UID" ]; then + echo "Error: Dashboard '${DASHBOARD_ID}' not found" + exit 1 + fi + + echo "Found dashboard '${DASHBOARD_ID}' with UID: ${DASHBOARD_UID}" +fi + +# Get full dashboard definition +DASHBOARD_RESPONSE=$(curl -s -u "${AUTH}" "${GRAFANA_URL}/api/dashboards/uid/${DASHBOARD_UID}") + +# Extract dashboard and meta from response +DASHBOARD=$(echo "${DASHBOARD_RESPONSE}" | jq '.dashboard') +FOLDER_ID=$(echo "${DASHBOARD_RESPONSE}" | jq '.meta.folderId') + +if [ "$DASHBOARD" = "null" ]; then + echo "Error: Could not retrieve dashboard definition" + exit 1 +fi + +# Process each key=value pair +for ARG in "$@"; do + KEY="${ARG%%=*}" + VALUE="${ARG#*=}" + + if [ -z "$KEY" ] || [ -z "$VALUE" ]; then + echo "Warning: Skipping invalid argument '${ARG}'" + continue + fi + + echo "Setting variable '${KEY}' = '${VALUE}'" + + # Check if variable already exists in templating.list + EXISTING_VAR=$(echo "${DASHBOARD}" | jq --arg key "$KEY" '.templating.list // [] | map(select(.name == $key)) | length') + + if [ "$EXISTING_VAR" -gt 0 ]; then + # Update existing variable + DASHBOARD=$(echo "${DASHBOARD}" | jq --arg key "$KEY" --arg val "$VALUE" ' + .templating.list = [ + .templating.list[] | + if .name == $key then + .query = $val | .current = {"text": $val, "value": $val} + else + . + end + ] + ') + else + # Add new variable + NEW_VAR=$(jq -n --arg key "$KEY" --arg val "$VALUE" '{ + "name": $key, + "type": "constant", + "hide": 2, + "query": $val, + "current": {"text": $val, "value": $val}, + "skipUrlSync": false + }') + + DASHBOARD=$(echo "${DASHBOARD}" | jq --argjson var "$NEW_VAR" ' + .templating.list = (.templating.list // []) + [$var] + ') + fi +done + +# Remove id and version to allow update (Grafana uses uid for identification) +DASHBOARD=$(echo "${DASHBOARD}" | jq 'del(.id) | del(.version)') + +# Create the save payload +PAYLOAD=$(jq -n \ + --argjson dashboard "$DASHBOARD" \ + --argjson folderId "$FOLDER_ID" \ + '{ + "dashboard": $dashboard, + "folderId": $folderId, + "overwrite": true + }') + +# Save the dashboard +SAVE_RESPONSE=$(curl -s -u "${AUTH}" \ + -H "Content-Type: application/json" \ + -X POST \ + "${GRAFANA_URL}/api/dashboards/db" \ + -d "${PAYLOAD}") + +# Check for errors +STATUS=$(echo "${SAVE_RESPONSE}" | jq -r '.status // empty') +if [ "$STATUS" = "success" ]; then + echo "Dashboard updated successfully" +else + ERROR_MSG=$(echo "${SAVE_RESPONSE}" | jq -r '.message // "Unknown error"') + echo "Error updating dashboard: ${ERROR_MSG}" + exit 1 +fi From 33973d0b347a490bab98815166f8af4b54becbfc Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Mon, 19 Jan 2026 16:12:52 -0300 Subject: [PATCH 20/23] fix makefile hosts --- Makefile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index a41e24008..cdf04e723 100644 --- a/Makefile +++ b/Makefile @@ -1702,11 +1702,11 @@ postgres_nodes_deploy: ## Deploy PostgreSQL Primary & Secondary. Usage: make pos fi @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/pg_node.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -e "host=postgres_primary" \ + -e "host=postgres_1" \ -e "env=$(ENV)" @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/pg_node.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -e "host=postgres_secondary" \ + -e "host=postgres_2" \ -e "env=$(ENV)" .PHONY: postgres_migrations @@ -1717,7 +1717,7 @@ postgres_migrations: ## Run database migrations. Usage: make postgres_migrations fi @ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/postgres_migrations.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -e "host=postgres_primary" \ + -e "host=postgres_1" \ -e "env=$(ENV)" .PHONY: postgres_status @@ -1745,7 +1745,7 @@ gateway_deploy: ## Deploy Gateway & Poller on both servers. Usage: make gateway_ fi; \ ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -e "host=gateway_primary" \ + -e "host=gateway_1" \ -e "env=$(ENV)" \ $$EXTRA_VARS @EXTRA_VARS=""; \ @@ -1754,12 +1754,12 @@ gateway_deploy: ## Deploy Gateway & Poller on both servers. Usage: make gateway_ fi; \ ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -e "host=gateway_secondary" \ + -e "host=gateway_2" \ -e "env=$(ENV)" \ $$EXTRA_VARS -.PHONY: gateway_primary_deploy -gateway_primary_deploy: ## Deploy Gateway & Poller on primary only. Usage: make gateway_primary_deploy ENV=hoodi [FORCE_REBUILD=true] +.PHONY: gateway_1_deploy +gateway_1_deploy: ## Deploy Gateway & Poller on gateway 1 only. Usage: make gateway_1_deploy ENV=hoodi [FORCE_REBUILD=true] @if [ -z "$(ENV)" ]; then \ echo "Error: ENV must be set (hoodi or mainnet)"; \ exit 1; \ @@ -1770,12 +1770,12 @@ gateway_primary_deploy: ## Deploy Gateway & Poller on primary only. Usage: make fi; \ ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -e "host=gateway_primary" \ + -e "host=gateway_1" \ -e "env=$(ENV)" \ $$EXTRA_VARS -.PHONY: gateway_secondary_deploy -gateway_secondary_deploy: ## Deploy Gateway & Poller on secondary only. Usage: make gateway_secondary_deploy ENV=hoodi [FORCE_REBUILD=true] +.PHONY: gateway_2_deploy +gateway_2_deploy: ## Deploy Gateway & Poller on gateway 2 only. Usage: make gateway_2_deploy ENV=hoodi [FORCE_REBUILD=true] @if [ -z "$(ENV)" ]; then \ echo "Error: ENV must be set (hoodi or mainnet)"; \ exit 1; \ @@ -1786,7 +1786,7 @@ gateway_secondary_deploy: ## Deploy Gateway & Poller on secondary only. Usage: m fi; \ ansible-playbook $(AGG_MODE_PLAYBOOKS_DIR)/gateway_stack.yaml \ -i $(AGG_MODE_ANSIBLE_DIR)/$(ENV)-inventory.yaml \ - -e "host=gateway_secondary" \ + -e "host=gateway_2" \ -e "env=$(ENV)" \ $$EXTRA_VARS From 3bbcc4a3048bc6e6d6ba61afe1f39a4814dc82c1 Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Mon, 19 Jan 2026 19:36:20 -0300 Subject: [PATCH 21/23] fix payments contract hoodi address --- infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index 56dc784f0..c136ef24e 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -48,7 +48,7 @@ backup_dir=/var/lib/backup # ============================================ gateway_network=hoodi gateway_max_daily_proofs=100 -gateway_payment_service_address=0x7222E0183cE1A96619d0c883e9bfc6b76D4e780e +gateway_payment_service_address=0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64 gateway_eth_rpc_url=https://aligned-hoodi-rpc-geth.tail665ae.ts.net gateway_postgres_1=agg-mode-hoodi-postgres-1 @@ -91,7 +91,7 @@ grafana_monitor_db=pg_auto_failover # Public gateway URL grafana_gateway_url=https://hoodi.gateway.alignedlayer.com # Dashboard variables (contract addresses for dashboards) -grafana_var_payments_contract=0x7222E0183cE1A96619d0c883e9bfc6b76D4e780e +grafana_var_payments_contract=0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64 grafana_var_proof_aggregator_contract=0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59 grafana_var_proof_aggregator_wallet=0x7eb3b63a4f3e7810cc9bbc346749e2491da4d7cc From 33e4ed94923e1af16d195d8f7bb0ae0bef9f03cd Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Tue, 20 Jan 2026 20:38:14 -0300 Subject: [PATCH 22/23] add alerts to grafana --- .../aligned/aggregation_mode_gateway.json | 948 +++++++++++++++--- .../provisioning/datasources/datasource.yaml | 12 +- .../ansible/playbooks/grafana_agg_mode.yaml | 46 + .../ansible/playbooks/ini/config-hoodi.ini | 3 + .../ansible/playbooks/ini/config-mainnet.ini | 3 + .../grafana/alerting/alert-rules.yaml.j2 | 514 ++++++++++ .../grafana/alerting/contact-points.yaml.j2 | 33 + .../alerting/notification-policies.yaml.j2 | 18 + .../templates/grafana/grafana_env.j2 | 2 + 9 files changed, 1407 insertions(+), 172 deletions(-) create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/alert-rules.yaml.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/contact-points.yaml.j2 create mode 100644 infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/notification-policies.yaml.j2 diff --git a/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json b/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json index b890171a0..9c8d4cb0f 100644 --- a/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json +++ b/grafana/provisioning/dashboards/aligned/aggregation_mode_gateway.json @@ -21,6 +21,76 @@ "id": 4, "links": [], "panels": [ + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 62, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "showInactiveAlerts": false, + "sortOrder": 3, + "stateFilter": { + "error": true, + "firing": true, + "noData": true, + "normal": false, + "pending": true, + "recovering": true + }, + "viewMode": "list" + }, + "pluginVersion": "12.3.1", + "title": "Firing Alerts", + "type": "alertlist" + }, + { + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 63, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "showInactiveAlerts": false, + "sortOrder": 3, + "stateFilter": { + "error": false, + "firing": false, + "noData": false, + "normal": true, + "pending": false, + "recovering": false + }, + "viewMode": "list" + }, + "pluginVersion": "12.3.1", + "title": "Alerts", + "type": "alertlist" + }, { "fieldConfig": { "defaults": {}, @@ -30,7 +100,7 @@ "h": 2, "w": 24, "x": 0, - "y": 0 + "y": 10 }, "id": 51, "links": [ @@ -80,7 +150,7 @@ "h": 6, "w": 8, "x": 0, - "y": 2 + "y": 12 }, "id": 21, "options": { @@ -194,7 +264,7 @@ "h": 6, "w": 8, "x": 8, - "y": 2 + "y": 12 }, "id": 22, "options": { @@ -299,7 +369,7 @@ "h": 2, "w": 24, "x": 0, - "y": 8 + "y": 18 }, "id": 46, "links": [ @@ -375,7 +445,7 @@ "h": 7, "w": 5, "x": 0, - "y": 10 + "y": 20 }, "id": 10, "options": { @@ -464,7 +534,7 @@ "h": 7, "w": 9, "x": 5, - "y": 10 + "y": 20 }, "id": 47, "options": { @@ -550,7 +620,7 @@ "h": 7, "w": 9, "x": 14, - "y": 10 + "y": 20 }, "id": 48, "options": { @@ -636,7 +706,7 @@ "h": 7, "w": 3, "x": 0, - "y": 17 + "y": 27 }, "id": 41, "options": { @@ -756,7 +826,7 @@ "h": 7, "w": 5, "x": 3, - "y": 17 + "y": 27 }, "id": 43, "options": { @@ -857,7 +927,7 @@ "h": 3, "w": 7, "x": 8, - "y": 17 + "y": 27 }, "id": 44, "options": { @@ -952,7 +1022,7 @@ "h": 3, "w": 9, "x": 15, - "y": 17 + "y": 27 }, "id": 45, "options": { @@ -1034,7 +1104,7 @@ "h": 4, "w": 16, "x": 8, - "y": 20 + "y": 30 }, "id": 40, "options": { @@ -1090,7 +1160,7 @@ "h": 2, "w": 24, "x": 0, - "y": 24 + "y": 34 }, "id": 49, "links": [ @@ -1178,7 +1248,7 @@ "h": 8, "w": 8, "x": 0, - "y": 26 + "y": 36 }, "id": 3, "options": { @@ -1209,64 +1279,574 @@ "refId": "A" } ], - "title": "User Errors Response Count (4xx)", - "type": "timeseries" + "title": "User Errors Response Count (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 36 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "floor(\n increase(api_response_code{job=\"gateway-http\",statuscode=~\"5..\"}[$__range])\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Server Errors Response Count (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "showValues": false, + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 36 + }, + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "hideZeros": false, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "floor(\n increase(api_response_code{service=\"gateway-http\",statuscode=~\"2..\",endpoint!=\"/metrics\", endpoint!=\"/\"}[$__range])\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Success Responses Count (2xx)", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 54, + "panels": [], + "title": "Proof Statistics (All Time)", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 45 + }, + "id": 55, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as count FROM tasks WHERE status = 'pending';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Pending Proofs", + "type": "stat" + }, + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 45 + }, + "id": 56, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as count FROM tasks WHERE status = 'processing';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Processing Proofs", + "type": "stat" + }, + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 45 + }, + "id": 57, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as count FROM tasks WHERE status = 'verified';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Verified Proofs", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 58, + "panels": [], + "title": "Proof Statistics (Last 24 Hours)", + "type": "row" + }, + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "yellow", + "value": 0 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 52 + }, + "id": 59, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "12.3.1", + "targets": [ + { + "datasource": { + "type": "postgres", + "uid": "postgres" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as count FROM tasks WHERE status = 'pending' AND status_updated_at > NOW() - INTERVAL '24 hours';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } + } + ], + "title": "Pending Proofs (24h)", + "type": "stat" }, { "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "postgres", + "uid": "postgres" }, "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green", + "color": "blue", "value": 0 - }, - { - "color": "red", - "value": 80 } ] } @@ -1274,86 +1854,72 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 8, "x": 8, - "y": 26 + "y": 52 }, - "id": 2, + "id": 60, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "postgres", + "uid": "postgres" }, "editorMode": "code", - "expr": "floor(\n increase(api_response_code{job=\"gateway-http\",statuscode=~\"5..\"}[$__range])\n)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as count FROM tasks WHERE status = 'processing' AND status_updated_at > NOW() - INTERVAL '24 hours';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } ], - "title": "Server Errors Response Count (5xx)", - "type": "timeseries" + "title": "Processing Proofs (24h)", + "type": "stat" }, { "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "postgres", + "uid": "postgres" }, "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "showValues": false, - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } + "mode": "thresholds" }, "mappings": [], "thresholds": { @@ -1362,10 +1928,6 @@ { "color": "green", "value": 0 - }, - { - "color": "red", - "value": 80 } ] } @@ -1373,42 +1935,62 @@ "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 8, "x": 16, - "y": 26 + "y": 52 }, - "id": 1, + "id": 61, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "tooltip": { - "hideZeros": false, - "mode": "single", - "sort": "none" - } + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true }, "pluginVersion": "12.3.1", "targets": [ { "datasource": { - "type": "prometheus", - "uid": "prometheus" + "type": "postgres", + "uid": "postgres" }, "editorMode": "code", - "expr": "floor(\n increase(api_response_code{service=\"gateway-http\",statuscode=~\"2..\",endpoint!=\"/metrics\", endpoint!=\"/\"}[$__range])\n)", - "instant": false, - "legendFormat": "__auto", - "range": true, - "refId": "A" + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(*) as count FROM tasks WHERE status = 'verified' AND status_updated_at > NOW() - INTERVAL '24 hours';", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } ], - "title": "Success Responses Count (2xx)", - "type": "timeseries" + "title": "Verified Proofs (24h)", + "type": "stat" }, { "fieldConfig": { @@ -1419,7 +2001,7 @@ "h": 2, "w": 24, "x": 0, - "y": 34 + "y": 58 }, "id": 50, "links": [ @@ -1473,7 +2055,7 @@ "h": 6, "w": 5, "x": 0, - "y": 36 + "y": 60 }, "id": 15, "options": { @@ -1538,7 +2120,7 @@ "h": 6, "w": 7, "x": 5, - "y": 36 + "y": 60 }, "id": 6, "options": { @@ -1654,7 +2236,7 @@ "h": 6, "w": 12, "x": 12, - "y": 36 + "y": 60 }, "id": 7, "options": { @@ -1712,7 +2294,7 @@ "h": 2, "w": 24, "x": 0, - "y": 42 + "y": 66 }, "id": 52, "links": [ @@ -1745,7 +2327,7 @@ "h": 3, "w": 11, "x": 0, - "y": 44 + "y": 68 }, "id": 32, "options": { @@ -1805,7 +2387,7 @@ "h": 3, "w": 4, "x": 11, - "y": 44 + "y": 68 }, "id": 37, "options": { @@ -1914,7 +2496,7 @@ "h": 3, "w": 4, "x": 15, - "y": 44 + "y": 68 }, "id": 35, "options": { @@ -2027,7 +2609,7 @@ "h": 3, "w": 5, "x": 19, - "y": 44 + "y": 68 }, "id": 34, "options": { @@ -2157,7 +2739,7 @@ "h": 3, "w": 11, "x": 0, - "y": 47 + "y": 71 }, "id": 36, "options": { @@ -2256,7 +2838,7 @@ "h": 3, "w": 4, "x": 11, - "y": 47 + "y": 71 }, "id": 33, "options": { @@ -2394,7 +2976,7 @@ "h": 3, "w": 6, "x": 15, - "y": 47 + "y": 71 }, "id": 38, "options": { @@ -2494,7 +3076,7 @@ "h": 2, "w": 24, "x": 0, - "y": 50 + "y": 74 }, "id": 53, "links": [ @@ -2527,7 +3109,7 @@ "h": 3, "w": 11, "x": 0, - "y": 52 + "y": 76 }, "id": 31, "options": { @@ -2587,7 +3169,7 @@ "h": 3, "w": 12, "x": 11, - "y": 52 + "y": 76 }, "id": 26, "options": { @@ -2686,7 +3268,7 @@ "h": 3, "w": 11, "x": 0, - "y": 55 + "y": 79 }, "id": 29, "options": { @@ -2787,7 +3369,7 @@ "h": 3, "w": 12, "x": 11, - "y": 55 + "y": 79 }, "id": 30, "options": { @@ -2909,7 +3491,7 @@ "h": 3, "w": 3, "x": 0, - "y": 58 + "y": 82 }, "id": 25, "options": { @@ -3019,7 +3601,7 @@ "h": 3, "w": 11, "x": 3, - "y": 58 + "y": 82 }, "id": 28, "options": { @@ -3102,7 +3684,7 @@ "h": 3, "w": 3, "x": 0, - "y": 61 + "y": 85 }, "id": 24, "options": { @@ -3208,7 +3790,7 @@ "h": 3, "w": 11, "x": 3, - "y": 61 + "y": 85 }, "id": 27, "options": { @@ -3269,7 +3851,41 @@ "schemaVersion": 42, "tags": [], "templating": { - "list": [] + "list": [ + { + "current": { + "text": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64", + "value": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64" + }, + "hide": 2, + "name": "payments_contract", + "query": "0xe6C9D0cf87cdaA8B2093c4b3830dde7267843F64", + "skipUrlSync": true, + "type": "constant" + }, + { + "current": { + "text": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59", + "value": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59" + }, + "hide": 2, + "name": "proof_aggregator_contract", + "query": "0x6B34AAaE780A5EAB4c91AB8F54f2a421E9c2FB59", + "skipUrlSync": true, + "type": "constant" + }, + { + "current": { + "text": "0x7eb3b63a4f3e7810cc9bbc346749e2491da4d7cc", + "value": "0x7eb3b63a4f3e7810cc9bbc346749e2491da4d7cc" + }, + "hide": 2, + "name": "proof_aggregator_wallet", + "query": "0x7eb3b63a4f3e7810cc9bbc346749e2491da4d7cc", + "skipUrlSync": true, + "type": "constant" + } + ] }, "time": { "from": "now-15m", @@ -3279,5 +3895,5 @@ "timezone": "", "title": "Aggregation Mode", "uid": "a66a5480-6a60-4b87-9d29-4f0f446edafd", - "version": 4 + "version": 21 } \ No newline at end of file diff --git a/grafana/provisioning/datasources/datasource.yaml b/grafana/provisioning/datasources/datasource.yaml index c69329b83..1f81249fc 100644 --- a/grafana/provisioning/datasources/datasource.yaml +++ b/grafana/provisioning/datasources/datasource.yaml @@ -19,14 +19,14 @@ datasources: access: proxy orgId: 1 url: "${POSTGRES_HOST}:${POSTGRES_PORT}" - database: ${POSTGRES_DB} - user: ${POSTGRES_USER} + user: "${POSTGRES_USER}" secureJsonData: - password: ${POSTGRES_PASSWORD} + password: "${POSTGRES_PASSWORD}" basicAuth: false isDefault: false editable: true jsonData: + database: "${POSTGRES_DB}" sslmode: disable postgresVersion: 1600 @@ -72,13 +72,13 @@ datasources: access: proxy orgId: 1 url: "${MONITOR_DB_HOST}:${MONITOR_DB_PORT}" - database: "${MONITOR_DB_DB}" user: "${MONITOR_DB_USER}" secureJsonData: - password: ${MONITOR_DB_PASSWORD} + password: "${MONITOR_DB_PASSWORD}" basicAuth: false isDefault: false editable: true jsonData: + database: "${MONITOR_DB_DB}" sslmode: require - postgresVersion: 1700 + postgresVersion: 1600 diff --git a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml index 4e72d66da..b56a689db 100644 --- a/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml +++ b/infra/aggregation_mode/ansible/playbooks/grafana_agg_mode.yaml @@ -31,6 +31,8 @@ grafana_var_payments_contract: "{{ lookup('ini', 'grafana_var_payments_contract', file=config_file) }}" grafana_var_proof_aggregator_contract: "{{ lookup('ini', 'grafana_var_proof_aggregator_contract', file=config_file) }}" grafana_var_proof_aggregator_wallet: "{{ lookup('ini', 'grafana_var_proof_aggregator_wallet', file=config_file) }}" + grafana_slack_webhook_url: "{{ lookup('ini', 'grafana_slack_webhook_url', file=config_file) }}" + grafana_pagerduty_integration_key: "{{ lookup('ini', 'grafana_pagerduty_integration_key', file=config_file) }}" no_log: true - name: Install required packages @@ -137,6 +139,50 @@ vars: ansible_ssh_user: "{{ admin_user }}" + - name: Create alerting provisioning directory + become: true + file: + path: /etc/grafana/provisioning/alerting + state: directory + owner: grafana + group: grafana + mode: '0755' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Template alerting contact points + become: true + template: + src: grafana/alerting/contact-points.yaml.j2 + dest: /etc/grafana/provisioning/alerting/contact-points.yaml + owner: grafana + group: grafana + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Template alerting notification policies + become: true + template: + src: grafana/alerting/notification-policies.yaml.j2 + dest: /etc/grafana/provisioning/alerting/notification-policies.yaml + owner: grafana + group: grafana + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + + - name: Template alerting rules + become: true + template: + src: grafana/alerting/alert-rules.yaml.j2 + dest: /etc/grafana/provisioning/alerting/alert-rules.yaml + owner: grafana + group: grafana + mode: '0644' + vars: + ansible_ssh_user: "{{ admin_user }}" + - name: Enable and start Grafana service become: true systemd_service: diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini index c136ef24e..a094f0074 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-hoodi.ini @@ -18,6 +18,9 @@ tls_key_source_path= grafana_admin_password= # Task sender private key (for sending proofs) task_sender_private_key= +# Alerting configuration (Slack and PagerDuty) +grafana_slack_webhook_url= +grafana_pagerduty_integration_key= # ============================================ # Environment diff --git a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini index b577b7c47..770bf100c 100644 --- a/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini +++ b/infra/aggregation_mode/ansible/playbooks/ini/config-mainnet.ini @@ -18,6 +18,9 @@ tls_key_source_path= grafana_admin_password= # Task sender private key (for sending proofs) task_sender_private_key= +# Alerting configuration (Slack and PagerDuty) +grafana_slack_webhook_url= +grafana_pagerduty_integration_key= # ============================================ # Environment diff --git a/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/alert-rules.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/alert-rules.yaml.j2 new file mode 100644 index 000000000..8b07a5604 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/alert-rules.yaml.j2 @@ -0,0 +1,514 @@ +apiVersion: 1 + +groups: + - orgId: 1 + name: service-availability + folder: Aggregation Mode Alerts + interval: 1m + rules: + # All Gateways Down - CRITICAL + - uid: all-gateways-down + title: All Gateways Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: sum(up{service="gateway"}) or vector(0) + instant: true + refId: A + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + datasource: + type: __expr__ + uid: __expr__ + expression: A + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + refId: C + type: threshold + noDataState: Alerting + execErrState: Alerting + for: 2m + labels: + severity: critical + service: gateway + annotations: + summary: All Gateway services are down + description: All Gateway instances have been down for more than 2 minutes. No gateway is available to process requests. + + # Gateway Instance Down - WARNING (fires per down instance) + - uid: gateway-instance-down + title: Gateway Instance Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: up{service="gateway"} + instant: true + refId: A + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + datasource: + type: __expr__ + uid: __expr__ + expression: A + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + refId: C + type: threshold + noDataState: OK + execErrState: Alerting + for: 2m + labels: + severity: warning + service: gateway + annotations: + summary: "Gateway {% raw %}{{ $labels.instance }}{% endraw %} is down" + description: "Gateway instance {% raw %}{{ $labels.instance }}{% endraw %} has been down for more than 2 minutes. Redundancy is degraded." + + # All Pollers Down - CRITICAL + - uid: all-pollers-down + title: All Pollers Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: sum(up{service="poller"}) or vector(0) + instant: true + refId: A + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + datasource: + type: __expr__ + uid: __expr__ + expression: A + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + refId: C + type: threshold + noDataState: Alerting + execErrState: Alerting + for: 2m + labels: + severity: critical + service: poller + annotations: + summary: All Poller services are down + description: All Poller instances have been down for more than 2 minutes. No poller is available to process proofs. + + # Poller Instance Down - WARNING (fires per down instance) + - uid: poller-instance-down + title: Poller Instance Down + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: up{service="poller"} + instant: true + refId: A + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + datasource: + type: __expr__ + uid: __expr__ + expression: A + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + refId: C + type: threshold + noDataState: OK + execErrState: Alerting + for: 2m + labels: + severity: warning + service: poller + annotations: + summary: "Poller {% raw %}{{ $labels.instance }}{% endraw %} is down" + description: "Poller instance {% raw %}{{ $labels.instance }}{% endraw %} has been down for more than 2 minutes. Redundancy is degraded." + + - orgId: 1 + name: database-health + folder: Aggregation Mode Alerts + interval: 1m + rules: + - uid: postgres-replication-unhealthy + title: PostgreSQL Replication Unhealthy + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: pgaf_monitor_host + model: + rawSql: | + SELECT + CASE + WHEN COUNT(*) FILTER (WHERE reportedstate = 'primary') >= 1 + AND COUNT(*) FILTER (WHERE reportedstate = 'secondary') >= 1 + THEN 1 + ELSE 0 + END as healthy + FROM pgautofailover.node + WHERE nodehost IS NOT NULL + refId: A + format: table + - refId: B + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + datasource: + type: __expr__ + uid: __expr__ + expression: A + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 1 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + refId: C + type: threshold + noDataState: Alerting + execErrState: Alerting + for: 5m + labels: + severity: critical + service: postgresql + annotations: + summary: PostgreSQL replication is unhealthy + description: The PostgreSQL cluster does not have both primary and secondary nodes healthy for more than 5 minutes + + - orgId: 1 + name: error-rates + folder: Aggregation Mode Alerts + interval: 1m + rules: + - uid: high-error-rate + title: High Proof Verification Error Rate + condition: C + data: + - refId: A + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: prometheus + model: + expr: sum(rate(proof_verification_errors_total[5m])) or vector(0) + instant: true + refId: A + - refId: B + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + datasource: + type: __expr__ + uid: __expr__ + expression: A + reducer: last + refId: B + type: reduce + - refId: C + relativeTimeRange: + from: 300 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0.1 + type: gt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: B + refId: C + type: threshold + noDataState: OK + execErrState: Alerting + for: 5m + labels: + severity: warning + service: proof-verification + annotations: + summary: High proof verification error rate detected + description: The proof verification error rate has exceeded 0.1 errors/second for more than 5 minutes + + - orgId: 1 + name: wallet-health + folder: Aggregation Mode Alerts + interval: 5m + rules: + # Balance check using hex string length as proxy + # eth_getBalance returns hex like "0x38D7EA4C68000" + # Using $length() since direct hex conversion is not supported + # Threshold: length < 16 means balance < 0x10000000000000 + # - 0xFFFFFFFFFFFFF (15 chars) = 4,503,599,627,370,495 wei ≈ 0.0045 ETH ≈ $13 USD + # - Alert fires when balance drops below ~0.0045 ETH + - uid: low-wallet-balance + title: Low Proof Aggregator Wallet Balance + condition: C + data: + - refId: A + relativeTimeRange: + from: 600 + to: 0 + datasourceUid: ethereum_rpc + model: + columns: [] + datasource: + type: yesoreyeram-infinity-datasource + uid: ethereum_rpc + filters: [] + format: table + global_query_id: "" + instant: true + intervalMs: 1000 + maxDataPoints: 43200 + parser: backend + refId: A + root_selector: "$length($.result)" + source: url + type: json + url: "" + url_options: + body_content_type: application/json + body_type: raw + data: '{"jsonrpc":"2.0","id":"1","method":"eth_getBalance","params":["{{ grafana_var_proof_aggregator_wallet }}","latest"]}' + method: POST + - refId: reducer + queryType: expression + relativeTimeRange: + from: 0 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 0 + - 0 + type: gt + operator: + type: and + query: + params: [] + reducer: + params: [] + type: avg + type: query + datasource: + name: Expression + type: __expr__ + uid: __expr__ + expression: A + intervalMs: 1000 + maxDataPoints: 43200 + reducer: last + refId: reducer + type: reduce + # Threshold: 16 chars = 0x10000000000000 ≈ 0.0045 ETH ≈ $13 USD + - refId: C + relativeTimeRange: + from: 0 + to: 0 + datasourceUid: __expr__ + model: + conditions: + - evaluator: + params: + - 16 + type: lt + operator: + type: and + query: + params: + - C + reducer: + params: [] + type: last + type: query + datasource: + type: __expr__ + uid: __expr__ + expression: reducer + intervalMs: 1000 + maxDataPoints: 43200 + refId: C + type: threshold + noDataState: NoData + execErrState: Error + for: 5m + labels: + severity: warning + service: wallet + annotations: + summary: Proof Aggregator wallet balance is low + description: The proof aggregator wallet balance is low. Please top up the wallet. diff --git a/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/contact-points.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/contact-points.yaml.j2 new file mode 100644 index 000000000..d7bd99fde --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/contact-points.yaml.j2 @@ -0,0 +1,33 @@ +apiVersion: 1 + +contactPoints: + - orgId: 1 + name: slack-alerts + receivers: + - uid: slack-alerts + type: slack + settings: + url: {{ grafana_slack_webhook_url }} + title: '[{{ "{{" }} .Status | toUpper {{ "}}" }}: {{ "{{" }} .CommonLabels.severity | toUpper {{ "}}" }}] {{ "{{" }} .CommonLabels.alertname {{ "}}" }}' + color: '{{ "{{" }} if eq .Status "resolved" {{ "}}" }}good{{ "{{" }} else if eq .CommonLabels.severity "critical" {{ "}}" }}danger{{ "{{" }} else if eq .CommonLabels.severity "warning" {{ "}}" }}warning{{ "{{" }} else {{ "}}" }}good{{ "{{" }} end {{ "}}" }}' + text: | + {{ "{{" }} range .Alerts {{ "}}" }} + *Alert:* {{ "{{" }} .Labels.alertname {{ "}}" }} + *Severity:* {{ "{{" }} .Labels.severity {{ "}}" }} + *Summary:* {{ "{{" }} .Annotations.summary {{ "}}" }} + *Description:* {{ "{{" }} .Annotations.description {{ "}}" }} + {{ "{{" }} end {{ "}}" }} +{% if grafana_pagerduty_integration_key %} + + - orgId: 1 + name: pagerduty-critical + receivers: + - uid: pagerduty-critical + type: pagerduty + settings: + integrationKey: {{ grafana_pagerduty_integration_key }} + severity: critical + class: aggregation-mode + component: '{{ "{{" }} .CommonLabels.service {{ "}}" }}' + group: aggregation-mode +{% endif %} diff --git a/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/notification-policies.yaml.j2 b/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/notification-policies.yaml.j2 new file mode 100644 index 000000000..1a4ac8586 --- /dev/null +++ b/infra/aggregation_mode/ansible/playbooks/templates/grafana/alerting/notification-policies.yaml.j2 @@ -0,0 +1,18 @@ +apiVersion: 1 + +policies: + - orgId: 1 + receiver: slack-alerts + group_by: + - alertname + - severity + group_wait: 30s + group_interval: 5m + repeat_interval: 4h +{% if grafana_pagerduty_integration_key %} + routes: + - receiver: pagerduty-critical + matchers: + - severity = critical + continue: true +{% endif %} diff --git a/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 b/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 index 8eef46a11..a30bb0a0f 100644 --- a/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 +++ b/infra/aggregation_mode/ansible/playbooks/templates/grafana/grafana_env.j2 @@ -13,3 +13,5 @@ MONITOR_DB_DB={{ grafana_monitor_db }} MONITOR_DB_USER={{ grafana_postgres_user }} MONITOR_DB_PASSWORD={{ grafana_postgres_password }} GATEWAY_URL={{ grafana_gateway_url }} +SLACK_WEBHOOK_URL={{ grafana_slack_webhook_url }} +PAGERDUTY_INTEGRATION_KEY={{ grafana_pagerduty_integration_key }} From 7e3b00886e392d60f796e1a27b262e45195f584e Mon Sep 17 00:00:00 2001 From: JuArce <52429267+JuArce@users.noreply.github.com> Date: Wed, 21 Jan 2026 12:08:53 -0300 Subject: [PATCH 23/23] skip ssl settings if already set --- .../aggregation_mode/ansible/playbooks/gateway.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/infra/aggregation_mode/ansible/playbooks/gateway.yaml b/infra/aggregation_mode/ansible/playbooks/gateway.yaml index 65661af9d..6eda20b35 100644 --- a/infra/aggregation_mode/ansible/playbooks/gateway.yaml +++ b/infra/aggregation_mode/ansible/playbooks/gateway.yaml @@ -53,6 +53,16 @@ vars: ansible_ssh_user: "{{ admin_user }}" + - name: Check if TLS certificate exists + stat: + path: "{{ gateway_tls_cert_path }}" + register: tls_cert_stat + + - name: Check if TLS key exists + stat: + path: "{{ gateway_tls_key_path }}" + register: tls_key_stat + - name: Create SSL directory file: path: /home/{{ ansible_user }}/.ssl @@ -60,6 +70,7 @@ mode: '0700' owner: "{{ ansible_user }}" group: "{{ ansible_user }}" + when: not tls_cert_stat.stat.exists or not tls_key_stat.stat.exists - name: Copy TLS certificate copy: @@ -68,6 +79,7 @@ mode: '0600' owner: "{{ ansible_user }}" group: "{{ ansible_user }}" + when: not tls_cert_stat.stat.exists - name: Copy TLS key copy: @@ -77,6 +89,7 @@ owner: "{{ ansible_user }}" group: "{{ ansible_user }}" no_log: true + when: not tls_key_stat.stat.exists - name: Clone aligned_layer repository git: