From f0f4276d4368ba0db70f222b5c064d427858feb5 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Fri, 16 Jan 2026 11:44:05 +0100 Subject: [PATCH 1/2] feat: add config file support to configure docker images --- .gitignore | 3 + README.md | 54 ++++++++++- client-cmds/default-client-config.yml | 22 +++++ client-cmds/grandine-cmd.sh | 5 +- client-cmds/lantern-cmd.sh | 5 +- client-cmds/lighthouse-cmd.sh | 5 +- client-cmds/qlean-cmd.sh | 8 +- client-cmds/ream-cmd.sh | 5 +- client-cmds/zeam-cmd.sh | 5 +- load-client-config.sh | 134 ++++++++++++++++++++++++++ parse-env.sh | 6 ++ spin-node.sh | 59 ++++++++++++ user-config.yml.example | 26 +++++ 13 files changed, 327 insertions(+), 10 deletions(-) create mode 100644 client-cmds/default-client-config.yml create mode 100644 load-client-config.sh create mode 100644 user-config.yml.example diff --git a/.gitignore b/.gitignore index a32ef5e..37e6f92 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ data *.old .DS_Store +# User configuration files +user-config.yml + # Generated genesis files (created by generate-genesis.sh) config.yaml validators.yaml diff --git a/README.md b/README.md index 3b2916a..70ecd54 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,44 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --generateGenesis --popupT NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --metrics ``` +### Using custom Docker images + +You can override default Docker images using the `--config-file` flag. This is useful for testing custom builds or using specific versions without modifying the codebase. + +**Basic usage (without custom images):** +```sh +# Uses default images from client-cmds/default-client-config.yml +NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis +``` + +**With custom config file:** +```sh +# Override specific client images +NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --config-file user-config.yml +``` + +**Example config file (user-config.yml):** +```yaml +clients: + - name: zeam + image: blockblaz/zeam:feature-branch + - name: ream + image: ghcr.io/reamlabs/ream:v2.0 +``` + +**Testing a specific client build:** +```sh +# Create custom config file for zeam /my-zeam-config.yml +clients: + - name: zeam + image: blockblaz/zeam:custom-tag + +# Run with custom zeam image +NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --config-file /my-zeam-config.yml +``` + +Only specify clients you want to override - others will use their defaults from `client-cmds/default-client-config.yml`. + ## Args 1. `NETWORK_DIR` is an env to specify the network directory. Should have a `genesis` directory with genesis config. A `data` folder will be created inside this `NETWORK_DIR` if not already there. @@ -119,6 +157,11 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --metrics - If not provided, defaults to `latest` for zeam, ream, and lantern, and `dd67521` for qlean - The script will automatically pull the specified Docker images before running containers - Example: `--tag devnet0` or `--tag devnet1` +11. `--config-file` specifies a custom configuration file to override default Docker images for specific clients. + - Path to a YAML file containing client image overrides (e.g., `user-config.yml` or `/path/to/my-config.yml`) + - Only clients specified in the config file are overridden; others use defaults from `client-cmds/default-client-config.yml` + - See [Using custom Docker images](#using-custom-docker-images) scenario for usage examples + - Example: `--config-file user-config.yml` or `--config-file /path/to/custom-config.yml` 12. `--metrics` enables metrics collection on all nodes. When specified, each client will activate its metrics endpoint according to its implementation. Metrics ports are configured per node in `validator-config.yaml`. ### Clients supported @@ -148,12 +191,19 @@ The quickstart uses separate directories for local and Ansible deployments: ``` lean-quickstart/ -├── local-devnet/ # Local development +├── client-cmds/ # Client command scripts +│ ├── default-client-config.yml # Default Docker images for all clients +│ ├── zeam-cmd.sh +│ ├── ream-cmd.sh +│ └── ... +├── user-config.yml.example # Example custom config (copy to user-config.yml) +├── user-config.yml # Your custom image overrides (gitignored) +├── local-devnet/ # Local development │ ├── genesis/ │ │ └── validator-config.yaml # Local IPs (127.0.0.1) │ └── data/ # Node data directories │ -└── ansible-devnet/ # Ansible/remote deployment +└── ansible-devnet/ # Ansible/remote deployment ├── genesis/ │ └── validator-config.yaml # Remote IPs (your server IPs) └── data/ # Node data directories diff --git a/client-cmds/default-client-config.yml b/client-cmds/default-client-config.yml new file mode 100644 index 0000000..cc700c0 --- /dev/null +++ b/client-cmds/default-client-config.yml @@ -0,0 +1,22 @@ +# Default Client Configuration +# This file contains the default Docker images for all supported clients +# These defaults are used when no custom config file is specified via --config-file + +clients: + - name: zeam + image: blockblaz/zeam:latest + + - name: ream + image: ghcr.io/reamlabs/ream:latest + + - name: qlean + image: qdrvm/qlean-mini:latest + + - name: lantern + image: piertwo/lantern:latest + + - name: lighthouse + image: hopinheimer/lighthouse:latest + + - name: grandine + image: sifrai/lean:latest diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index b8456a6..f07e6a8 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Docker image (set from default-client-config.yml or user config via --config-file) +# grandineImage is exported by spin-node.sh before sourcing this file + node_binary="$grandine_bin \ --genesis $configDir/config.yaml \ --validator-registry-path $configDir/validators.yaml \ @@ -10,7 +13,7 @@ node_binary="$grandine_bin \ --address 0.0.0.0 \ --hash-sig-key-dir $configDir/hash-sig-keys" -node_docker="sifrai/lean:unstable \ +node_docker="$grandineImage \ --genesis /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --bootnodes /config/nodes.yaml \ diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index 3075a95..cd3dcc3 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -1,7 +1,8 @@ #!/bin/bash #-----------------------lantern setup---------------------- -LANTERN_IMAGE="piertwo/lantern:v0.0.1" +# Docker image (set from default-client-config.yml or user config via --config-file) +# lanternImage is exported by spin-node.sh before sourcing this file devnet_flag="" if [ -n "$devnet" ]; then @@ -24,7 +25,7 @@ node_binary="$scriptDir/lantern/build/lantern_cli \ --log-level debug \ --hash-sig-key-dir $configDir/hash-sig-keys" -node_docker="$LANTERN_IMAGE --data-dir /data \ +node_docker="$lanternImage --data-dir /data \ --genesis-config /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --genesis-state /config/genesis.ssz \ diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index 1e129c2..51938c6 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -3,6 +3,9 @@ # Metrics enabled by default metrics_flag="--metrics" +# Docker image (set from default-client-config.yml or user config via --config-file) +# lighthouseImage is exported by spin-node.sh before sourcing this file + node_binary="$lighthouse_bin lean_node \ --datadir \"$dataDir/$item\" \ --config \"$configDir/config.yaml\" \ @@ -16,7 +19,7 @@ node_binary="$lighthouse_bin lean_node \ --metrics-address 0.0.0.0 \ --metrics-port $metricsPort" -node_docker="hopinheimer/lighthouse:latest lighthouse lean_node \ +node_docker="$lighthouseImage lighthouse lean_node \ --datadir /data \ --config /config/config.yaml \ --validators /config/validator-config.yaml \ diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index 178ca98..a90b775 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -3,6 +3,10 @@ #-----------------------qlean setup---------------------- # expects "qlean" submodule or symlink inside "lean-quickstart" root directory # https://github.com/qdrvm/qlean-mini + +# Docker image (set from default-client-config.yml or user config via --config-file) +# qleanImage is exported by spin-node.sh before sourcing this file + node_binary="$scriptDir/qlean/build/src/executable/qlean \ --modules-dir $scriptDir/qlean/build/src/modules \ --genesis $configDir/config.yaml \ @@ -16,8 +20,8 @@ node_binary="$scriptDir/qlean/build/src/executable/qlean \ --listen-addr /ip4/0.0.0.0/udp/$quicPort/quic-v1 \ --prometheus-port $metricsPort \ -ldebug" - -node_docker="qdrvm/qlean-mini:3a96a1f \ + +node_docker="$qleanImage \ --genesis /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --validator-keys-manifest /config/hash-sig-keys/validator-keys-manifest.yaml \ diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index 0c5105d..ad30479 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -4,6 +4,9 @@ # Metrics enabled by default metrics_flag="--metrics" +# Docker image (set from default-client-config.yml or user config via --config-file) +# reamImage is exported by spin-node.sh before sourcing this file + # modify the path to the ream binary as per your system node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ lean_node \ @@ -17,7 +20,7 @@ node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ --metrics-port $metricsPort \ --http-address 0.0.0.0" -node_docker="ghcr.io/reamlabs/ream:latest --data-dir /data \ +node_docker="$reamImage --data-dir /data \ lean_node \ --network /config/config.yaml \ --validator-registry-path /config/validators.yaml \ diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 1485465..23c2085 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -6,6 +6,9 @@ # Metrics enabled by default metrics_flag="--metrics_enable" +# Docker image (set from default-client-config.yml or user config via --config-file) +# zeamImage is exported by spin-node.sh before sourcing this file + node_binary="$scriptDir/../zig-out/bin/zeam node \ --custom_genesis $configDir \ --validator_config $validatorConfig \ @@ -14,7 +17,7 @@ node_binary="$scriptDir/../zig-out/bin/zeam node \ $metrics_flag \ --metrics_port $metricsPort" -node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet1 node \ +node_docker="--security-opt seccomp=unconfined $zeamImage node \ --custom_genesis /config \ --validator_config $validatorConfig \ --data-dir /data \ diff --git a/load-client-config.sh b/load-client-config.sh new file mode 100644 index 0000000..3d014aa --- /dev/null +++ b/load-client-config.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# Load client configuration from default and optional user config files + +# Arrays to store client names and images (bash 3.2 compatible) +CLIENT_NAMES=() +CLIENT_IMAGES_LIST=() +KNOWN_CLIENTS=("zeam" "ream" "qlean" "lantern" "lighthouse" "grandine") + +# Function to load default config +load_default_config() { + local default_config="$scriptDir/client-cmds/default-client-config.yml" + + if [ ! -f "$default_config" ]; then + echo "⚠️ Warning: Default config not found at $default_config" + return 1 + fi + + # Load default images using yq + while IFS= read -r line; do + local client_name=$(echo "$line" | awk '{print $1}') + local client_image=$(echo "$line" | awk '{print $2}') + CLIENT_NAMES+=("$client_name") + CLIENT_IMAGES_LIST+=("$client_image") + done < <(yq eval '.clients[] | .name + " " + .image' "$default_config") + + echo "✓ Loaded default client images from $default_config" +} + +# Function to find index of client name +find_client_index() { + local search_name="$1" + local i=0 + for name in "${CLIENT_NAMES[@]}"; do + if [ "$name" == "$search_name" ]; then + echo "$i" + return 0 + fi + ((i++)) + done + echo "-1" +} + +# Function to load user config and override defaults +load_user_config() { + local user_config="$1" + + if [ -z "$user_config" ]; then + return 0 + fi + + if [ ! -f "$user_config" ]; then + echo "⚠️ Warning: User config file not found at $user_config - using defaults" + return 1 + fi + + echo "Loading user config from $user_config..." + + # Load user-specified images + local override_count=0 + while IFS= read -r line; do + local client_name=$(echo "$line" | awk '{print $1}') + local client_image=$(echo "$line" | awk '{print $2}') + + # Validate client name + local valid_client=false + for known in "${KNOWN_CLIENTS[@]}"; do + if [ "$client_name" == "$known" ]; then + valid_client=true + break + fi + done + + if [ "$valid_client" == "false" ]; then + echo "⚠️ Warning: Unknown client '$client_name' in config - skipping" + continue + fi + + # Validate image format (basic check) + if [[ ! "$client_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then + echo "⚠️ Warning: Invalid image format '$client_image' for client '$client_name' - using default" + continue + fi + + # Find and update the client image + local idx=$(find_client_index "$client_name") + if [ "$idx" != "-1" ]; then + CLIENT_IMAGES_LIST[$idx]="$client_image" + echo " ✓ Override $client_name: $client_image" + ((override_count++)) + fi + done < <(yq eval '.clients[] | .name + " " + .image' "$user_config" 2>/dev/null) + + if [ $override_count -eq 0 ]; then + echo "⚠️ No valid overrides found in user config" + else + echo "✓ Applied $override_count custom image(s) from user config" + fi +} + +# Function to get image for a specific client +get_client_image() { + local client_name="$1" + local idx=$(find_client_index "$client_name") + if [ "$idx" != "-1" ]; then + echo "${CLIENT_IMAGES_LIST[$idx]}" + fi +} + +# Function to display loaded configuration +display_client_config() { + echo "" + echo "==================================================" + echo "Client Configuration:" + echo "==================================================" + printf "%-12s | %s\n" "Client" "Docker Image" + echo "--------------------------------------------------" + local i=0 + for client in "${CLIENT_NAMES[@]}"; do + if [ -n "${CLIENT_IMAGES_LIST[$i]}" ]; then + printf "%-12s | %s\n" "$client" "${CLIENT_IMAGES_LIST[$i]}" + fi + ((i++)) + done + echo "==================================================" + echo "" +} + +# Load default configuration +load_default_config + +# Load user configuration if provided +if [ -n "$configFile" ]; then + load_user_config "$configFile" +fi diff --git a/parse-env.sh b/parse-env.sh index 91b2a6a..da57287 100755 --- a/parse-env.sh +++ b/parse-env.sh @@ -76,6 +76,11 @@ while [[ $# -gt 0 ]]; do shift # past argument shift # past value ;; + --config-file) + configFile="$2" + shift # past argument + shift # past value + ;; --stop) stopNodes=true shift @@ -109,4 +114,5 @@ echo "generateGenesis = $generateGenesis" echo "cleanData = $cleanData" echo "popupTerminal = $popupTerminal" echo "dockerTag = ${dockerTag:-latest}" +echo "configFile = ${configFile:-none}" echo "enableMetrics = $enableMetrics" diff --git a/spin-node.sh b/spin-node.sh index 09eae2d..8af9957 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -10,6 +10,9 @@ fi # 0. parse env and args source "$(dirname $0)/parse-env.sh" +# Load client configuration (default + user config if provided) +source "$(dirname $0)/load-client-config.sh" + # Check if yq is installed (needed for deployment mode detection) if ! command -v yq &> /dev/null; then echo "Error: yq is required but not installed. Please install yq first." @@ -229,6 +232,9 @@ elif [[ "$OSTYPE" == "linux"* ]]; then done fi spinned_pids=() +declare -A node_images +declare -A node_modes + for item in "${spin_nodes[@]}"; do echo -e "\n\nspining $item: client=$client (mode=$node_setup)" printf '%*s' $(tput cols) | tr ' ' '-' @@ -248,11 +254,45 @@ for item in "${spin_nodes[@]}"; do IFS='_' read -r -a elements <<< "$item" client="${elements[0]}" + # Get docker image from config (always set - from default-client-config.yml or user override) + client_image=$(get_client_image "$client") + + # Export the image variable for this client (will be used by client-cmd.sh) + case "$client" in + zeam) + export zeamImage="$client_image" + ;; + ream) + export reamImage="$client_image" + ;; + qlean) + export qleanImage="$client_image" + ;; + lantern) + export lanternImage="$client_image" + ;; + lighthouse) + export lighthouseImage="$client_image" + ;; + grandine) + export grandineImage="$client_image" + ;; + esac + echo " ✓ Using image for $client: $client_image" + # get client specific cmd and its mode (docker, binary) sourceCmd="source client-cmds/$client-cmd.sh" echo "$sourceCmd" eval $sourceCmd + # Store the final image for display + if [ "$node_setup" == "docker" ]; then + node_images["$item"]=$(echo "$node_docker" | grep -oE '[^ ]+:[^ ]+' | head -1) + fi + + # Store node mode + node_modes["$item"]="$node_setup" + # spin nodes if [ "$node_setup" == "binary" ] then @@ -292,6 +332,25 @@ done; container_names="${spin_nodes[*]}" process_ids="${spinned_pids[*]}" +# Display summary table +echo "" +echo "==================================================" +echo "Deployed Nodes Summary:" +echo "==================================================" +printf "%-15s | %-10s | %s\n" "Node" "Mode" "Docker Image" +echo "--------------------------------------------------" +for node in "${spin_nodes[@]}"; do + mode="${node_modes[$node]}" + if [ "$mode" == "docker" ]; then + image="${node_images[$node]}" + printf "%-15s | %-10s | %s\n" "$node" "$mode" "$image" + else + printf "%-15s | %-10s | %s\n" "$node" "$mode" "N/A (binary mode)" + fi +done +echo "==================================================" +echo "" + cleanup() { echo -e "\n\ncleaning up" printf '%*s' $(tput cols) | tr ' ' '-' diff --git a/user-config.yml.example b/user-config.yml.example new file mode 100644 index 0000000..15c0611 --- /dev/null +++ b/user-config.yml.example @@ -0,0 +1,26 @@ +# User Client Configuration Example +# Copy this file to user-config.yml and customize your client images +# Usage: ./spin-node.sh --config-file user-config.yml --node all +# +# Only specify clients you want to override - others will use defaults +# from client-cmds/default-client-config.yml + +clients: + - name: zeam + image: blockblaz/zeam:custom-tag + + - name: ream + image: ghcr.io/reamlabs/ream:my-branch + + # Add more clients as needed: + # - name: qlean + # image: qdrvm/qlean-mini:custom-tag + # + # - name: lantern + # image: piertwo/lantern:custom-tag + # + # - name: lighthouse + # image: hopinheimer/lighthouse:custom-tag + # + # - name: grandine + # image: sifrai/lean:custom-tag From 39dc7a6952f8231d88f022ce5d7e87f298a34c9f Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Wed, 21 Jan 2026 14:06:48 +0100 Subject: [PATCH 2/2] fix: move image to the validator-config.yml --- README.md | 23 ++- ansible-devnet/genesis/validator-config.yaml | 40 +++-- ansible/roles/grandine/tasks/main.yml | 28 +++- ansible/roles/lantern/tasks/main.yml | 28 +++- ansible/roles/lighthouse/tasks/main.yml | 28 +++- ansible/roles/qlean/tasks/main.yml | 28 +++- ansible/roles/ream/tasks/main.yml | 28 +++- ansible/roles/zeam/tasks/main.yml | 31 ++-- client-cmds/default-client-config.yml | 22 --- client-cmds/grandine-cmd.sh | 2 +- client-cmds/lantern-cmd.sh | 2 +- client-cmds/lighthouse-cmd.sh | 2 +- client-cmds/qlean-cmd.sh | 2 +- client-cmds/ream-cmd.sh | 2 +- client-cmds/zeam-cmd.sh | 2 +- load-client-config.sh | 156 ++++++++++++------- local-devnet/genesis/validator-config.yaml | 10 +- run-ansible.sh | 69 ++++++++ spin-node.sh | 23 ++- user-config.yml.example | 26 ++-- 20 files changed, 389 insertions(+), 163 deletions(-) delete mode 100644 client-cmds/default-client-config.yml diff --git a/README.md b/README.md index 0eff1bf..830260d 100644 --- a/README.md +++ b/README.md @@ -76,37 +76,37 @@ You can override default Docker images using the `--config-file` flag. This is u **Basic usage (without custom images):** ```sh -# Uses default images from client-cmds/default-client-config.yml +# Uses default images from validator-config.yaml in your network directory NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis ``` **With custom config file:** ```sh -# Override specific client images +# Override specific node images NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --config-file user-config.yml ``` **Example config file (user-config.yml):** ```yaml -clients: - - name: zeam +nodes: + - name: zeam_0 image: blockblaz/zeam:feature-branch - - name: ream + - name: ream_0 image: ghcr.io/reamlabs/ream:v2.0 ``` **Testing a specific client build:** ```sh # Create custom config file for zeam /my-zeam-config.yml -clients: - - name: zeam +nodes: + - name: zeam_0 image: blockblaz/zeam:custom-tag # Run with custom zeam image NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --config-file /my-zeam-config.yml ``` -Only specify clients you want to override - others will use their defaults from `client-cmds/default-client-config.yml`. +Only specify nodes you want to override - others will use their defaults from `validator-config.yaml`. ## Args @@ -157,9 +157,9 @@ Only specify clients you want to override - others will use their defaults from - If not provided, defaults to `latest` for zeam, ream, and lantern, and `dd67521` for qlean - The script will automatically pull the specified Docker images before running containers - Example: `--tag devnet0` or `--tag devnet1` -11. `--config-file` specifies a custom configuration file to override default Docker images for specific clients. - - Path to a YAML file containing client image overrides (e.g., `user-config.yml` or `/path/to/my-config.yml`) - - Only clients specified in the config file are overridden; others use defaults from `client-cmds/default-client-config.yml` +11. `--config-file` specifies a custom configuration file to override default Docker images for specific nodes. + - Path to a YAML file containing node image overrides (e.g., `user-config.yml` or `/path/to/my-config.yml`) + - Only nodes specified in the config file are overridden; others use defaults from `validator-config.yaml` - See [Using custom Docker images](#using-custom-docker-images) scenario for usage examples - Example: `--config-file user-config.yml` or `--config-file /path/to/custom-config.yml` 12. `--metrics` enables metrics collection on all nodes. When specified, each client will activate its metrics endpoint according to its implementation. Metrics ports are configured per node in `validator-config.yaml`. @@ -192,7 +192,6 @@ The quickstart uses separate directories for local and Ansible deployments: ``` lean-quickstart/ ├── client-cmds/ # Client command scripts -│ ├── default-client-config.yml # Default Docker images for all clients │ ├── zeam-cmd.sh │ ├── ream-cmd.sh │ └── ... diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index d6c9dd2..5f633f7 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -8,6 +8,7 @@ validators: - name: "zeam_0" # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f + image: blockblaz/zeam:latest privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: # verify /ip4/127.0.0.1/udp/9000/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f @@ -19,6 +20,7 @@ validators: - name: "ream_0" # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 + image: ghcr.io/reamlabs/ream:latest privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -30,6 +32,7 @@ validators: - name: "qlean_0" # TODO: add a third entry in nodes.yaml corresponding to this + image: qdrvm/qlean-mini:latest privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -41,6 +44,7 @@ validators: - name: "lantern_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + image: piertwo/lantern:v0.0.1 privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -49,21 +53,23 @@ validators: metricsPort: 8081 count: 1 - - name: "lighthouse_0" - # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 - # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" - # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - enrFields: - ip: "46.224.135.169" - quic: 9001 - metricsPort: 8081 - count: 1 + # - name: "lighthouse_0" + # # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 + # # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + # image: hopinheimer/lighthouse:latest + # privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" + # # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + # enrFields: + # ip: "46.224.135.169" + # quic: 9001 + # metricsPort: 8081 + # count: 1 - - name: "grandine_0" - privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" - enrFields: - ip: "46.62.163.74" - quic: 9001 - metricsPort: 8081 - count: 1 \ No newline at end of file + # - name: "grandine_0" + # image: sifrai/lean:unstable + # privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" + # enrFields: + # ip: "46.62.163.74" + # quic: 9001 + # metricsPort: 8081 + # count: 1 diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index 3bdda8a..be3426a 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -2,12 +2,30 @@ # Grandine role: Deploy and manage Grandine nodes # Converts client-cmds/grandine-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/grandine-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (grandine_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^grandine_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "grandine") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^grandine_")) | .image' "$validator_config" | head -1 register: grandine_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index 9932194..574abbf 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -2,12 +2,30 @@ # Lantern role: Deploy and manage Lantern nodes # Converts client-cmds/lantern-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the docker image from LANTERN_IMAGE line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^LANTERN_IMAGE=' "$project_root/client-cmds/lantern-cmd.sh" | head -1 | sed -E 's/.*LANTERN_IMAGE="([^"]+)".*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (lantern_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^lantern_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "lantern") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^lantern_")) | .image' "$validator_config" | head -1 register: lantern_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index 5103473..869fd01 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -2,12 +2,30 @@ # Lighthouse role: Deploy and manage Lighthouse nodes # Converts client-cmds/lighthouse-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/lighthouse-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (lighthouse_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^lighthouse_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "lighthouse") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^lighthouse_")) | .image' "$validator_config" | head -1 register: lighthouse_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index 09f2c67..fadb768 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -2,12 +2,30 @@ # Qlean role: Deploy and manage Qlean nodes # Converts client-cmds/qlean-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/qlean-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (qlean_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^qlean_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "qlean") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^qlean_")) | .image' "$validator_config" | head -1 register: qlean_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index 1ed3326..9917a7c 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -2,12 +2,30 @@ # Ream role: Deploy and manage Ream nodes # Converts client-cmds/ream-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/ream-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (ream_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^ream_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "ream") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^ream_")) | .image' "$validator_config" | head -1 register: ream_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index a0936c9..71340bf 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -2,17 +2,30 @@ # Zeam role: Deploy and manage Zeam nodes # Converts client-cmds/zeam-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the docker image from node_docker line (find word containing / and :) - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - script_path="$project_root/client-cmds/zeam-cmd.sh" - if [ ! -f "$script_path" ]; then - echo "ERROR: Script not found at $script_path (project_root=$project_root, playbook_dir={{ playbook_dir }})" >&2 - exit 1 + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (zeam_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^zeam_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "zeam") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi fi - grep -E '^node_docker=' "$script_path" | head -1 | grep -oE '[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+:[a-zA-Z0-9._-]+' | head -1 + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^zeam_")) | .image' "$validator_config" | head -1 register: zeam_docker_image_raw changed_when: false delegate_to: localhost diff --git a/client-cmds/default-client-config.yml b/client-cmds/default-client-config.yml deleted file mode 100644 index cc700c0..0000000 --- a/client-cmds/default-client-config.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Default Client Configuration -# This file contains the default Docker images for all supported clients -# These defaults are used when no custom config file is specified via --config-file - -clients: - - name: zeam - image: blockblaz/zeam:latest - - - name: ream - image: ghcr.io/reamlabs/ream:latest - - - name: qlean - image: qdrvm/qlean-mini:latest - - - name: lantern - image: piertwo/lantern:latest - - - name: lighthouse - image: hopinheimer/lighthouse:latest - - - name: grandine - image: sifrai/lean:latest diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index f07e6a8..0541367 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # grandineImage is exported by spin-node.sh before sourcing this file node_binary="$grandine_bin \ diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index cd3dcc3..0aa98bb 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -1,7 +1,7 @@ #!/bin/bash #-----------------------lantern setup---------------------- -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # lanternImage is exported by spin-node.sh before sourcing this file devnet_flag="" diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index 51938c6..2c345ab 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -3,7 +3,7 @@ # Metrics enabled by default metrics_flag="--metrics" -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # lighthouseImage is exported by spin-node.sh before sourcing this file node_binary="$lighthouse_bin lean_node \ diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index 689e650..6f12f88 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -4,7 +4,7 @@ # expects "qlean" submodule or symlink inside "lean-quickstart" root directory # https://github.com/qdrvm/qlean-mini -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # qleanImage is exported by spin-node.sh before sourcing this file node_binary="$scriptDir/qlean/build/src/executable/qlean \ diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index ad30479..a4f191c 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -4,7 +4,7 @@ # Metrics enabled by default metrics_flag="--metrics" -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # reamImage is exported by spin-node.sh before sourcing this file # modify the path to the ream binary as per your system diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 23c2085..f8aa964 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -6,7 +6,7 @@ # Metrics enabled by default metrics_flag="--metrics_enable" -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # zeamImage is exported by spin-node.sh before sourcing this file node_binary="$scriptDir/../zig-out/bin/zeam node \ diff --git a/load-client-config.sh b/load-client-config.sh index 3d014aa..1800cc2 100644 --- a/load-client-config.sh +++ b/load-client-config.sh @@ -1,31 +1,11 @@ #!/bin/bash -# Load client configuration from default and optional user config files +# Load client configuration from validator-config.yaml and optional user config file -# Arrays to store client names and images (bash 3.2 compatible) +# Associative array to store client images (requires bash 4+, but we'll use parallel arrays for bash 3.2 compatibility) CLIENT_NAMES=() CLIENT_IMAGES_LIST=() KNOWN_CLIENTS=("zeam" "ream" "qlean" "lantern" "lighthouse" "grandine") -# Function to load default config -load_default_config() { - local default_config="$scriptDir/client-cmds/default-client-config.yml" - - if [ ! -f "$default_config" ]; then - echo "⚠️ Warning: Default config not found at $default_config" - return 1 - fi - - # Load default images using yq - while IFS= read -r line; do - local client_name=$(echo "$line" | awk '{print $1}') - local client_image=$(echo "$line" | awk '{print $2}') - CLIENT_NAMES+=("$client_name") - CLIENT_IMAGES_LIST+=("$client_image") - done < <(yq eval '.clients[] | .name + " " + .image' "$default_config") - - echo "✓ Loaded default client images from $default_config" -} - # Function to find index of client name find_client_index() { local search_name="$1" @@ -40,7 +20,35 @@ find_client_index() { echo "-1" } +# Function to extract client type from node name (e.g., zeam_0 -> zeam) +get_client_type() { + echo "$1" | sed 's/_[0-9]*$//' +} + +# Function to load default config from validator-config.yaml +load_default_config() { + local validator_config="$configDir/validator-config.yaml" + + if [ ! -f "$validator_config" ]; then + echo "⚠️ Warning: validator-config.yaml not found at $validator_config" + return 1 + fi + + # Load images from validators array using yq + for client in "${KNOWN_CLIENTS[@]}"; do + # Find the first validator matching this client type (e.g., zeam_0 for zeam) + local image=$(yq eval ".validators[] | select(.name | test(\"^${client}_\")) | .image" "$validator_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + CLIENT_NAMES+=("$client") + CLIENT_IMAGES_LIST+=("$image") + fi + done + + echo "✓ Loaded default client images from $validator_config" +} + # Function to load user config and override defaults +# User config format uses node names (e.g., zeam_0) instead of client names load_user_config() { local user_config="$1" @@ -55,40 +63,84 @@ load_user_config() { echo "Loading user config from $user_config..." - # Load user-specified images + # Load user-specified images (supports both 'nodes' and 'clients' format for backwards compatibility) local override_count=0 - while IFS= read -r line; do - local client_name=$(echo "$line" | awk '{print $1}') - local client_image=$(echo "$line" | awk '{print $2}') - - # Validate client name - local valid_client=false - for known in "${KNOWN_CLIENTS[@]}"; do - if [ "$client_name" == "$known" ]; then - valid_client=true - break + + # Try 'nodes' format first (new format with zeam_0, etc.) + local nodes_exist=$(yq eval '.nodes // empty' "$user_config" 2>/dev/null) + + if [ -n "$nodes_exist" ]; then + # New format: nodes with zeam_0 style names + while IFS= read -r line; do + local node_name=$(echo "$line" | awk '{print $1}') + local node_image=$(echo "$line" | awk '{print $2}') + + # Extract client type from node name (zeam_0 -> zeam) + local client_type=$(get_client_type "$node_name") + + # Validate client type + local valid_client=false + for known in "${KNOWN_CLIENTS[@]}"; do + if [ "$client_type" == "$known" ]; then + valid_client=true + break + fi + done + + if [ "$valid_client" == "false" ]; then + echo "⚠️ Warning: Unknown client type '$client_type' from node '$node_name' - skipping" + continue fi - done - if [ "$valid_client" == "false" ]; then - echo "⚠️ Warning: Unknown client '$client_name' in config - skipping" - continue - fi + # Validate image format (basic check) + if [[ ! "$node_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then + echo "⚠️ Warning: Invalid image format '$node_image' for node '$node_name' - using default" + continue + fi - # Validate image format (basic check) - if [[ ! "$client_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then - echo "⚠️ Warning: Invalid image format '$client_image' for client '$client_name' - using default" - continue - fi + # Find and update the client image + local idx=$(find_client_index "$client_type") + if [ "$idx" != "-1" ]; then + CLIENT_IMAGES_LIST[$idx]="$node_image" + echo " ✓ Override $client_type (from $node_name): $node_image" + ((override_count++)) + fi + done < <(yq eval '.nodes[] | .name + " " + .image' "$user_config" 2>/dev/null) + else + # Fallback: try 'clients' format (old format for backwards compatibility) + while IFS= read -r line; do + local client_name=$(echo "$line" | awk '{print $1}') + local client_image=$(echo "$line" | awk '{print $2}') + + # Validate client name + local valid_client=false + for known in "${KNOWN_CLIENTS[@]}"; do + if [ "$client_name" == "$known" ]; then + valid_client=true + break + fi + done + + if [ "$valid_client" == "false" ]; then + echo "⚠️ Warning: Unknown client '$client_name' in config - skipping" + continue + fi - # Find and update the client image - local idx=$(find_client_index "$client_name") - if [ "$idx" != "-1" ]; then - CLIENT_IMAGES_LIST[$idx]="$client_image" - echo " ✓ Override $client_name: $client_image" - ((override_count++)) - fi - done < <(yq eval '.clients[] | .name + " " + .image' "$user_config" 2>/dev/null) + # Validate image format (basic check) + if [[ ! "$client_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then + echo "⚠️ Warning: Invalid image format '$client_image' for client '$client_name' - using default" + continue + fi + + # Find and update the client image + local idx=$(find_client_index "$client_name") + if [ "$idx" != "-1" ]; then + CLIENT_IMAGES_LIST[$idx]="$client_image" + echo " ✓ Override $client_name: $client_image" + ((override_count++)) + fi + done < <(yq eval '.clients[] | .name + " " + .image' "$user_config" 2>/dev/null) + fi if [ $override_count -eq 0 ]; then echo "⚠️ No valid overrides found in user config" @@ -125,7 +177,7 @@ display_client_config() { echo "" } -# Load default configuration +# Load default configuration from validator-config.yaml load_default_config # Load user configuration if provided diff --git a/local-devnet/genesis/validator-config.yaml b/local-devnet/genesis/validator-config.yaml index 9e1a058..0278d85 100644 --- a/local-devnet/genesis/validator-config.yaml +++ b/local-devnet/genesis/validator-config.yaml @@ -8,6 +8,7 @@ validators: - name: "zeam_0" # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f + image: blockblaz/zeam:latest privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: # verify /ip4/127.0.0.1/udp/9000/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f @@ -19,6 +20,7 @@ validators: - name: "ream_0" # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 + image: ghcr.io/reamlabs/ream:latest privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -30,6 +32,7 @@ validators: - name: "qlean_0" # TODO: add a third entry in nodes.yaml corresponding to this + image: qdrvm/qlean-mini:latest privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -41,6 +44,7 @@ validators: - name: "lantern_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + image: piertwo/lantern:v0.0.1 privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -48,10 +52,11 @@ validators: quic: 9004 metricsPort: 8084 count: 1 - + - name: "lighthouse_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + image: hopinheimer/lighthouse:latest privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -61,9 +66,10 @@ validators: count: 1 - name: "grandine_0" + image: sifrai/lean:unstable privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" enrFields: ip: "127.0.0.1" quic: 9006 metricsPort: 8086 - count: 1 \ No newline at end of file + count: 1 diff --git a/run-ansible.sh b/run-ansible.sh index a613417..f138b0a 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -26,6 +26,7 @@ validator_config_file="$5" sshKeyFile="$6" useRoot="$7" # Flag to use root user (defaults to current user) action="$8" # Action: "stop" to stop nodes, otherwise deploy +userConfigFile="$9" # User config file for docker images (--config-file) # Determine SSH user: use root if --useRoot flag is set, otherwise use current user if [ "$useRoot" == "true" ]; then @@ -119,6 +120,16 @@ fi # Use default deployment mode (can be overridden by adding a 'deployment_mode' field per node in validator-config.yaml) EXTRA_VARS="$EXTRA_VARS deployment_mode=$DEFAULT_DEPLOYMENT_MODE" +# Pass user config file path for docker images if provided +if [ -n "$userConfigFile" ]; then + # Convert to absolute path if relative + if [[ "$userConfigFile" != /* ]]; then + userConfigFile="$(cd "$(dirname "$userConfigFile")" 2>/dev/null && pwd)/$(basename "$userConfigFile")" + fi + EXTRA_VARS="$EXTRA_VARS user_config_file=$userConfigFile" + echo "Using user config file for docker images: $userConfigFile" +fi + # Determine which playbook to run if [ "$action" == "stop" ]; then PLAYBOOK="$ANSIBLE_DIR/playbooks/stop-nodes.yml" @@ -148,6 +159,64 @@ if [ $EXIT_CODE -eq 0 ]; then if [ "$action" == "stop" ]; then echo "✅ Ansible stop operation completed successfully!" else + # Display summary of deployed nodes + echo "" + echo "==================================================" + echo "Deployed Nodes Summary:" + echo "==================================================" + printf "%-15s | %-10s | %s\n" "Node" "Mode" "Image / Binary Path" + echo "--------------------------------------------------" + + # Parse node names (comma or space separated) + if [[ "$node" == *","* ]]; then + IFS=',' read -r -a deployed_nodes <<< "$node" + else + IFS=' ' read -r -a deployed_nodes <<< "$node" + fi + + # Get config file paths - use validator-config.yaml from genesis dir + validator_config="$configDir/validator-config.yaml" + + for node_name in "${deployed_nodes[@]}"; do + # Extract client type from node name (e.g., zeam_0 -> zeam) + client_type=$(echo "$node_name" | sed 's/_[0-9]*$//') + + if [ "$DEFAULT_DEPLOYMENT_MODE" == "binary" ]; then + # Extract binary path from client-cmd.sh + client_cmd_file="$scriptDir/client-cmds/${client_type}-cmd.sh" + binary_path="" + if [ -f "$client_cmd_file" ]; then + # Extract the first word from node_binary line (the actual binary path) + binary_path=$(grep -E '^node_binary=' "$client_cmd_file" | head -1 | sed -E 's/node_binary="([^ "\\]+).*/\1/') + # Resolve $scriptDir to actual path + binary_path=$(echo "$binary_path" | sed "s|\\\$scriptDir|$scriptDir|g") + # For unresolved variables (like $grandine_bin), show as custom binary path + if [[ "$binary_path" == \$* ]]; then + binary_path="" + fi + fi + printf "%-15s | %-10s | %s\n" "$node_name" "binary" "${binary_path:-unknown}" + else + # Get image from user config or validator-config.yaml + docker_image="" + if [ -n "$userConfigFile" ] && [ -f "$userConfigFile" ]; then + # Try nodes format first (zeam_0 style) + docker_image=$(yq eval ".nodes[] | select(.name | test(\"^${client_type}_\")) | .image" "$userConfigFile" 2>/dev/null | head -1) + # Try clients format (backwards compatibility) + if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then + docker_image=$(yq eval ".clients[] | select(.name == \"$client_type\") | .image" "$userConfigFile" 2>/dev/null) + fi + fi + if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then + # Read from validators array in validator-config.yaml + docker_image=$(yq eval ".validators[] | select(.name | test(\"^${client_type}_\")) | .image" "$validator_config" 2>/dev/null | head -1) + fi + + printf "%-15s | %-10s | %s\n" "$node_name" "docker" "${docker_image:-unknown}" + fi + done + echo "==================================================" + echo "" echo "✅ Ansible deployment completed successfully!" fi else diff --git a/spin-node.sh b/spin-node.sh index 8af9957..7d77829 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -148,16 +148,16 @@ if [ "$deployment_mode" == "ansible" ]; then # Handle stop action if [ -n "$stopNodes" ] && [ "$stopNodes" == "true" ]; then echo "Stopping nodes via Ansible..." - if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "stop"; then + if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "stop" "$configFile"; then echo "❌ Ansible stop operation failed. Exiting." exit 1 fi exit 0 fi - + # Call separate Ansible execution script # If Ansible deployment fails, exit immediately (don't fall through to local deployment) - if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot"; then + if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "" "$configFile"; then echo "❌ Ansible deployment failed. Exiting." exit 1 fi @@ -254,7 +254,7 @@ for item in "${spin_nodes[@]}"; do IFS='_' read -r -a elements <<< "$item" client="${elements[0]}" - # Get docker image from config (always set - from default-client-config.yml or user override) + # Get docker image from config (always set - from validator-config.yaml or user override) client_image=$(get_client_image "$client") # Export the image variable for this client (will be used by client-cmd.sh) @@ -300,12 +300,25 @@ for item in "${spin_nodes[@]}"; do else # Extract image name from node_docker (find word containing ':' which is the image:tag) docker_image=$(echo "$node_docker" | grep -oE '[^ ]+:[^ ]+' | head -1) - # Pull image first + # Pull image first if [ -n "$dockerWithSudo" ]; then sudo docker pull "$docker_image" || true else docker pull "$docker_image" || true fi + + # Check if the image exists locally (either pulled successfully or was cached) + if [ -n "$dockerWithSudo" ]; then + image_exists=$(sudo docker image inspect "$docker_image" > /dev/null 2>&1 && echo "yes" || echo "no") + else + image_exists=$(docker image inspect "$docker_image" > /dev/null 2>&1 && echo "yes" || echo "no") + fi + + if [ "$image_exists" == "no" ]; then + echo "⚠️ Skipping $item: Docker image '$docker_image' does not exist and could not be pulled" + continue + fi + execCmd="docker run --rm --pull=never" if [ -n "$dockerWithSudo" ] then diff --git a/user-config.yml.example b/user-config.yml.example index 15c0611..70c237c 100644 --- a/user-config.yml.example +++ b/user-config.yml.example @@ -1,26 +1,26 @@ -# User Client Configuration Example -# Copy this file to user-config.yml and customize your client images -# Usage: ./spin-node.sh --config-file user-config.yml --node all +# User Configuration Example +# Copy this file to user-config.yml or .yml and customize your docker images +# Usage: ./spin-node.sh --config-file .yml --node zeam_0,ream_0 # -# Only specify clients you want to override - others will use defaults -# from client-cmds/default-client-config.yml +# Only specify nodes you want to override - others will use defaults +# from validator-config.yaml in your network directory -clients: - - name: zeam +nodes: + - name: zeam_0 image: blockblaz/zeam:custom-tag - - name: ream + - name: ream_0 image: ghcr.io/reamlabs/ream:my-branch - # Add more clients as needed: - # - name: qlean + # Add more nodes as needed: + # - name: qlean_0 # image: qdrvm/qlean-mini:custom-tag # - # - name: lantern + # - name: lantern_0 # image: piertwo/lantern:custom-tag # - # - name: lighthouse + # - name: lighthouse_0 # image: hopinheimer/lighthouse:custom-tag # - # - name: grandine + # - name: grandine_0 # image: sifrai/lean:custom-tag