From f0f4276d4368ba0db70f222b5c064d427858feb5 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Fri, 16 Jan 2026 11:44:05 +0100 Subject: [PATCH 01/15] feat: add config file support to configure docker images --- .gitignore | 3 + README.md | 54 ++++++++++- client-cmds/default-client-config.yml | 22 +++++ client-cmds/grandine-cmd.sh | 5 +- client-cmds/lantern-cmd.sh | 5 +- client-cmds/lighthouse-cmd.sh | 5 +- client-cmds/qlean-cmd.sh | 8 +- client-cmds/ream-cmd.sh | 5 +- client-cmds/zeam-cmd.sh | 5 +- load-client-config.sh | 134 ++++++++++++++++++++++++++ parse-env.sh | 6 ++ spin-node.sh | 59 ++++++++++++ user-config.yml.example | 26 +++++ 13 files changed, 327 insertions(+), 10 deletions(-) create mode 100644 client-cmds/default-client-config.yml create mode 100644 load-client-config.sh create mode 100644 user-config.yml.example diff --git a/.gitignore b/.gitignore index a32ef5e..37e6f92 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,9 @@ data *.old .DS_Store +# User configuration files +user-config.yml + # Generated genesis files (created by generate-genesis.sh) config.yaml validators.yaml diff --git a/README.md b/README.md index 3b2916a..70ecd54 100644 --- a/README.md +++ b/README.md @@ -70,6 +70,44 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --generateGenesis --popupT NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --metrics ``` +### Using custom Docker images + +You can override default Docker images using the `--config-file` flag. This is useful for testing custom builds or using specific versions without modifying the codebase. + +**Basic usage (without custom images):** +```sh +# Uses default images from client-cmds/default-client-config.yml +NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis +``` + +**With custom config file:** +```sh +# Override specific client images +NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --config-file user-config.yml +``` + +**Example config file (user-config.yml):** +```yaml +clients: + - name: zeam + image: blockblaz/zeam:feature-branch + - name: ream + image: ghcr.io/reamlabs/ream:v2.0 +``` + +**Testing a specific client build:** +```sh +# Create custom config file for zeam /my-zeam-config.yml +clients: + - name: zeam + image: blockblaz/zeam:custom-tag + +# Run with custom zeam image +NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --config-file /my-zeam-config.yml +``` + +Only specify clients you want to override - others will use their defaults from `client-cmds/default-client-config.yml`. + ## Args 1. `NETWORK_DIR` is an env to specify the network directory. Should have a `genesis` directory with genesis config. A `data` folder will be created inside this `NETWORK_DIR` if not already there. @@ -119,6 +157,11 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --metrics - If not provided, defaults to `latest` for zeam, ream, and lantern, and `dd67521` for qlean - The script will automatically pull the specified Docker images before running containers - Example: `--tag devnet0` or `--tag devnet1` +11. `--config-file` specifies a custom configuration file to override default Docker images for specific clients. + - Path to a YAML file containing client image overrides (e.g., `user-config.yml` or `/path/to/my-config.yml`) + - Only clients specified in the config file are overridden; others use defaults from `client-cmds/default-client-config.yml` + - See [Using custom Docker images](#using-custom-docker-images) scenario for usage examples + - Example: `--config-file user-config.yml` or `--config-file /path/to/custom-config.yml` 12. `--metrics` enables metrics collection on all nodes. When specified, each client will activate its metrics endpoint according to its implementation. Metrics ports are configured per node in `validator-config.yaml`. ### Clients supported @@ -148,12 +191,19 @@ The quickstart uses separate directories for local and Ansible deployments: ``` lean-quickstart/ -├── local-devnet/ # Local development +├── client-cmds/ # Client command scripts +│ ├── default-client-config.yml # Default Docker images for all clients +│ ├── zeam-cmd.sh +│ ├── ream-cmd.sh +│ └── ... +├── user-config.yml.example # Example custom config (copy to user-config.yml) +├── user-config.yml # Your custom image overrides (gitignored) +├── local-devnet/ # Local development │ ├── genesis/ │ │ └── validator-config.yaml # Local IPs (127.0.0.1) │ └── data/ # Node data directories │ -└── ansible-devnet/ # Ansible/remote deployment +└── ansible-devnet/ # Ansible/remote deployment ├── genesis/ │ └── validator-config.yaml # Remote IPs (your server IPs) └── data/ # Node data directories diff --git a/client-cmds/default-client-config.yml b/client-cmds/default-client-config.yml new file mode 100644 index 0000000..cc700c0 --- /dev/null +++ b/client-cmds/default-client-config.yml @@ -0,0 +1,22 @@ +# Default Client Configuration +# This file contains the default Docker images for all supported clients +# These defaults are used when no custom config file is specified via --config-file + +clients: + - name: zeam + image: blockblaz/zeam:latest + + - name: ream + image: ghcr.io/reamlabs/ream:latest + + - name: qlean + image: qdrvm/qlean-mini:latest + + - name: lantern + image: piertwo/lantern:latest + + - name: lighthouse + image: hopinheimer/lighthouse:latest + + - name: grandine + image: sifrai/lean:latest diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index b8456a6..f07e6a8 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -1,5 +1,8 @@ #!/bin/bash +# Docker image (set from default-client-config.yml or user config via --config-file) +# grandineImage is exported by spin-node.sh before sourcing this file + node_binary="$grandine_bin \ --genesis $configDir/config.yaml \ --validator-registry-path $configDir/validators.yaml \ @@ -10,7 +13,7 @@ node_binary="$grandine_bin \ --address 0.0.0.0 \ --hash-sig-key-dir $configDir/hash-sig-keys" -node_docker="sifrai/lean:unstable \ +node_docker="$grandineImage \ --genesis /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --bootnodes /config/nodes.yaml \ diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index 3075a95..cd3dcc3 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -1,7 +1,8 @@ #!/bin/bash #-----------------------lantern setup---------------------- -LANTERN_IMAGE="piertwo/lantern:v0.0.1" +# Docker image (set from default-client-config.yml or user config via --config-file) +# lanternImage is exported by spin-node.sh before sourcing this file devnet_flag="" if [ -n "$devnet" ]; then @@ -24,7 +25,7 @@ node_binary="$scriptDir/lantern/build/lantern_cli \ --log-level debug \ --hash-sig-key-dir $configDir/hash-sig-keys" -node_docker="$LANTERN_IMAGE --data-dir /data \ +node_docker="$lanternImage --data-dir /data \ --genesis-config /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --genesis-state /config/genesis.ssz \ diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index 1e129c2..51938c6 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -3,6 +3,9 @@ # Metrics enabled by default metrics_flag="--metrics" +# Docker image (set from default-client-config.yml or user config via --config-file) +# lighthouseImage is exported by spin-node.sh before sourcing this file + node_binary="$lighthouse_bin lean_node \ --datadir \"$dataDir/$item\" \ --config \"$configDir/config.yaml\" \ @@ -16,7 +19,7 @@ node_binary="$lighthouse_bin lean_node \ --metrics-address 0.0.0.0 \ --metrics-port $metricsPort" -node_docker="hopinheimer/lighthouse:latest lighthouse lean_node \ +node_docker="$lighthouseImage lighthouse lean_node \ --datadir /data \ --config /config/config.yaml \ --validators /config/validator-config.yaml \ diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index 178ca98..a90b775 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -3,6 +3,10 @@ #-----------------------qlean setup---------------------- # expects "qlean" submodule or symlink inside "lean-quickstart" root directory # https://github.com/qdrvm/qlean-mini + +# Docker image (set from default-client-config.yml or user config via --config-file) +# qleanImage is exported by spin-node.sh before sourcing this file + node_binary="$scriptDir/qlean/build/src/executable/qlean \ --modules-dir $scriptDir/qlean/build/src/modules \ --genesis $configDir/config.yaml \ @@ -16,8 +20,8 @@ node_binary="$scriptDir/qlean/build/src/executable/qlean \ --listen-addr /ip4/0.0.0.0/udp/$quicPort/quic-v1 \ --prometheus-port $metricsPort \ -ldebug" - -node_docker="qdrvm/qlean-mini:3a96a1f \ + +node_docker="$qleanImage \ --genesis /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --validator-keys-manifest /config/hash-sig-keys/validator-keys-manifest.yaml \ diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index 0c5105d..ad30479 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -4,6 +4,9 @@ # Metrics enabled by default metrics_flag="--metrics" +# Docker image (set from default-client-config.yml or user config via --config-file) +# reamImage is exported by spin-node.sh before sourcing this file + # modify the path to the ream binary as per your system node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ lean_node \ @@ -17,7 +20,7 @@ node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ --metrics-port $metricsPort \ --http-address 0.0.0.0" -node_docker="ghcr.io/reamlabs/ream:latest --data-dir /data \ +node_docker="$reamImage --data-dir /data \ lean_node \ --network /config/config.yaml \ --validator-registry-path /config/validators.yaml \ diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 1485465..23c2085 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -6,6 +6,9 @@ # Metrics enabled by default metrics_flag="--metrics_enable" +# Docker image (set from default-client-config.yml or user config via --config-file) +# zeamImage is exported by spin-node.sh before sourcing this file + node_binary="$scriptDir/../zig-out/bin/zeam node \ --custom_genesis $configDir \ --validator_config $validatorConfig \ @@ -14,7 +17,7 @@ node_binary="$scriptDir/../zig-out/bin/zeam node \ $metrics_flag \ --metrics_port $metricsPort" -node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet1 node \ +node_docker="--security-opt seccomp=unconfined $zeamImage node \ --custom_genesis /config \ --validator_config $validatorConfig \ --data-dir /data \ diff --git a/load-client-config.sh b/load-client-config.sh new file mode 100644 index 0000000..3d014aa --- /dev/null +++ b/load-client-config.sh @@ -0,0 +1,134 @@ +#!/bin/bash +# Load client configuration from default and optional user config files + +# Arrays to store client names and images (bash 3.2 compatible) +CLIENT_NAMES=() +CLIENT_IMAGES_LIST=() +KNOWN_CLIENTS=("zeam" "ream" "qlean" "lantern" "lighthouse" "grandine") + +# Function to load default config +load_default_config() { + local default_config="$scriptDir/client-cmds/default-client-config.yml" + + if [ ! -f "$default_config" ]; then + echo "⚠️ Warning: Default config not found at $default_config" + return 1 + fi + + # Load default images using yq + while IFS= read -r line; do + local client_name=$(echo "$line" | awk '{print $1}') + local client_image=$(echo "$line" | awk '{print $2}') + CLIENT_NAMES+=("$client_name") + CLIENT_IMAGES_LIST+=("$client_image") + done < <(yq eval '.clients[] | .name + " " + .image' "$default_config") + + echo "✓ Loaded default client images from $default_config" +} + +# Function to find index of client name +find_client_index() { + local search_name="$1" + local i=0 + for name in "${CLIENT_NAMES[@]}"; do + if [ "$name" == "$search_name" ]; then + echo "$i" + return 0 + fi + ((i++)) + done + echo "-1" +} + +# Function to load user config and override defaults +load_user_config() { + local user_config="$1" + + if [ -z "$user_config" ]; then + return 0 + fi + + if [ ! -f "$user_config" ]; then + echo "⚠️ Warning: User config file not found at $user_config - using defaults" + return 1 + fi + + echo "Loading user config from $user_config..." + + # Load user-specified images + local override_count=0 + while IFS= read -r line; do + local client_name=$(echo "$line" | awk '{print $1}') + local client_image=$(echo "$line" | awk '{print $2}') + + # Validate client name + local valid_client=false + for known in "${KNOWN_CLIENTS[@]}"; do + if [ "$client_name" == "$known" ]; then + valid_client=true + break + fi + done + + if [ "$valid_client" == "false" ]; then + echo "⚠️ Warning: Unknown client '$client_name' in config - skipping" + continue + fi + + # Validate image format (basic check) + if [[ ! "$client_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then + echo "⚠️ Warning: Invalid image format '$client_image' for client '$client_name' - using default" + continue + fi + + # Find and update the client image + local idx=$(find_client_index "$client_name") + if [ "$idx" != "-1" ]; then + CLIENT_IMAGES_LIST[$idx]="$client_image" + echo " ✓ Override $client_name: $client_image" + ((override_count++)) + fi + done < <(yq eval '.clients[] | .name + " " + .image' "$user_config" 2>/dev/null) + + if [ $override_count -eq 0 ]; then + echo "⚠️ No valid overrides found in user config" + else + echo "✓ Applied $override_count custom image(s) from user config" + fi +} + +# Function to get image for a specific client +get_client_image() { + local client_name="$1" + local idx=$(find_client_index "$client_name") + if [ "$idx" != "-1" ]; then + echo "${CLIENT_IMAGES_LIST[$idx]}" + fi +} + +# Function to display loaded configuration +display_client_config() { + echo "" + echo "==================================================" + echo "Client Configuration:" + echo "==================================================" + printf "%-12s | %s\n" "Client" "Docker Image" + echo "--------------------------------------------------" + local i=0 + for client in "${CLIENT_NAMES[@]}"; do + if [ -n "${CLIENT_IMAGES_LIST[$i]}" ]; then + printf "%-12s | %s\n" "$client" "${CLIENT_IMAGES_LIST[$i]}" + fi + ((i++)) + done + echo "==================================================" + echo "" +} + +# Load default configuration +load_default_config + +# Load user configuration if provided +if [ -n "$configFile" ]; then + load_user_config "$configFile" +fi diff --git a/parse-env.sh b/parse-env.sh index 91b2a6a..da57287 100755 --- a/parse-env.sh +++ b/parse-env.sh @@ -76,6 +76,11 @@ while [[ $# -gt 0 ]]; do shift # past argument shift # past value ;; + --config-file) + configFile="$2" + shift # past argument + shift # past value + ;; --stop) stopNodes=true shift @@ -109,4 +114,5 @@ echo "generateGenesis = $generateGenesis" echo "cleanData = $cleanData" echo "popupTerminal = $popupTerminal" echo "dockerTag = ${dockerTag:-latest}" +echo "configFile = ${configFile:-none}" echo "enableMetrics = $enableMetrics" diff --git a/spin-node.sh b/spin-node.sh index 09eae2d..8af9957 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -10,6 +10,9 @@ fi # 0. parse env and args source "$(dirname $0)/parse-env.sh" +# Load client configuration (default + user config if provided) +source "$(dirname $0)/load-client-config.sh" + # Check if yq is installed (needed for deployment mode detection) if ! command -v yq &> /dev/null; then echo "Error: yq is required but not installed. Please install yq first." @@ -229,6 +232,9 @@ elif [[ "$OSTYPE" == "linux"* ]]; then done fi spinned_pids=() +declare -A node_images +declare -A node_modes + for item in "${spin_nodes[@]}"; do echo -e "\n\nspining $item: client=$client (mode=$node_setup)" printf '%*s' $(tput cols) | tr ' ' '-' @@ -248,11 +254,45 @@ for item in "${spin_nodes[@]}"; do IFS='_' read -r -a elements <<< "$item" client="${elements[0]}" + # Get docker image from config (always set - from default-client-config.yml or user override) + client_image=$(get_client_image "$client") + + # Export the image variable for this client (will be used by client-cmd.sh) + case "$client" in + zeam) + export zeamImage="$client_image" + ;; + ream) + export reamImage="$client_image" + ;; + qlean) + export qleanImage="$client_image" + ;; + lantern) + export lanternImage="$client_image" + ;; + lighthouse) + export lighthouseImage="$client_image" + ;; + grandine) + export grandineImage="$client_image" + ;; + esac + echo " ✓ Using image for $client: $client_image" + # get client specific cmd and its mode (docker, binary) sourceCmd="source client-cmds/$client-cmd.sh" echo "$sourceCmd" eval $sourceCmd + # Store the final image for display + if [ "$node_setup" == "docker" ]; then + node_images["$item"]=$(echo "$node_docker" | grep -oE '[^ ]+:[^ ]+' | head -1) + fi + + # Store node mode + node_modes["$item"]="$node_setup" + # spin nodes if [ "$node_setup" == "binary" ] then @@ -292,6 +332,25 @@ done; container_names="${spin_nodes[*]}" process_ids="${spinned_pids[*]}" +# Display summary table +echo "" +echo "==================================================" +echo "Deployed Nodes Summary:" +echo "==================================================" +printf "%-15s | %-10s | %s\n" "Node" "Mode" "Docker Image" +echo "--------------------------------------------------" +for node in "${spin_nodes[@]}"; do + mode="${node_modes[$node]}" + if [ "$mode" == "docker" ]; then + image="${node_images[$node]}" + printf "%-15s | %-10s | %s\n" "$node" "$mode" "$image" + else + printf "%-15s | %-10s | %s\n" "$node" "$mode" "N/A (binary mode)" + fi +done +echo "==================================================" +echo "" + cleanup() { echo -e "\n\ncleaning up" printf '%*s' $(tput cols) | tr ' ' '-' diff --git a/user-config.yml.example b/user-config.yml.example new file mode 100644 index 0000000..15c0611 --- /dev/null +++ b/user-config.yml.example @@ -0,0 +1,26 @@ +# User Client Configuration Example +# Copy this file to user-config.yml and customize your client images +# Usage: ./spin-node.sh --config-file user-config.yml --node all +# +# Only specify clients you want to override - others will use defaults +# from client-cmds/default-client-config.yml + +clients: + - name: zeam + image: blockblaz/zeam:custom-tag + + - name: ream + image: ghcr.io/reamlabs/ream:my-branch + + # Add more clients as needed: + # - name: qlean + # image: qdrvm/qlean-mini:custom-tag + # + # - name: lantern + # image: piertwo/lantern:custom-tag + # + # - name: lighthouse + # image: hopinheimer/lighthouse:custom-tag + # + # - name: grandine + # image: sifrai/lean:custom-tag From 39dc7a6952f8231d88f022ce5d7e87f298a34c9f Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Wed, 21 Jan 2026 14:06:48 +0100 Subject: [PATCH 02/15] fix: move image to the validator-config.yml --- README.md | 23 ++- ansible-devnet/genesis/validator-config.yaml | 40 +++-- ansible/roles/grandine/tasks/main.yml | 28 +++- ansible/roles/lantern/tasks/main.yml | 28 +++- ansible/roles/lighthouse/tasks/main.yml | 28 +++- ansible/roles/qlean/tasks/main.yml | 28 +++- ansible/roles/ream/tasks/main.yml | 28 +++- ansible/roles/zeam/tasks/main.yml | 31 ++-- client-cmds/default-client-config.yml | 22 --- client-cmds/grandine-cmd.sh | 2 +- client-cmds/lantern-cmd.sh | 2 +- client-cmds/lighthouse-cmd.sh | 2 +- client-cmds/qlean-cmd.sh | 2 +- client-cmds/ream-cmd.sh | 2 +- client-cmds/zeam-cmd.sh | 2 +- load-client-config.sh | 156 ++++++++++++------- local-devnet/genesis/validator-config.yaml | 10 +- run-ansible.sh | 69 ++++++++ spin-node.sh | 23 ++- user-config.yml.example | 26 ++-- 20 files changed, 389 insertions(+), 163 deletions(-) delete mode 100644 client-cmds/default-client-config.yml diff --git a/README.md b/README.md index 0eff1bf..830260d 100644 --- a/README.md +++ b/README.md @@ -76,37 +76,37 @@ You can override default Docker images using the `--config-file` flag. This is u **Basic usage (without custom images):** ```sh -# Uses default images from client-cmds/default-client-config.yml +# Uses default images from validator-config.yaml in your network directory NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis ``` **With custom config file:** ```sh -# Override specific client images +# Override specific node images NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --config-file user-config.yml ``` **Example config file (user-config.yml):** ```yaml -clients: - - name: zeam +nodes: + - name: zeam_0 image: blockblaz/zeam:feature-branch - - name: ream + - name: ream_0 image: ghcr.io/reamlabs/ream:v2.0 ``` **Testing a specific client build:** ```sh # Create custom config file for zeam /my-zeam-config.yml -clients: - - name: zeam +nodes: + - name: zeam_0 image: blockblaz/zeam:custom-tag # Run with custom zeam image NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --config-file /my-zeam-config.yml ``` -Only specify clients you want to override - others will use their defaults from `client-cmds/default-client-config.yml`. +Only specify nodes you want to override - others will use their defaults from `validator-config.yaml`. ## Args @@ -157,9 +157,9 @@ Only specify clients you want to override - others will use their defaults from - If not provided, defaults to `latest` for zeam, ream, and lantern, and `dd67521` for qlean - The script will automatically pull the specified Docker images before running containers - Example: `--tag devnet0` or `--tag devnet1` -11. `--config-file` specifies a custom configuration file to override default Docker images for specific clients. - - Path to a YAML file containing client image overrides (e.g., `user-config.yml` or `/path/to/my-config.yml`) - - Only clients specified in the config file are overridden; others use defaults from `client-cmds/default-client-config.yml` +11. `--config-file` specifies a custom configuration file to override default Docker images for specific nodes. + - Path to a YAML file containing node image overrides (e.g., `user-config.yml` or `/path/to/my-config.yml`) + - Only nodes specified in the config file are overridden; others use defaults from `validator-config.yaml` - See [Using custom Docker images](#using-custom-docker-images) scenario for usage examples - Example: `--config-file user-config.yml` or `--config-file /path/to/custom-config.yml` 12. `--metrics` enables metrics collection on all nodes. When specified, each client will activate its metrics endpoint according to its implementation. Metrics ports are configured per node in `validator-config.yaml`. @@ -192,7 +192,6 @@ The quickstart uses separate directories for local and Ansible deployments: ``` lean-quickstart/ ├── client-cmds/ # Client command scripts -│ ├── default-client-config.yml # Default Docker images for all clients │ ├── zeam-cmd.sh │ ├── ream-cmd.sh │ └── ... diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index d6c9dd2..5f633f7 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -8,6 +8,7 @@ validators: - name: "zeam_0" # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f + image: blockblaz/zeam:latest privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: # verify /ip4/127.0.0.1/udp/9000/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f @@ -19,6 +20,7 @@ validators: - name: "ream_0" # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 + image: ghcr.io/reamlabs/ream:latest privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -30,6 +32,7 @@ validators: - name: "qlean_0" # TODO: add a third entry in nodes.yaml corresponding to this + image: qdrvm/qlean-mini:latest privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -41,6 +44,7 @@ validators: - name: "lantern_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + image: piertwo/lantern:v0.0.1 privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -49,21 +53,23 @@ validators: metricsPort: 8081 count: 1 - - name: "lighthouse_0" - # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 - # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" - # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - enrFields: - ip: "46.224.135.169" - quic: 9001 - metricsPort: 8081 - count: 1 + # - name: "lighthouse_0" + # # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 + # # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + # image: hopinheimer/lighthouse:latest + # privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" + # # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + # enrFields: + # ip: "46.224.135.169" + # quic: 9001 + # metricsPort: 8081 + # count: 1 - - name: "grandine_0" - privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" - enrFields: - ip: "46.62.163.74" - quic: 9001 - metricsPort: 8081 - count: 1 \ No newline at end of file + # - name: "grandine_0" + # image: sifrai/lean:unstable + # privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" + # enrFields: + # ip: "46.62.163.74" + # quic: 9001 + # metricsPort: 8081 + # count: 1 diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index 3bdda8a..be3426a 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -2,12 +2,30 @@ # Grandine role: Deploy and manage Grandine nodes # Converts client-cmds/grandine-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/grandine-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (grandine_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^grandine_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "grandine") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^grandine_")) | .image' "$validator_config" | head -1 register: grandine_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index 9932194..574abbf 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -2,12 +2,30 @@ # Lantern role: Deploy and manage Lantern nodes # Converts client-cmds/lantern-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the docker image from LANTERN_IMAGE line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^LANTERN_IMAGE=' "$project_root/client-cmds/lantern-cmd.sh" | head -1 | sed -E 's/.*LANTERN_IMAGE="([^"]+)".*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (lantern_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^lantern_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "lantern") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^lantern_")) | .image' "$validator_config" | head -1 register: lantern_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index 5103473..869fd01 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -2,12 +2,30 @@ # Lighthouse role: Deploy and manage Lighthouse nodes # Converts client-cmds/lighthouse-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/lighthouse-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (lighthouse_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^lighthouse_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "lighthouse") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^lighthouse_")) | .image' "$validator_config" | head -1 register: lighthouse_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index 09f2c67..fadb768 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -2,12 +2,30 @@ # Qlean role: Deploy and manage Qlean nodes # Converts client-cmds/qlean-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/qlean-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (qlean_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^qlean_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "qlean") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^qlean_")) | .image' "$validator_config" | head -1 register: qlean_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index 1ed3326..9917a7c 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -2,12 +2,30 @@ # Ream role: Deploy and manage Ream nodes # Converts client-cmds/ream-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/ream-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (ream_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^ream_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "ream") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + fi + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^ream_")) | .image' "$validator_config" | head -1 register: ream_docker_image_raw changed_when: false delegate_to: localhost diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index a0936c9..71340bf 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -2,17 +2,30 @@ # Zeam role: Deploy and manage Zeam nodes # Converts client-cmds/zeam-cmd.sh logic to Ansible tasks -- name: Extract docker image from client-cmd.sh +- name: Extract docker image from config file shell: | - # Extract the docker image from node_docker line (find word containing / and :) - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - script_path="$project_root/client-cmds/zeam-cmd.sh" - if [ ! -f "$script_path" ]; then - echo "ERROR: Script not found at $script_path (project_root=$project_root, playbook_dir={{ playbook_dir }})" >&2 - exit 1 + # Read image from user config file if provided, otherwise from validator-config.yaml + validator_config="{{ genesis_dir }}/validator-config.yaml" + user_config="{{ user_config_file | default('') }}" + + # Try user config first if provided (supports both 'nodes' and 'clients' format) + if [ -n "$user_config" ] && [ -f "$user_config" ]; then + # Try nodes format first (zeam_0 style) + image=$(yq eval '.nodes[] | select(.name | test("^zeam_")) | .image' "$user_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi + # Try clients format (backwards compatibility) + image=$(yq eval '.clients[] | select(.name == "zeam") | .image' "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ]; then + echo "$image" + exit 0 + fi fi - grep -E '^node_docker=' "$script_path" | head -1 | grep -oE '[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+:[a-zA-Z0-9._-]+' | head -1 + + # Fall back to validator-config.yaml (read from validators array) + yq eval '.validators[] | select(.name | test("^zeam_")) | .image' "$validator_config" | head -1 register: zeam_docker_image_raw changed_when: false delegate_to: localhost diff --git a/client-cmds/default-client-config.yml b/client-cmds/default-client-config.yml deleted file mode 100644 index cc700c0..0000000 --- a/client-cmds/default-client-config.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Default Client Configuration -# This file contains the default Docker images for all supported clients -# These defaults are used when no custom config file is specified via --config-file - -clients: - - name: zeam - image: blockblaz/zeam:latest - - - name: ream - image: ghcr.io/reamlabs/ream:latest - - - name: qlean - image: qdrvm/qlean-mini:latest - - - name: lantern - image: piertwo/lantern:latest - - - name: lighthouse - image: hopinheimer/lighthouse:latest - - - name: grandine - image: sifrai/lean:latest diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index f07e6a8..0541367 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # grandineImage is exported by spin-node.sh before sourcing this file node_binary="$grandine_bin \ diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index cd3dcc3..0aa98bb 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -1,7 +1,7 @@ #!/bin/bash #-----------------------lantern setup---------------------- -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # lanternImage is exported by spin-node.sh before sourcing this file devnet_flag="" diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index 51938c6..2c345ab 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -3,7 +3,7 @@ # Metrics enabled by default metrics_flag="--metrics" -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # lighthouseImage is exported by spin-node.sh before sourcing this file node_binary="$lighthouse_bin lean_node \ diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index 689e650..6f12f88 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -4,7 +4,7 @@ # expects "qlean" submodule or symlink inside "lean-quickstart" root directory # https://github.com/qdrvm/qlean-mini -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # qleanImage is exported by spin-node.sh before sourcing this file node_binary="$scriptDir/qlean/build/src/executable/qlean \ diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index ad30479..a4f191c 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -4,7 +4,7 @@ # Metrics enabled by default metrics_flag="--metrics" -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # reamImage is exported by spin-node.sh before sourcing this file # modify the path to the ream binary as per your system diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 23c2085..f8aa964 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -6,7 +6,7 @@ # Metrics enabled by default metrics_flag="--metrics_enable" -# Docker image (set from default-client-config.yml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --config-file) # zeamImage is exported by spin-node.sh before sourcing this file node_binary="$scriptDir/../zig-out/bin/zeam node \ diff --git a/load-client-config.sh b/load-client-config.sh index 3d014aa..1800cc2 100644 --- a/load-client-config.sh +++ b/load-client-config.sh @@ -1,31 +1,11 @@ #!/bin/bash -# Load client configuration from default and optional user config files +# Load client configuration from validator-config.yaml and optional user config file -# Arrays to store client names and images (bash 3.2 compatible) +# Associative array to store client images (requires bash 4+, but we'll use parallel arrays for bash 3.2 compatibility) CLIENT_NAMES=() CLIENT_IMAGES_LIST=() KNOWN_CLIENTS=("zeam" "ream" "qlean" "lantern" "lighthouse" "grandine") -# Function to load default config -load_default_config() { - local default_config="$scriptDir/client-cmds/default-client-config.yml" - - if [ ! -f "$default_config" ]; then - echo "⚠️ Warning: Default config not found at $default_config" - return 1 - fi - - # Load default images using yq - while IFS= read -r line; do - local client_name=$(echo "$line" | awk '{print $1}') - local client_image=$(echo "$line" | awk '{print $2}') - CLIENT_NAMES+=("$client_name") - CLIENT_IMAGES_LIST+=("$client_image") - done < <(yq eval '.clients[] | .name + " " + .image' "$default_config") - - echo "✓ Loaded default client images from $default_config" -} - # Function to find index of client name find_client_index() { local search_name="$1" @@ -40,7 +20,35 @@ find_client_index() { echo "-1" } +# Function to extract client type from node name (e.g., zeam_0 -> zeam) +get_client_type() { + echo "$1" | sed 's/_[0-9]*$//' +} + +# Function to load default config from validator-config.yaml +load_default_config() { + local validator_config="$configDir/validator-config.yaml" + + if [ ! -f "$validator_config" ]; then + echo "⚠️ Warning: validator-config.yaml not found at $validator_config" + return 1 + fi + + # Load images from validators array using yq + for client in "${KNOWN_CLIENTS[@]}"; do + # Find the first validator matching this client type (e.g., zeam_0 for zeam) + local image=$(yq eval ".validators[] | select(.name | test(\"^${client}_\")) | .image" "$validator_config" 2>/dev/null | head -1) + if [ -n "$image" ] && [ "$image" != "null" ]; then + CLIENT_NAMES+=("$client") + CLIENT_IMAGES_LIST+=("$image") + fi + done + + echo "✓ Loaded default client images from $validator_config" +} + # Function to load user config and override defaults +# User config format uses node names (e.g., zeam_0) instead of client names load_user_config() { local user_config="$1" @@ -55,40 +63,84 @@ load_user_config() { echo "Loading user config from $user_config..." - # Load user-specified images + # Load user-specified images (supports both 'nodes' and 'clients' format for backwards compatibility) local override_count=0 - while IFS= read -r line; do - local client_name=$(echo "$line" | awk '{print $1}') - local client_image=$(echo "$line" | awk '{print $2}') - - # Validate client name - local valid_client=false - for known in "${KNOWN_CLIENTS[@]}"; do - if [ "$client_name" == "$known" ]; then - valid_client=true - break + + # Try 'nodes' format first (new format with zeam_0, etc.) + local nodes_exist=$(yq eval '.nodes // empty' "$user_config" 2>/dev/null) + + if [ -n "$nodes_exist" ]; then + # New format: nodes with zeam_0 style names + while IFS= read -r line; do + local node_name=$(echo "$line" | awk '{print $1}') + local node_image=$(echo "$line" | awk '{print $2}') + + # Extract client type from node name (zeam_0 -> zeam) + local client_type=$(get_client_type "$node_name") + + # Validate client type + local valid_client=false + for known in "${KNOWN_CLIENTS[@]}"; do + if [ "$client_type" == "$known" ]; then + valid_client=true + break + fi + done + + if [ "$valid_client" == "false" ]; then + echo "⚠️ Warning: Unknown client type '$client_type' from node '$node_name' - skipping" + continue fi - done - if [ "$valid_client" == "false" ]; then - echo "⚠️ Warning: Unknown client '$client_name' in config - skipping" - continue - fi + # Validate image format (basic check) + if [[ ! "$node_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then + echo "⚠️ Warning: Invalid image format '$node_image' for node '$node_name' - using default" + continue + fi - # Validate image format (basic check) - if [[ ! "$client_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then - echo "⚠️ Warning: Invalid image format '$client_image' for client '$client_name' - using default" - continue - fi + # Find and update the client image + local idx=$(find_client_index "$client_type") + if [ "$idx" != "-1" ]; then + CLIENT_IMAGES_LIST[$idx]="$node_image" + echo " ✓ Override $client_type (from $node_name): $node_image" + ((override_count++)) + fi + done < <(yq eval '.nodes[] | .name + " " + .image' "$user_config" 2>/dev/null) + else + # Fallback: try 'clients' format (old format for backwards compatibility) + while IFS= read -r line; do + local client_name=$(echo "$line" | awk '{print $1}') + local client_image=$(echo "$line" | awk '{print $2}') + + # Validate client name + local valid_client=false + for known in "${KNOWN_CLIENTS[@]}"; do + if [ "$client_name" == "$known" ]; then + valid_client=true + break + fi + done + + if [ "$valid_client" == "false" ]; then + echo "⚠️ Warning: Unknown client '$client_name' in config - skipping" + continue + fi - # Find and update the client image - local idx=$(find_client_index "$client_name") - if [ "$idx" != "-1" ]; then - CLIENT_IMAGES_LIST[$idx]="$client_image" - echo " ✓ Override $client_name: $client_image" - ((override_count++)) - fi - done < <(yq eval '.clients[] | .name + " " + .image' "$user_config" 2>/dev/null) + # Validate image format (basic check) + if [[ ! "$client_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then + echo "⚠️ Warning: Invalid image format '$client_image' for client '$client_name' - using default" + continue + fi + + # Find and update the client image + local idx=$(find_client_index "$client_name") + if [ "$idx" != "-1" ]; then + CLIENT_IMAGES_LIST[$idx]="$client_image" + echo " ✓ Override $client_name: $client_image" + ((override_count++)) + fi + done < <(yq eval '.clients[] | .name + " " + .image' "$user_config" 2>/dev/null) + fi if [ $override_count -eq 0 ]; then echo "⚠️ No valid overrides found in user config" @@ -125,7 +177,7 @@ display_client_config() { echo "" } -# Load default configuration +# Load default configuration from validator-config.yaml load_default_config # Load user configuration if provided diff --git a/local-devnet/genesis/validator-config.yaml b/local-devnet/genesis/validator-config.yaml index 9e1a058..0278d85 100644 --- a/local-devnet/genesis/validator-config.yaml +++ b/local-devnet/genesis/validator-config.yaml @@ -8,6 +8,7 @@ validators: - name: "zeam_0" # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f + image: blockblaz/zeam:latest privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: # verify /ip4/127.0.0.1/udp/9000/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f @@ -19,6 +20,7 @@ validators: - name: "ream_0" # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 + image: ghcr.io/reamlabs/ream:latest privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -30,6 +32,7 @@ validators: - name: "qlean_0" # TODO: add a third entry in nodes.yaml corresponding to this + image: qdrvm/qlean-mini:latest privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -41,6 +44,7 @@ validators: - name: "lantern_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + image: piertwo/lantern:v0.0.1 privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -48,10 +52,11 @@ validators: quic: 9004 metricsPort: 8084 count: 1 - + - name: "lighthouse_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + image: hopinheimer/lighthouse:latest privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -61,9 +66,10 @@ validators: count: 1 - name: "grandine_0" + image: sifrai/lean:unstable privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" enrFields: ip: "127.0.0.1" quic: 9006 metricsPort: 8086 - count: 1 \ No newline at end of file + count: 1 diff --git a/run-ansible.sh b/run-ansible.sh index a613417..f138b0a 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -26,6 +26,7 @@ validator_config_file="$5" sshKeyFile="$6" useRoot="$7" # Flag to use root user (defaults to current user) action="$8" # Action: "stop" to stop nodes, otherwise deploy +userConfigFile="$9" # User config file for docker images (--config-file) # Determine SSH user: use root if --useRoot flag is set, otherwise use current user if [ "$useRoot" == "true" ]; then @@ -119,6 +120,16 @@ fi # Use default deployment mode (can be overridden by adding a 'deployment_mode' field per node in validator-config.yaml) EXTRA_VARS="$EXTRA_VARS deployment_mode=$DEFAULT_DEPLOYMENT_MODE" +# Pass user config file path for docker images if provided +if [ -n "$userConfigFile" ]; then + # Convert to absolute path if relative + if [[ "$userConfigFile" != /* ]]; then + userConfigFile="$(cd "$(dirname "$userConfigFile")" 2>/dev/null && pwd)/$(basename "$userConfigFile")" + fi + EXTRA_VARS="$EXTRA_VARS user_config_file=$userConfigFile" + echo "Using user config file for docker images: $userConfigFile" +fi + # Determine which playbook to run if [ "$action" == "stop" ]; then PLAYBOOK="$ANSIBLE_DIR/playbooks/stop-nodes.yml" @@ -148,6 +159,64 @@ if [ $EXIT_CODE -eq 0 ]; then if [ "$action" == "stop" ]; then echo "✅ Ansible stop operation completed successfully!" else + # Display summary of deployed nodes + echo "" + echo "==================================================" + echo "Deployed Nodes Summary:" + echo "==================================================" + printf "%-15s | %-10s | %s\n" "Node" "Mode" "Image / Binary Path" + echo "--------------------------------------------------" + + # Parse node names (comma or space separated) + if [[ "$node" == *","* ]]; then + IFS=',' read -r -a deployed_nodes <<< "$node" + else + IFS=' ' read -r -a deployed_nodes <<< "$node" + fi + + # Get config file paths - use validator-config.yaml from genesis dir + validator_config="$configDir/validator-config.yaml" + + for node_name in "${deployed_nodes[@]}"; do + # Extract client type from node name (e.g., zeam_0 -> zeam) + client_type=$(echo "$node_name" | sed 's/_[0-9]*$//') + + if [ "$DEFAULT_DEPLOYMENT_MODE" == "binary" ]; then + # Extract binary path from client-cmd.sh + client_cmd_file="$scriptDir/client-cmds/${client_type}-cmd.sh" + binary_path="" + if [ -f "$client_cmd_file" ]; then + # Extract the first word from node_binary line (the actual binary path) + binary_path=$(grep -E '^node_binary=' "$client_cmd_file" | head -1 | sed -E 's/node_binary="([^ "\\]+).*/\1/') + # Resolve $scriptDir to actual path + binary_path=$(echo "$binary_path" | sed "s|\\\$scriptDir|$scriptDir|g") + # For unresolved variables (like $grandine_bin), show as custom binary path + if [[ "$binary_path" == \$* ]]; then + binary_path="" + fi + fi + printf "%-15s | %-10s | %s\n" "$node_name" "binary" "${binary_path:-unknown}" + else + # Get image from user config or validator-config.yaml + docker_image="" + if [ -n "$userConfigFile" ] && [ -f "$userConfigFile" ]; then + # Try nodes format first (zeam_0 style) + docker_image=$(yq eval ".nodes[] | select(.name | test(\"^${client_type}_\")) | .image" "$userConfigFile" 2>/dev/null | head -1) + # Try clients format (backwards compatibility) + if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then + docker_image=$(yq eval ".clients[] | select(.name == \"$client_type\") | .image" "$userConfigFile" 2>/dev/null) + fi + fi + if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then + # Read from validators array in validator-config.yaml + docker_image=$(yq eval ".validators[] | select(.name | test(\"^${client_type}_\")) | .image" "$validator_config" 2>/dev/null | head -1) + fi + + printf "%-15s | %-10s | %s\n" "$node_name" "docker" "${docker_image:-unknown}" + fi + done + echo "==================================================" + echo "" echo "✅ Ansible deployment completed successfully!" fi else diff --git a/spin-node.sh b/spin-node.sh index 8af9957..7d77829 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -148,16 +148,16 @@ if [ "$deployment_mode" == "ansible" ]; then # Handle stop action if [ -n "$stopNodes" ] && [ "$stopNodes" == "true" ]; then echo "Stopping nodes via Ansible..." - if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "stop"; then + if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "stop" "$configFile"; then echo "❌ Ansible stop operation failed. Exiting." exit 1 fi exit 0 fi - + # Call separate Ansible execution script # If Ansible deployment fails, exit immediately (don't fall through to local deployment) - if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot"; then + if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "" "$configFile"; then echo "❌ Ansible deployment failed. Exiting." exit 1 fi @@ -254,7 +254,7 @@ for item in "${spin_nodes[@]}"; do IFS='_' read -r -a elements <<< "$item" client="${elements[0]}" - # Get docker image from config (always set - from default-client-config.yml or user override) + # Get docker image from config (always set - from validator-config.yaml or user override) client_image=$(get_client_image "$client") # Export the image variable for this client (will be used by client-cmd.sh) @@ -300,12 +300,25 @@ for item in "${spin_nodes[@]}"; do else # Extract image name from node_docker (find word containing ':' which is the image:tag) docker_image=$(echo "$node_docker" | grep -oE '[^ ]+:[^ ]+' | head -1) - # Pull image first + # Pull image first if [ -n "$dockerWithSudo" ]; then sudo docker pull "$docker_image" || true else docker pull "$docker_image" || true fi + + # Check if the image exists locally (either pulled successfully or was cached) + if [ -n "$dockerWithSudo" ]; then + image_exists=$(sudo docker image inspect "$docker_image" > /dev/null 2>&1 && echo "yes" || echo "no") + else + image_exists=$(docker image inspect "$docker_image" > /dev/null 2>&1 && echo "yes" || echo "no") + fi + + if [ "$image_exists" == "no" ]; then + echo "⚠️ Skipping $item: Docker image '$docker_image' does not exist and could not be pulled" + continue + fi + execCmd="docker run --rm --pull=never" if [ -n "$dockerWithSudo" ] then diff --git a/user-config.yml.example b/user-config.yml.example index 15c0611..70c237c 100644 --- a/user-config.yml.example +++ b/user-config.yml.example @@ -1,26 +1,26 @@ -# User Client Configuration Example -# Copy this file to user-config.yml and customize your client images -# Usage: ./spin-node.sh --config-file user-config.yml --node all +# User Configuration Example +# Copy this file to user-config.yml or .yml and customize your docker images +# Usage: ./spin-node.sh --config-file .yml --node zeam_0,ream_0 # -# Only specify clients you want to override - others will use defaults -# from client-cmds/default-client-config.yml +# Only specify nodes you want to override - others will use defaults +# from validator-config.yaml in your network directory -clients: - - name: zeam +nodes: + - name: zeam_0 image: blockblaz/zeam:custom-tag - - name: ream + - name: ream_0 image: ghcr.io/reamlabs/ream:my-branch - # Add more clients as needed: - # - name: qlean + # Add more nodes as needed: + # - name: qlean_0 # image: qdrvm/qlean-mini:custom-tag # - # - name: lantern + # - name: lantern_0 # image: piertwo/lantern:custom-tag # - # - name: lighthouse + # - name: lighthouse_0 # image: hopinheimer/lighthouse:custom-tag # - # - name: grandine + # - name: grandine_0 # image: sifrai/lean:custom-tag From dead69cbdb4c592dd07851d67db053287aa38ff9 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Tue, 3 Feb 2026 10:53:42 +0100 Subject: [PATCH 03/15] fix: add deploy-validator-config.yaml, delete annotated-validators.yaml --- .gitignore | 4 +- README.md | 35 +--- ansible-devnet/genesis/validator-config.yaml | 11 +- ansible/playbooks/copy-genesis.yml | 2 - ansible/playbooks/deploy-nodes.yml | 4 +- ansible/playbooks/generate-genesis.yml | 1 - ansible/roles/ethlambda/tasks/main.yml | 40 +--- ansible/roles/grandine/tasks/main.yml | 58 +----- ansible/roles/lantern/tasks/main.yml | 59 +----- ansible/roles/lighthouse/tasks/main.yml | 59 +----- ansible/roles/qlean/tasks/main.yml | 68 ++----- ansible/roles/ream/tasks/main.yml | 59 +----- ansible/roles/zeam/tasks/main.yml | 63 +------ client-cmds/ethlambda-cmd.sh | 5 +- client-cmds/grandine-cmd.sh | 2 +- client-cmds/lantern-cmd.sh | 2 +- client-cmds/lighthouse-cmd.sh | 2 +- client-cmds/qlean-cmd.sh | 2 +- client-cmds/ream-cmd.sh | 2 +- client-cmds/zeam-cmd.sh | 2 +- generate-genesis.sh | 97 ---------- load-client-config.sh | 186 ------------------- local-devnet/genesis/validator-config.yaml | 3 +- merge-config.sh | 74 ++++++++ parse-env.sh | 2 +- parse-vc.sh | 71 +++---- run-ansible.sh | 45 +---- spin-node.sh | 61 +++--- user-config.yml.example | 13 +- 29 files changed, 252 insertions(+), 780 deletions(-) delete mode 100644 load-client-config.sh create mode 100755 merge-config.sh diff --git a/.gitignore b/.gitignore index 37e6f92..f61eae5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,7 +13,9 @@ validators.yaml nodes.yaml genesis.json genesis.ssz -annotated_validators.yaml + +# Deploy config (created by merge-config.sh, merges validator-config + user overrides) +deploy-validator-config.yaml # Hash-sig validator keys (contains sensitive secret keys) hash-sig-keys/ diff --git a/README.md b/README.md index ab7d829..26dce97 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --metrics ### Using custom Docker images -You can override default Docker images using the `--config-file` flag. This is useful for testing custom builds or using specific versions without modifying the codebase. +You can override default Docker images using the `--configFile` flag. This is useful for testing custom builds or using specific versions without modifying the codebase. **Basic usage (without custom images):** ```sh @@ -83,7 +83,7 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis **With custom config file:** ```sh # Override specific node images -NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --config-file user-config.yml +NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --configFile user-config.yml ``` **Example config file (user-config.yml):** @@ -103,7 +103,7 @@ nodes: image: blockblaz/zeam:custom-tag # Run with custom zeam image -NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --config-file /my-zeam-config.yml +NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --configFile /my-zeam-config.yml ``` Only specify nodes you want to override - others will use their defaults from `validator-config.yaml`. @@ -157,11 +157,11 @@ Only specify nodes you want to override - others will use their defaults from `v - If not provided, defaults to `latest` for zeam, ream, and lantern, and `dd67521` for qlean - The script will automatically pull the specified Docker images before running containers - Example: `--tag devnet0` or `--tag devnet1` -11. `--config-file` specifies a custom configuration file to override default Docker images for specific nodes. +11. `--configFile` specifies a custom configuration file to override default Docker images for specific nodes. - Path to a YAML file containing node image overrides (e.g., `user-config.yml` or `/path/to/my-config.yml`) - Only nodes specified in the config file are overridden; others use defaults from `validator-config.yaml` - See [Using custom Docker images](#using-custom-docker-images) scenario for usage examples - - Example: `--config-file user-config.yml` or `--config-file /path/to/custom-config.yml` + - Example: `--configFile user-config.yml` or `--configFile /path/to/custom-config.yml` 12. `--metrics` enables metrics collection on all nodes. When specified, each client will activate its metrics endpoint according to its implementation. Metrics ports are configured per node in `validator-config.yaml`. ### Clients supported @@ -328,31 +328,6 @@ qlean_0: - 2 ``` -**Recommended:** `annotated_validators.yaml` is also generated and should be preferred by client software as it includes public keys and private key file references directly, eliminating the need for clients to derive key filenames from validator indices: - -```yaml -zeam_0: - - index: 0 - pubkey_hex: 4b3c31094bcc9b45446b2028eae5ad192b2df16778837b10230af102255c9c5f72d7ba43eae30b2c6a779f47367ebf5a42f6c959 - privkey_file: validator_0_sk.json - - index: 3 - pubkey_hex: 8df32a54d2fbdf3a88035b2fe3931320cb900d364d6e7c56b19c0f3c6006ce5b3ebe802a65fe1b420183f62e830a953cb33b7804 - privkey_file: validator_3_sk.json - -ream_0: - - index: 1 - pubkey_hex: 5b15f72f90bd655b039f9839c36951454b89c605f8c334581cfa832bdd0c994a1350094f7e22617d77607b067b0aa2439e0ead7d - privkey_file: validator_1_sk.json - - index: 4 - pubkey_hex: 71bf8f73980591574de34a0db471da74f5cfd84d4731d53f47bf3023b26c2638ac5bd24993ea71492fedbd6c4afe5c299213b76b - privkey_file: validator_4_sk.json - -qlean_0: - - index: 2 - pubkey_hex: b87e69568a347d1aa811cc158634fb1f4e247c5509ad2b1652a8d758ec0ab0796954e307b97dd6284fbb30088c2e595546fdf663 - privkey_file: validator_2_sk.json -``` - `nodes.yaml` provide enrs of all the nodes so that clients don't have to run a discovery protocol: ```yaml diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index cde5717..2b13eb1 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -8,7 +8,7 @@ validators: - name: "zeam_0" # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f - image: blockblaz/zeam:latest + image: blockblaz/zeam:devnet2 privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: # verify /ip4/127.0.0.1/udp/9000/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f @@ -20,7 +20,7 @@ validators: - name: "ream_0" # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 - image: ghcr.io/reamlabs/ream:latest + image: ghcr.io/reamlabs/ream:latest-devnet2 privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -32,7 +32,7 @@ validators: - name: "qlean_0" # TODO: add a third entry in nodes.yaml corresponding to this - image: qdrvm/qlean-mini:latest + image: qdrvm/qlean-mini:devnet-2 privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -44,7 +44,7 @@ validators: - name: "lantern_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - image: piertwo/lantern:v0.0.1 + image: piertwo/lantern:v0.0.2 privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -56,6 +56,7 @@ validators: - name: "lighthouse_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv + image: hopinheimer/lighthouse:latest privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -65,6 +66,7 @@ validators: count: 1 - name: "grandine_0" + image: sifrai/lean:devnet-2 privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" enrFields: ip: "46.62.163.74" @@ -73,6 +75,7 @@ validators: count: 1 - name: "ethlambda_0" + image: ghcr.io/lambdaclass/ethlambda:devnet2 privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" enrFields: ip: "91.99.52.24" diff --git a/ansible/playbooks/copy-genesis.yml b/ansible/playbooks/copy-genesis.yml index 76a25f8..c8718ac 100644 --- a/ansible/playbooks/copy-genesis.yml +++ b/ansible/playbooks/copy-genesis.yml @@ -47,7 +47,6 @@ loop: - config.yaml - validators.yaml - - annotated_validators.yaml - nodes.yaml - genesis.json - genesis.ssz @@ -137,7 +136,6 @@ loop: - config.yaml - validators.yaml - - annotated_validators.yaml - nodes.yaml - genesis.json - genesis.ssz diff --git a/ansible/playbooks/deploy-nodes.yml b/ansible/playbooks/deploy-nodes.yml index f89610a..891b66f 100644 --- a/ansible/playbooks/deploy-nodes.yml +++ b/ansible/playbooks/deploy-nodes.yml @@ -117,9 +117,9 @@ - deploy - sync - - name: Sync validator-config.yaml to remote host + - name: Sync deploy-validator-config.yaml to remote host (as validator-config.yaml) copy: - src: "{{ local_genesis_dir }}/validator-config.yaml" + src: "{{ local_genesis_dir }}/deploy-validator-config.yaml" dest: "{{ genesis_dir }}/validator-config.yaml" mode: '0644' force: yes diff --git a/ansible/playbooks/generate-genesis.yml b/ansible/playbooks/generate-genesis.yml index 7006d9c..fde89b9 100644 --- a/ansible/playbooks/generate-genesis.yml +++ b/ansible/playbooks/generate-genesis.yml @@ -81,7 +81,6 @@ - nodes.yaml - genesis.json - genesis.ssz - - annotated_validators.yaml {% for node in node_names_list %} - {{ node }}.key {% endfor %} diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index 3fda6f8..2509552 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -1,49 +1,25 @@ --- # Ethlambda role: Deploy and manage Ethlambda nodes # Converts client-cmds/ethlambda-cmd.sh logic to Ansible tasks - -- name: Extract docker image from client-cmd.sh - shell: | - # Extract the first word (docker image) from node_docker line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_docker=' "$project_root/client-cmds/ethlambda-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' - register: ethlambda_docker_image_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Extract deployment mode from client-cmd.sh - shell: | - # Extract the value from node_setup line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_setup=' "$project_root/client-cmds/ethlambda-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' - register: ethlambda_deployment_mode_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Set docker image and deployment mode from client-cmd.sh - set_fact: - ethlambda_docker_image: "{{ ethlambda_docker_image_raw.stdout | trim | default('ghcr.io/lambdaclass/ethlambda:devnet1') }}" - deployment_mode: "{{ ethlambda_deployment_mode_raw.stdout | trim | default('docker') }}" +# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) - name: Extract node configuration from validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" register: ethlambda_node_config changed_when: false - delegate_to: localhost loop: + - image - enrFields.quic - metricsPort when: node_name is defined -- name: Set node ports +- name: Set node configuration set_fact: - ethlambda_quic_port: "{{ ethlambda_node_config.results[0].stdout }}" - ethlambda_metrics_port: "{{ ethlambda_node_config.results[1].stdout }}" + ethlambda_docker_image: "{{ ethlambda_node_config.results[0].stdout | trim | default('ghcr.io/lambdaclass/ethlambda:latest') }}" + ethlambda_quic_port: "{{ ethlambda_node_config.results[1].stdout | trim }}" + ethlambda_metrics_port: "{{ ethlambda_node_config.results[2].stdout | trim }}" + deployment_mode: "{{ deployment_mode | default('docker') }}" when: ethlambda_node_config is defined - name: Ensure node key file exists diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index 0e9d8f3..25e1679 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -1,67 +1,23 @@ --- # Grandine role: Deploy and manage Grandine nodes # Converts client-cmds/grandine-cmd.sh logic to Ansible tasks - -- name: Extract docker image from config file - shell: | - # Read image from user config file if provided, otherwise from validator-config.yaml - validator_config="{{ genesis_dir }}/validator-config.yaml" - user_config="{{ user_config_file | default('') }}" - - # Try user config first if provided (supports both 'nodes' and 'clients' format) - if [ -n "$user_config" ] && [ -f "$user_config" ]; then - # Try nodes format first (grandine_0 style) - image=$(yq eval '.nodes[] | select(.name | test("^grandine_")) | .image' "$user_config" 2>/dev/null | head -1) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - # Try clients format (backwards compatibility) - image=$(yq eval '.clients[] | select(.name == "grandine") | .image' "$user_config" 2>/dev/null) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - fi - - # Fall back to validator-config.yaml (read from validators array) - yq eval '.validators[] | select(.name | test("^grandine_")) | .image' "$validator_config" | head -1 - register: grandine_docker_image_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Extract deployment mode from client-cmd.sh - shell: | - # Extract the value from node_setup line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_setup=' "$project_root/client-cmds/grandine-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' - register: grandine_deployment_mode_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Set docker image and deployment mode from client-cmd.sh - set_fact: - grandine_docker_image: "{{ grandine_docker_image_raw.stdout | trim | default('sifrai/lean:unstable') }}" - deployment_mode: "{{ grandine_deployment_mode_raw.stdout | trim | default('docker') }}" +# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) - name: Extract node configuration from validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" register: grandine_node_config changed_when: false - delegate_to: localhost loop: + - image - enrFields.quic - - privkey when: node_name is defined -- name: Set node ports +- name: Set node configuration set_fact: - grandine_quic_port: "{{ grandine_node_config.results[0].stdout }}" - grandine_privkey: "{{ grandine_node_config.results[1].stdout }}" + grandine_docker_image: "{{ grandine_node_config.results[0].stdout | trim | default('sifrai/lean:unstable') }}" + grandine_quic_port: "{{ grandine_node_config.results[1].stdout | trim }}" + deployment_mode: "{{ deployment_mode | default('docker') }}" when: grandine_node_config is defined - name: Ensure node key file exists diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index 4a6663c..54e3829 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -1,68 +1,25 @@ --- # Lantern role: Deploy and manage Lantern nodes # Converts client-cmds/lantern-cmd.sh logic to Ansible tasks - -- name: Extract docker image from config file - shell: | - # Read image from user config file if provided, otherwise from validator-config.yaml - validator_config="{{ genesis_dir }}/validator-config.yaml" - user_config="{{ user_config_file | default('') }}" - - # Try user config first if provided (supports both 'nodes' and 'clients' format) - if [ -n "$user_config" ] && [ -f "$user_config" ]; then - # Try nodes format first (lantern_0 style) - image=$(yq eval '.nodes[] | select(.name | test("^lantern_")) | .image' "$user_config" 2>/dev/null | head -1) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - # Try clients format (backwards compatibility) - image=$(yq eval '.clients[] | select(.name == "lantern") | .image' "$user_config" 2>/dev/null) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - fi - - # Fall back to validator-config.yaml (read from validators array) - yq eval '.validators[] | select(.name | test("^lantern_")) | .image' "$validator_config" | head -1 - register: lantern_docker_image_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Extract deployment mode from client-cmd.sh - shell: | - # Extract the value from node_setup line - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_setup=' "$project_root/client-cmds/lantern-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' - register: lantern_deployment_mode_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Set docker image and deployment mode from client-cmd.sh - set_fact: - lantern_docker_image: "{{ lantern_docker_image_raw.stdout | trim | default('piertwo/lantern:v0.0.1') }}" - deployment_mode: "{{ lantern_deployment_mode_raw.stdout | trim | default('docker') }}" +# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) - name: Extract node configuration from validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" register: lantern_node_config changed_when: false - delegate_to: localhost loop: + - image - enrFields.quic - metricsPort - - privkey when: node_name is defined -- name: Set node ports +- name: Set node configuration set_fact: - lantern_quic_port: "{{ lantern_node_config.results[0].stdout }}" - lantern_metrics_port: "{{ lantern_node_config.results[1].stdout }}" - lantern_privkey: "{{ lantern_node_config.results[2].stdout }}" + lantern_docker_image: "{{ lantern_node_config.results[0].stdout | trim | default('piertwo/lantern:latest') }}" + lantern_quic_port: "{{ lantern_node_config.results[1].stdout | trim }}" + lantern_metrics_port: "{{ lantern_node_config.results[2].stdout | trim }}" + deployment_mode: "{{ deployment_mode | default('docker') }}" when: lantern_node_config is defined - name: Ensure node key file exists diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index 43b6a2f..db66fd0 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -1,68 +1,25 @@ --- # Lighthouse role: Deploy and manage Lighthouse nodes # Converts client-cmds/lighthouse-cmd.sh logic to Ansible tasks - -- name: Extract docker image from config file - shell: | - # Read image from user config file if provided, otherwise from validator-config.yaml - validator_config="{{ genesis_dir }}/validator-config.yaml" - user_config="{{ user_config_file | default('') }}" - - # Try user config first if provided (supports both 'nodes' and 'clients' format) - if [ -n "$user_config" ] && [ -f "$user_config" ]; then - # Try nodes format first (lighthouse_0 style) - image=$(yq eval '.nodes[] | select(.name | test("^lighthouse_")) | .image' "$user_config" 2>/dev/null | head -1) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - # Try clients format (backwards compatibility) - image=$(yq eval '.clients[] | select(.name == "lighthouse") | .image' "$user_config" 2>/dev/null) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - fi - - # Fall back to validator-config.yaml (read from validators array) - yq eval '.validators[] | select(.name | test("^lighthouse_")) | .image' "$validator_config" | head -1 - register: lighthouse_docker_image_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Extract deployment mode from client-cmd.sh - shell: | - # Extract the value from node_setup line - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_setup=' "$project_root/client-cmds/lighthouse-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' - register: lighthouse_deployment_mode_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Set docker image and deployment mode from client-cmd.sh - set_fact: - lighthouse_docker_image: "{{ lighthouse_docker_image_raw.stdout | trim | default('hopinheimer/lighthouse:latest') }}" - deployment_mode: "{{ lighthouse_deployment_mode_raw.stdout | trim | default('docker') }}" +# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) - name: Extract node configuration from validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" register: lighthouse_node_config changed_when: false - delegate_to: localhost loop: + - image - enrFields.quic - metricsPort - - privkey when: node_name is defined -- name: Set node ports +- name: Set node configuration set_fact: - lighthouse_quic_port: "{{ lighthouse_node_config.results[0].stdout }}" - lighthouse_metrics_port: "{{ lighthouse_node_config.results[1].stdout }}" - lighthouse_privkey: "{{ lighthouse_node_config.results[2].stdout }}" + lighthouse_docker_image: "{{ lighthouse_node_config.results[0].stdout | trim | default('hopinheimer/lighthouse:latest') }}" + lighthouse_quic_port: "{{ lighthouse_node_config.results[1].stdout | trim }}" + lighthouse_metrics_port: "{{ lighthouse_node_config.results[2].stdout | trim }}" + deployment_mode: "{{ deployment_mode | default('docker') }}" when: lighthouse_node_config is defined - name: Ensure node key file exists diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index 8348b2a..2d7afa3 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -1,82 +1,38 @@ --- # Qlean role: Deploy and manage Qlean nodes # Converts client-cmds/qlean-cmd.sh logic to Ansible tasks - -- name: Extract docker image from config file - shell: | - # Read image from user config file if provided, otherwise from validator-config.yaml - validator_config="{{ genesis_dir }}/validator-config.yaml" - user_config="{{ user_config_file | default('') }}" - - # Try user config first if provided (supports both 'nodes' and 'clients' format) - if [ -n "$user_config" ] && [ -f "$user_config" ]; then - # Try nodes format first (qlean_0 style) - image=$(yq eval '.nodes[] | select(.name | test("^qlean_")) | .image' "$user_config" 2>/dev/null | head -1) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - # Try clients format (backwards compatibility) - image=$(yq eval '.clients[] | select(.name == "qlean") | .image' "$user_config" 2>/dev/null) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - fi - - # Fall back to validator-config.yaml (read from validators array) - yq eval '.validators[] | select(.name | test("^qlean_")) | .image' "$validator_config" | head -1 - register: qlean_docker_image_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Extract deployment mode from client-cmd.sh - shell: | - # Extract the value from node_setup line - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_setup=' "$project_root/client-cmds/qlean-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' - register: qlean_deployment_mode_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Set docker image and deployment mode from client-cmd.sh - set_fact: - qlean_docker_image: "{{ qlean_docker_image_raw.stdout | trim | default('qdrvm/qlean-mini:dd67521') }}" - deployment_mode: "{{ qlean_deployment_mode_raw.stdout | trim | default('docker') }}" +# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) - name: Extract node configuration from validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" register: qlean_node_config changed_when: false - delegate_to: localhost loop: + - image - enrFields.quic - metricsPort - - privkey when: node_name is defined -- name: Set node ports +- name: Set node configuration set_fact: - qlean_quic_port: "{{ qlean_node_config.results[0].stdout }}" - qlean_metrics_port: "{{ qlean_node_config.results[1].stdout }}" - qlean_privkey: "{{ qlean_node_config.results[2].stdout }}" + qlean_docker_image: "{{ qlean_node_config.results[0].stdout | trim | default('qdrvm/qlean-mini:latest') }}" + qlean_quic_port: "{{ qlean_node_config.results[1].stdout | trim }}" + qlean_metrics_port: "{{ qlean_node_config.results[2].stdout | trim }}" + deployment_mode: "{{ deployment_mode | default('docker') }}" when: qlean_node_config is defined - name: Extract validator index from validators.yaml shell: | - yq eval '."{{ node_name }}" | .[0]' "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validators.yaml" - register: qlean_validator_index + yq eval '."{{ node_name }}" | .[0]' "{{ genesis_dir }}/validators.yaml" + register: qlean_validator_index_raw changed_when: false failed_when: false - delegate_to: localhost - name: Set validator index set_fact: - qlean_validator_index: "{{ qlean_validator_index.stdout | trim | default('0') }}" - when: qlean_validator_index is defined + qlean_validator_index: "{{ qlean_validator_index_raw.stdout | trim | default('0') }}" + when: qlean_validator_index_raw is defined - name: Ensure node key file exists stat: diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index 80b66c9..a535a92 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -1,68 +1,25 @@ --- # Ream role: Deploy and manage Ream nodes # Converts client-cmds/ream-cmd.sh logic to Ansible tasks - -- name: Extract docker image from config file - shell: | - # Read image from user config file if provided, otherwise from validator-config.yaml - validator_config="{{ genesis_dir }}/validator-config.yaml" - user_config="{{ user_config_file | default('') }}" - - # Try user config first if provided (supports both 'nodes' and 'clients' format) - if [ -n "$user_config" ] && [ -f "$user_config" ]; then - # Try nodes format first (ream_0 style) - image=$(yq eval '.nodes[] | select(.name | test("^ream_")) | .image' "$user_config" 2>/dev/null | head -1) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - # Try clients format (backwards compatibility) - image=$(yq eval '.clients[] | select(.name == "ream") | .image' "$user_config" 2>/dev/null) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - fi - - # Fall back to validator-config.yaml (read from validators array) - yq eval '.validators[] | select(.name | test("^ream_")) | .image' "$validator_config" | head -1 - register: ream_docker_image_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Extract deployment mode from client-cmd.sh - shell: | - # Extract the value from node_setup line - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_setup=' "$project_root/client-cmds/ream-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' - register: ream_deployment_mode_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Set docker image and deployment mode from client-cmd.sh - set_fact: - ream_docker_image: "{{ ream_docker_image_raw.stdout | trim | default('ghcr.io/reamlabs/ream:latest') }}" - deployment_mode: "{{ ream_deployment_mode_raw.stdout | trim | default('docker') }}" +# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) - name: Extract node configuration from validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" register: ream_node_config changed_when: false - delegate_to: localhost loop: + - image - enrFields.quic - metricsPort - - privkey when: node_name is defined -- name: Set node ports +- name: Set node configuration set_fact: - ream_quic_port: "{{ ream_node_config.results[0].stdout }}" - ream_metrics_port: "{{ ream_node_config.results[1].stdout }}" - ream_privkey: "{{ ream_node_config.results[2].stdout }}" + ream_docker_image: "{{ ream_node_config.results[0].stdout | trim | default('ghcr.io/reamlabs/ream:latest') }}" + ream_quic_port: "{{ ream_node_config.results[1].stdout | trim }}" + ream_metrics_port: "{{ ream_node_config.results[2].stdout | trim }}" + deployment_mode: "{{ deployment_mode | default('docker') }}" when: ream_node_config is defined - name: Ensure node key file exists diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index 43ed2f2..4e14784 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -1,69 +1,24 @@ --- # Zeam role: Deploy and manage Zeam nodes # Converts client-cmds/zeam-cmd.sh logic to Ansible tasks - -- name: Extract docker image from config file - shell: | - # Read image from user config file if provided, otherwise from validator-config.yaml - validator_config="{{ genesis_dir }}/validator-config.yaml" - user_config="{{ user_config_file | default('') }}" - - # Try user config first if provided (supports both 'nodes' and 'clients' format) - if [ -n "$user_config" ] && [ -f "$user_config" ]; then - # Try nodes format first (zeam_0 style) - image=$(yq eval '.nodes[] | select(.name | test("^zeam_")) | .image' "$user_config" 2>/dev/null | head -1) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - # Try clients format (backwards compatibility) - image=$(yq eval '.clients[] | select(.name == "zeam") | .image' "$user_config" 2>/dev/null) - if [ -n "$image" ] && [ "$image" != "null" ]; then - echo "$image" - exit 0 - fi - fi - - # Fall back to validator-config.yaml (read from validators array) - yq eval '.validators[] | select(.name | test("^zeam_")) | .image' "$validator_config" | head -1 - register: zeam_docker_image_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Extract deployment mode from client-cmd.sh - shell: | - # Extract the value from node_setup line - # playbook_dir points to ansible/playbooks, go up two levels to reach project root - project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" - grep -E '^node_setup=' "$project_root/client-cmds/zeam-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' - register: zeam_deployment_mode_raw - changed_when: false - delegate_to: localhost - run_once: true - -- name: Set docker image and deployment mode from client-cmd.sh - set_fact: - zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('blockblaz/zeam:latest') }}" - deployment_mode: "{{ zeam_deployment_mode_raw.stdout | trim | default('docker') }}" - delegate_to: localhost - run_once: true +# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) - name: Extract node configuration from validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" - register: node_config + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" + register: zeam_node_config changed_when: false - delegate_to: localhost loop: + - image - metricsPort - - privkey when: node_name is defined -- name: Set node metrics port +- name: Set node configuration set_fact: - zeam_metrics_port: "{{ node_config.results[0].stdout }}" - when: node_config is defined + zeam_docker_image: "{{ zeam_node_config.results[0].stdout | trim | default('blockblaz/zeam:latest') }}" + zeam_metrics_port: "{{ zeam_node_config.results[1].stdout | trim }}" + deployment_mode: "{{ deployment_mode | default('docker') }}" + when: zeam_node_config is defined - name: Ensure node key file exists stat: diff --git a/client-cmds/ethlambda-cmd.sh b/client-cmds/ethlambda-cmd.sh index 50bd63c..0b7b9c5 100644 --- a/client-cmds/ethlambda-cmd.sh +++ b/client-cmds/ethlambda-cmd.sh @@ -2,6 +2,9 @@ #-----------------------ethlambda setup---------------------- +# Docker image (set from validator-config.yaml or user config via --configFile) +# ethlambdaImage is exported by spin-node.sh before sourcing this file + binary_path="$scriptDir/../ethlambda/target/release/ethlambda" # Command when running as binary @@ -14,7 +17,7 @@ node_binary="$binary_path \ --metrics-port $metricsPort" # Command when running as docker container -node_docker="ghcr.io/lambdaclass/ethlambda:devnet2 \ +node_docker="$ethlambdaImage \ --custom-network-config-dir /config \ --gossipsub-port $quicPort \ --node-id $item \ diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index 0541367..3a28272 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Docker image (set from validator-config.yaml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --configFile) # grandineImage is exported by spin-node.sh before sourcing this file node_binary="$grandine_bin \ diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index 0aa98bb..6e78723 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -1,7 +1,7 @@ #!/bin/bash #-----------------------lantern setup---------------------- -# Docker image (set from validator-config.yaml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --configFile) # lanternImage is exported by spin-node.sh before sourcing this file devnet_flag="" diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index 2c345ab..ef2c2f7 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -3,7 +3,7 @@ # Metrics enabled by default metrics_flag="--metrics" -# Docker image (set from validator-config.yaml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --configFile) # lighthouseImage is exported by spin-node.sh before sourcing this file node_binary="$lighthouse_bin lean_node \ diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index 6f12f88..ff1613f 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -4,7 +4,7 @@ # expects "qlean" submodule or symlink inside "lean-quickstart" root directory # https://github.com/qdrvm/qlean-mini -# Docker image (set from validator-config.yaml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --configFile) # qleanImage is exported by spin-node.sh before sourcing this file node_binary="$scriptDir/qlean/build/src/executable/qlean \ diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index a4f191c..c188703 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -4,7 +4,7 @@ # Metrics enabled by default metrics_flag="--metrics" -# Docker image (set from validator-config.yaml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --configFile) # reamImage is exported by spin-node.sh before sourcing this file # modify the path to the ream binary as per your system diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 00a3505..17755a6 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -6,7 +6,7 @@ # Metrics enabled by default metrics_flag="--metrics_enable" -# Docker image (set from validator-config.yaml or user config via --config-file) +# Docker image (set from validator-config.yaml or user config via --configFile) # zeamImage is exported by spin-node.sh before sourcing this file node_binary="$scriptDir/../zig-out/bin/zeam node \ diff --git a/generate-genesis.sh b/generate-genesis.sh index cd29b23..d05ad6c 100755 --- a/generate-genesis.sh +++ b/generate-genesis.sh @@ -521,103 +521,6 @@ rm -f "$GENESIS_VALIDATORS_TMP" echo "" -# ======================================== -# Generate annotated_validators.yaml with key metadata -# ======================================== -echo "🔧 Generating annotated_validators.yaml..." - -VALIDATORS_OUTPUT_FILE="$GENESIS_DIR/validators.yaml" -ANNOTATED_VALIDATORS_FILE="$GENESIS_DIR/annotated_validators.yaml" - -if [ ! -f "$VALIDATORS_OUTPUT_FILE" ]; then - echo " ❌ Error: validators.yaml not found at $VALIDATORS_OUTPUT_FILE" - exit 1 -fi - -ASSIGNMENT_HAS_WRAPPER=$(yq eval 'has("validators")' "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) -if [ "$ASSIGNMENT_HAS_WRAPPER" != "true" ]; then - ASSIGNMENT_HAS_WRAPPER="false" -fi - -ASSIGNMENT_NODE_NAMES=() -if [ "$ASSIGNMENT_HAS_WRAPPER" = "true" ]; then - while IFS= read -r node_name; do - if [ -n "$node_name" ]; then - ASSIGNMENT_NODE_NAMES+=("$node_name") - fi - done < <(yq eval '.validators | keys | .[]' "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) -else - while IFS= read -r node_name; do - if [ -n "$node_name" ]; then - ASSIGNMENT_NODE_NAMES+=("$node_name") - fi - done < <(yq eval 'keys | .[]' "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) -fi - -if [ ${#ASSIGNMENT_NODE_NAMES[@]} -eq 0 ]; then - echo " ❌ Error: No validator assignments found in validators.yaml" - exit 1 -fi - -NODE_ASSIGNMENTS_TMP=$(mktemp) - -for idx in "${!ASSIGNMENT_NODE_NAMES[@]}"; do - node=${ASSIGNMENT_NODE_NAMES[$idx]} - - if [ "$ASSIGNMENT_HAS_WRAPPER" = "true" ]; then - INDEX_QUERY=".validators.\"$node\"[]" - else - INDEX_QUERY=".\"$node\"[]" - fi - - echo "$node:" >> "$NODE_ASSIGNMENTS_TMP" - - ENTRY_FOUND=false - - while IFS= read -r raw_index; do - # Trim whitespace using bash parameter expansion (avoids xargs which can fail in sandboxed environments) - raw_index="${raw_index#"${raw_index%%[![:space:]]*}"}" - raw_index="${raw_index%"${raw_index##*[![:space:]]}"}" - if [ -z "$raw_index" ] || [ "$raw_index" == "null" ]; then - continue - fi - - ENTRY_FOUND=true - - PUBKEY_HEX_VALUE=$(yq eval ".validators[$raw_index].$PUBKEY_FIELD" "$MANIFEST_FILE" 2>/dev/null) - - if [ -z "$PUBKEY_HEX_VALUE" ] || [ "$PUBKEY_HEX_VALUE" == "null" ]; then - echo " ❌ Error: Missing pubkey for validator index $raw_index in manifest" - rm -f "$NODE_ASSIGNMENTS_TMP" - exit 1 - fi - - PUBKEY_HEX_NO_PREFIX="${PUBKEY_HEX_VALUE#0x}" - PRIVKEY_FILENAME="validator_${raw_index}_sk.ssz" - - cat << EOF >> "$NODE_ASSIGNMENTS_TMP" - - index: $raw_index - pubkey_hex: $PUBKEY_HEX_NO_PREFIX - privkey_file: $PRIVKEY_FILENAME -EOF - done < <(yq eval "$INDEX_QUERY" "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) - - if [ "$ENTRY_FOUND" = false ]; then - echo " []" >> "$NODE_ASSIGNMENTS_TMP" - fi - - if [ "$idx" -lt $(( ${#ASSIGNMENT_NODE_NAMES[@]} - 1 )) ]; then - echo "" >> "$NODE_ASSIGNMENTS_TMP" - fi -done - -cat "$NODE_ASSIGNMENTS_TMP" > "$ANNOTATED_VALIDATORS_FILE" -rm -f "$NODE_ASSIGNMENTS_TMP" - -echo " ✅ Generated annotated_validators.yaml with pubkey and privkey metadata" - -echo "" - # ======================================== # Step 4: Generate Private Key Files # ======================================== diff --git a/load-client-config.sh b/load-client-config.sh deleted file mode 100644 index 1800cc2..0000000 --- a/load-client-config.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/bin/bash -# Load client configuration from validator-config.yaml and optional user config file - -# Associative array to store client images (requires bash 4+, but we'll use parallel arrays for bash 3.2 compatibility) -CLIENT_NAMES=() -CLIENT_IMAGES_LIST=() -KNOWN_CLIENTS=("zeam" "ream" "qlean" "lantern" "lighthouse" "grandine") - -# Function to find index of client name -find_client_index() { - local search_name="$1" - local i=0 - for name in "${CLIENT_NAMES[@]}"; do - if [ "$name" == "$search_name" ]; then - echo "$i" - return 0 - fi - ((i++)) - done - echo "-1" -} - -# Function to extract client type from node name (e.g., zeam_0 -> zeam) -get_client_type() { - echo "$1" | sed 's/_[0-9]*$//' -} - -# Function to load default config from validator-config.yaml -load_default_config() { - local validator_config="$configDir/validator-config.yaml" - - if [ ! -f "$validator_config" ]; then - echo "⚠️ Warning: validator-config.yaml not found at $validator_config" - return 1 - fi - - # Load images from validators array using yq - for client in "${KNOWN_CLIENTS[@]}"; do - # Find the first validator matching this client type (e.g., zeam_0 for zeam) - local image=$(yq eval ".validators[] | select(.name | test(\"^${client}_\")) | .image" "$validator_config" 2>/dev/null | head -1) - if [ -n "$image" ] && [ "$image" != "null" ]; then - CLIENT_NAMES+=("$client") - CLIENT_IMAGES_LIST+=("$image") - fi - done - - echo "✓ Loaded default client images from $validator_config" -} - -# Function to load user config and override defaults -# User config format uses node names (e.g., zeam_0) instead of client names -load_user_config() { - local user_config="$1" - - if [ -z "$user_config" ]; then - return 0 - fi - - if [ ! -f "$user_config" ]; then - echo "⚠️ Warning: User config file not found at $user_config - using defaults" - return 1 - fi - - echo "Loading user config from $user_config..." - - # Load user-specified images (supports both 'nodes' and 'clients' format for backwards compatibility) - local override_count=0 - - # Try 'nodes' format first (new format with zeam_0, etc.) - local nodes_exist=$(yq eval '.nodes // empty' "$user_config" 2>/dev/null) - - if [ -n "$nodes_exist" ]; then - # New format: nodes with zeam_0 style names - while IFS= read -r line; do - local node_name=$(echo "$line" | awk '{print $1}') - local node_image=$(echo "$line" | awk '{print $2}') - - # Extract client type from node name (zeam_0 -> zeam) - local client_type=$(get_client_type "$node_name") - - # Validate client type - local valid_client=false - for known in "${KNOWN_CLIENTS[@]}"; do - if [ "$client_type" == "$known" ]; then - valid_client=true - break - fi - done - - if [ "$valid_client" == "false" ]; then - echo "⚠️ Warning: Unknown client type '$client_type' from node '$node_name' - skipping" - continue - fi - - # Validate image format (basic check) - if [[ ! "$node_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then - echo "⚠️ Warning: Invalid image format '$node_image' for node '$node_name' - using default" - continue - fi - - # Find and update the client image - local idx=$(find_client_index "$client_type") - if [ "$idx" != "-1" ]; then - CLIENT_IMAGES_LIST[$idx]="$node_image" - echo " ✓ Override $client_type (from $node_name): $node_image" - ((override_count++)) - fi - done < <(yq eval '.nodes[] | .name + " " + .image' "$user_config" 2>/dev/null) - else - # Fallback: try 'clients' format (old format for backwards compatibility) - while IFS= read -r line; do - local client_name=$(echo "$line" | awk '{print $1}') - local client_image=$(echo "$line" | awk '{print $2}') - - # Validate client name - local valid_client=false - for known in "${KNOWN_CLIENTS[@]}"; do - if [ "$client_name" == "$known" ]; then - valid_client=true - break - fi - done - - if [ "$valid_client" == "false" ]; then - echo "⚠️ Warning: Unknown client '$client_name' in config - skipping" - continue - fi - - # Validate image format (basic check) - if [[ ! "$client_image" =~ ^[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+$ ]]; then - echo "⚠️ Warning: Invalid image format '$client_image' for client '$client_name' - using default" - continue - fi - - # Find and update the client image - local idx=$(find_client_index "$client_name") - if [ "$idx" != "-1" ]; then - CLIENT_IMAGES_LIST[$idx]="$client_image" - echo " ✓ Override $client_name: $client_image" - ((override_count++)) - fi - done < <(yq eval '.clients[] | .name + " " + .image' "$user_config" 2>/dev/null) - fi - - if [ $override_count -eq 0 ]; then - echo "⚠️ No valid overrides found in user config" - else - echo "✓ Applied $override_count custom image(s) from user config" - fi -} - -# Function to get image for a specific client -get_client_image() { - local client_name="$1" - local idx=$(find_client_index "$client_name") - if [ "$idx" != "-1" ]; then - echo "${CLIENT_IMAGES_LIST[$idx]}" - fi -} - -# Function to display loaded configuration -display_client_config() { - echo "" - echo "==================================================" - echo "Client Configuration:" - echo "==================================================" - printf "%-12s | %s\n" "Client" "Docker Image" - echo "--------------------------------------------------" - local i=0 - for client in "${CLIENT_NAMES[@]}"; do - if [ -n "${CLIENT_IMAGES_LIST[$i]}" ]; then - printf "%-12s | %s\n" "$client" "${CLIENT_IMAGES_LIST[$i]}" - fi - ((i++)) - done - echo "==================================================" - echo "" -} - -# Load default configuration from validator-config.yaml -load_default_config - -# Load user configuration if provided -if [ -n "$configFile" ]; then - load_user_config "$configFile" -fi diff --git a/local-devnet/genesis/validator-config.yaml b/local-devnet/genesis/validator-config.yaml index 160fd0c..91e02ad 100644 --- a/local-devnet/genesis/validator-config.yaml +++ b/local-devnet/genesis/validator-config.yaml @@ -44,7 +44,7 @@ validators: - name: "lantern_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - image: piertwo/lantern:v0.0.1 + image: piertwo/lantern:v0.0.2 privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -77,6 +77,7 @@ validators: - name: "ethlambda_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAmPV5jU62WtmDkCEmfq1jzbBDkGbHNsDN78gJyvmv2TuC5 + image: ghcr.io/lambdaclass/ethlambda:latest privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" # verify /ip4/127.0.0.1/udp/9007/quic-v1/p2p/16Uiu2HAmPV5jU62WtmDkCEmfq1jzbBDkGbHNsDN78gJyvmv2TuC5 enrFields: diff --git a/merge-config.sh b/merge-config.sh new file mode 100755 index 0000000..902380a --- /dev/null +++ b/merge-config.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Merge user-config.yml overrides into validator-config.yaml +# Creates deploy-validator-config.yaml in the same directory as the base config +# +# Usage: ./merge-config.sh [user-config.yml] +# +# If user-config.yml is not provided or doesn't exist, the base config is copied as-is. + +set -e + +base_config="$1" # validator-config.yaml (required) +user_config="$2" # user-config.yml (optional) + +if [ -z "$base_config" ]; then + echo "Usage: $0 [user-config.yml]" + exit 1 +fi + +if [ ! -f "$base_config" ]; then + echo "Error: Base config not found: $base_config" + exit 1 +fi + +# Check if yq is installed +if ! command -v yq &> /dev/null; then + echo "Error: yq is required but not installed." + exit 1 +fi + +# Output file is in the same directory as base config +config_dir=$(dirname "$base_config") +output_config="$config_dir/deploy-validator-config.yaml" + +# Start with base config +cp "$base_config" "$output_config" + +# If no user config provided or doesn't exist, we're done +if [ -z "$user_config" ] || [ ! -f "$user_config" ]; then + echo "✓ Created deploy-validator-config.yaml (no user overrides)" + exit 0 +fi + +echo "Merging user config overrides from $user_config..." + +# Get all node names from user config +node_names=$(yq eval '.nodes[].name' "$user_config" 2>/dev/null) + +if [ -z "$node_names" ]; then + echo "✓ Created deploy-validator-config.yaml (no nodes in user config)" + exit 0 +fi + +override_count=0 + +for node in $node_names; do + # Check if this node exists in base config + node_exists=$(yq eval ".validators[] | select(.name == \"$node\") | .name" "$output_config" 2>/dev/null) + if [ -z "$node_exists" ] || [ "$node_exists" == "null" ]; then + echo " ⚠ Skipping $node: not found in validator-config.yaml" + continue + fi + + # Merge image if specified + image=$(yq eval ".nodes[] | select(.name == \"$node\") | .image // \"\"" "$user_config" 2>/dev/null) + if [ -n "$image" ] && [ "$image" != "null" ] && [ "$image" != "" ]; then + yq eval -i "(.validators[] | select(.name == \"$node\")).image = \"$image\"" "$output_config" + echo " ✓ $node: image = $image" + ((override_count++)) + fi + + # Add more mergeable fields here as needed +done + +echo "✓ Created deploy-validator-config.yaml ($override_count override(s) applied)" diff --git a/parse-env.sh b/parse-env.sh index da57287..008742f 100755 --- a/parse-env.sh +++ b/parse-env.sh @@ -76,7 +76,7 @@ while [[ $# -gt 0 ]]; do shift # past argument shift # past value ;; - --config-file) + --configFile) configFile="$2" shift # past argument shift # past value diff --git a/parse-vc.sh b/parse-vc.sh index 8b50e82..dd376dd 100644 --- a/parse-vc.sh +++ b/parse-vc.sh @@ -1,12 +1,14 @@ #!/bin/bash -# parse validator-config to load values related to the $item -# needed for ream and qlean (or any other client), zeam picks directly from validator-config +# parse deploy-validator-config.yaml to load values related to the $item +# needed for ream and qlean (or any other client), zeam picks directly from config # 1. load quic port and export it in $quicPort # 2. private key and dump it into a file $client.key and export it in $privKeyPath # 3. devnet and export it in $devnet +# 4. docker image (already merged in deploy-validator-config.yaml) # $item, $configDir (genesis dir) is available here +# Note: deploy-validator-config.yaml already has user overrides merged (from merge-config.sh) # Check if yq is installed if ! command -v yq &> /dev/null; then @@ -16,47 +18,48 @@ if ! command -v yq &> /dev/null; then exit 1 fi -# Validate that validator config file exists -validator_config_file="$configDir/validator-config.yaml" -if [ ! -f "$validator_config_file" ]; then - echo "Error: Validator config file not found at $validator_config_file" +# Use resolved config (has user overrides merged) +deploy_config_file="$configDir/deploy-validator-config.yaml" +if [ ! -f "$deploy_config_file" ]; then + echo "Error: Resolved validator config file not found at $deploy_config_file" + echo "This file should have been created by merge-config.sh" exit 1 fi # Automatically extract QUIC port using yq -quicPort=$(yq eval ".validators[] | select(.name == \"$item\") | .enrFields.quic" "$validator_config_file") +quicPort=$(yq eval ".validators[] | select(.name == \"$item\") | .enrFields.quic" "$deploy_config_file") # Validate that we found a QUIC port for this node if [ -z "$quicPort" ] || [ "$quicPort" == "null" ]; then - echo "Error: No QUIC port found for node '$item' in $validator_config_file" + echo "Error: No QUIC port found for node '$item' in $deploy_config_file" echo "Available nodes:" - yq eval '.validators[].name' "$validator_config_file" + yq eval '.validators[].name' "$deploy_config_file" exit 1 fi # Automatically extract metrics port using yq -metricsPort=$(yq eval ".validators[] | select(.name == \"$item\") | .metricsPort" "$validator_config_file") +metricsPort=$(yq eval ".validators[] | select(.name == \"$item\") | .metricsPort" "$deploy_config_file") # Validate that we found a metrics port for this node if [ -z "$metricsPort" ] || [ "$metricsPort" == "null" ]; then - echo "Error: No metrics port found for node '$item' in $validator_config_file" + echo "Error: No metrics port found for node '$item' in $deploy_config_file" echo "Available nodes:" - yq eval '.validators[].name' "$validator_config_file" + yq eval '.validators[].name' "$deploy_config_file" exit 1 fi # Automatically extract devnet using yq (optional - only ream uses it) -devnet=$(yq eval ".validators[] | select(.name == \"$item\") | .devnet" "$validator_config_file") +devnet=$(yq eval ".validators[] | select(.name == \"$item\") | .devnet" "$deploy_config_file") if [ -z "$devnet" ] || [ "$devnet" == "null" ]; then devnet="" fi # Automatically extract private key using yq -privKey=$(yq eval ".validators[] | select(.name == \"$item\") | .privkey" "$validator_config_file") +privKey=$(yq eval ".validators[] | select(.name == \"$item\") | .privkey" "$deploy_config_file") # Validate that we found a private key for this node if [ -z "$privKey" ] || [ "$privKey" == "null" ]; then - echo "Error: No private key found for node '$item' in $validator_config_file" + echo "Error: No private key found for node '$item' in $deploy_config_file" exit 1 fi @@ -65,44 +68,48 @@ privKeyPath="$item.key" echo "$privKey" > "$configDir/$privKeyPath" # Extract hash-sig key configuration from top-level config -keyType=$(yq eval ".config.keyType" "$validator_config_file") -hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name == \"$item\") | .key" "$validator_config_file") +keyType=$(yq eval ".config.keyType" "$deploy_config_file") +hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name == \"$item\") | .key" "$deploy_config_file") # Load hash-sig keys if configured if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$hashSigKeyIndex" ]; then # Set hash-sig key paths hashSigPkPath="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_pk.json" hashSigSkPath="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_sk.json" - + # Validate that hash-sig keys exist if [ ! -f "$hashSigPkPath" ]; then echo "Warning: Hash-sig public key not found at $hashSigPkPath" echo "Run genesis generator to create hash-sig keys: ./generate-genesis.sh $configDir" fi - + if [ ! -f "$hashSigSkPath" ]; then echo "Warning: Hash-sig secret key not found at $hashSigSkPath" echo "Run genesis generator to create hash-sig keys: ./generate-genesis.sh $configDir" fi - + # Export hash-sig key paths for client use export HASH_SIG_PK_PATH="$hashSigPkPath" export HASH_SIG_SK_PATH="$hashSigSkPath" export HASH_SIG_KEY_INDEX="$hashSigKeyIndex" - - echo "Node: $item" - echo "QUIC Port: $quicPort" - echo "Metrics Port: $metricsPort" - echo "Devnet: ${devnet:-}" - echo "Private Key File: $privKeyPath" +fi + +# Load docker image for this node (already merged in resolved config) +docker_image=$(yq eval ".validators[] | select(.name == \"$item\") | .image" "$deploy_config_file" 2>/dev/null) +if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then + echo "Warning: No docker image found for $item" + docker_image="" +fi + +echo "Node: $item" +echo "QUIC Port: $quicPort" +echo "Metrics Port: $metricsPort" +echo "Devnet: ${devnet:-}" +echo "Private Key File: $privKeyPath" +if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$hashSigKeyIndex" ]; then echo "Key Type: $keyType" echo "Hash-Sig Key Index: $hashSigKeyIndex" echo "Hash-Sig Public Key: $hashSigPkPath" echo "Hash-Sig Secret Key: $hashSigSkPath" -else - echo "Node: $item" - echo "QUIC Port: $quicPort" - echo "Metrics Port: $metricsPort" - echo "Devnet: ${devnet:-}" - echo "Private Key File: $privKeyPath" fi +echo "Docker Image: ${docker_image:-}" diff --git a/run-ansible.sh b/run-ansible.sh index 40f732b..b7d92f6 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -22,11 +22,10 @@ configDir="$1" node="$2" cleanData="$3" validatorConfig="$4" -validator_config_file="$5" +deploy_config_file="$5" # deploy-validator-config.yaml (with user overrides merged) sshKeyFile="$6" useRoot="$7" # Flag to use root user (defaults to current user) action="$8" # Action: "stop" to stop nodes, otherwise deploy -userConfigFile="$9" # User config file for docker images (--config-file) # Determine SSH user: use root if --useRoot flag is set, otherwise use current user if [ "$useRoot" == "true" ]; then @@ -36,9 +35,9 @@ else fi # Validate required arguments -if [ -z "$configDir" ] || [ -z "$validator_config_file" ]; then +if [ -z "$configDir" ] || [ -z "$deploy_config_file" ]; then echo "Error: Missing required arguments" - echo "Usage: $0 [sshKeyFile] [useRoot]" + echo "Usage: $0 [sshKeyFile] [useRoot] [action]" exit 1 fi @@ -50,10 +49,10 @@ echo "SSH user for remote connections: $sshUser" ANSIBLE_DIR="$scriptDir/ansible" INVENTORY_FILE="$ANSIBLE_DIR/inventory/hosts.yml" -# Generate inventory if it doesn't exist or if validator config is newer -if [ ! -f "$INVENTORY_FILE" ] || [ "$validator_config_file" -nt "$INVENTORY_FILE" ]; then - echo "Generating Ansible inventory from validator-config.yaml..." - "$scriptDir/generate-ansible-inventory.sh" "$validator_config_file" "$INVENTORY_FILE" +# Generate inventory if it doesn't exist or if resolved config is newer +if [ ! -f "$INVENTORY_FILE" ] || [ "$deploy_config_file" -nt "$INVENTORY_FILE" ]; then + echo "Generating Ansible inventory from deploy-validator-config.yaml..." + "$scriptDir/generate-ansible-inventory.sh" "$deploy_config_file" "$INVENTORY_FILE" fi # Update inventory with SSH key file and user if provided @@ -120,16 +119,6 @@ fi # Use default deployment mode (can be overridden by adding a 'deployment_mode' field per node in validator-config.yaml) EXTRA_VARS="$EXTRA_VARS deployment_mode=$DEFAULT_DEPLOYMENT_MODE" -# Pass user config file path for docker images if provided -if [ -n "$userConfigFile" ]; then - # Convert to absolute path if relative - if [[ "$userConfigFile" != /* ]]; then - userConfigFile="$(cd "$(dirname "$userConfigFile")" 2>/dev/null && pwd)/$(basename "$userConfigFile")" - fi - EXTRA_VARS="$EXTRA_VARS user_config_file=$userConfigFile" - echo "Using user config file for docker images: $userConfigFile" -fi - # Determine which playbook to run if [ "$action" == "stop" ]; then PLAYBOOK="$ANSIBLE_DIR/playbooks/stop-nodes.yml" @@ -174,9 +163,6 @@ if [ $EXIT_CODE -eq 0 ]; then IFS=' ' read -r -a deployed_nodes <<< "$node" fi - # Get config file paths - use validator-config.yaml from genesis dir - validator_config="$configDir/validator-config.yaml" - for node_name in "${deployed_nodes[@]}"; do # Extract client type from node name (e.g., zeam_0 -> zeam) client_type=$(echo "$node_name" | sed 's/_[0-9]*$//') @@ -197,21 +183,8 @@ if [ $EXIT_CODE -eq 0 ]; then fi printf "%-15s | %-10s | %s\n" "$node_name" "binary" "${binary_path:-unknown}" else - # Get image from user config or validator-config.yaml - docker_image="" - if [ -n "$userConfigFile" ] && [ -f "$userConfigFile" ]; then - # Try nodes format first (zeam_0 style) - docker_image=$(yq eval ".nodes[] | select(.name | test(\"^${client_type}_\")) | .image" "$userConfigFile" 2>/dev/null | head -1) - # Try clients format (backwards compatibility) - if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then - docker_image=$(yq eval ".clients[] | select(.name == \"$client_type\") | .image" "$userConfigFile" 2>/dev/null) - fi - fi - if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then - # Read from validators array in validator-config.yaml - docker_image=$(yq eval ".validators[] | select(.name | test(\"^${client_type}_\")) | .image" "$validator_config" 2>/dev/null | head -1) - fi - + # Get image from resolved config (overrides already merged) + docker_image=$(yq eval ".validators[] | select(.name == \"$node_name\") | .image" "$deploy_config_file" 2>/dev/null) printf "%-15s | %-10s | %s\n" "$node_name" "docker" "${docker_image:-unknown}" fi done diff --git a/spin-node.sh b/spin-node.sh index e21bb20..25bf0f6 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -10,9 +10,6 @@ fi # 0. parse env and args source "$(dirname $0)/parse-env.sh" -# Load client configuration (default + user config if provided) -source "$(dirname $0)/load-client-config.sh" - # Check if yq is installed (needed for deployment mode detection) if ! command -v yq &> /dev/null; then echo "Error: yq is required but not installed. Please install yq first." @@ -59,12 +56,18 @@ source "$(dirname $0)/set-up.sh" # ✅ Genesis generator implemented using PK's eth-beacon-genesis tool # Generates: validators.yaml, nodes.yaml, genesis.json, genesis.ssz, and .key files -# 2. collect the nodes that the user has asked us to spin and perform setup +# 2. Create deploy-validator-config.yaml (merges user config overrides if provided) +echo "" +echo "Creating resolved validator config..." +"$scriptDir/merge-config.sh" "$validator_config_file" "$configFile" +deploy_config_file="$configDir/deploy-validator-config.yaml" + +# 3. collect the nodes that the user has asked us to spin and perform setup -# Load nodes from validator config file -if [ -f "$validator_config_file" ]; then - # Use yq to extract node names from validator config - nodes=($(yq eval '.validators[].name' "$validator_config_file")) +# Load nodes from resolved validator config file +if [ -f "$deploy_config_file" ]; then + # Use yq to extract node names from resolved config + nodes=($(yq eval '.validators[].name' "$deploy_config_file")) # Validate that we found nodes if [ ${#nodes[@]} -eq 0 ]; then @@ -72,10 +75,8 @@ if [ -f "$validator_config_file" ]; then exit 1 fi else - echo "Error: Validator config file not found at $validator_config_file" - if [ "$deployment_mode" == "ansible" ]; then - echo "Please create ansible-devnet/genesis/validator-config.yaml for Ansible deployments" - fi + echo "Error: Resolved validator config file not found at $deploy_config_file" + echo "This file should have been created by merge-config.sh" nodes=() exit 1 fi @@ -148,7 +149,7 @@ if [ "$deployment_mode" == "ansible" ]; then # Handle stop action if [ -n "$stopNodes" ] && [ "$stopNodes" == "true" ]; then echo "Stopping nodes via Ansible..." - if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "stop" "$configFile"; then + if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$deploy_config_file" "$sshKeyFile" "$useRoot" "stop"; then echo "❌ Ansible stop operation failed. Exiting." exit 1 fi @@ -157,7 +158,7 @@ if [ "$deployment_mode" == "ansible" ]; then # Call separate Ansible execution script # If Ansible deployment fails, exit immediately (don't fall through to local deployment) - if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "" "$configFile"; then + if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$deploy_config_file" "$sshKeyFile" "$useRoot" ""; then echo "❌ Ansible deployment failed. Exiting." exit 1 fi @@ -169,12 +170,12 @@ fi # Handle stop action for local deployment if [ -n "$stopNodes" ] && [ "$stopNodes" == "true" ]; then echo "Stopping local nodes..." - - # Load nodes from validator config file - if [ -f "$validator_config_file" ]; then - nodes=($(yq eval '.validators[].name' "$validator_config_file")) + + # Load nodes from resolved validator config file + if [ -f "$deploy_config_file" ]; then + nodes=($(yq eval '.validators[].name' "$deploy_config_file")) else - echo "Error: Validator config file not found at $validator_config_file" + echo "Error: Resolved validator config file not found at $deploy_config_file" exit 1 fi @@ -252,38 +253,38 @@ for item in "${spin_nodes[@]}"; do eval "$cmd" fi - # parse validator-config.yaml for $item to load args values + # parse validator-config.yaml for $item to load args values (including docker_image) source parse-vc.sh # extract client config IFS='_' read -r -a elements <<< "$item" client="${elements[0]}" - # Get docker image from config (always set - from validator-config.yaml or user override) - client_image=$(get_client_image "$client") - # Export the image variable for this client (will be used by client-cmd.sh) + # docker_image is set by parse-vc.sh case "$client" in zeam) - export zeamImage="$client_image" + export zeamImage="$docker_image" ;; ream) - export reamImage="$client_image" + export reamImage="$docker_image" ;; qlean) - export qleanImage="$client_image" + export qleanImage="$docker_image" ;; lantern) - export lanternImage="$client_image" + export lanternImage="$docker_image" ;; lighthouse) - export lighthouseImage="$client_image" + export lighthouseImage="$docker_image" ;; grandine) - export grandineImage="$client_image" + export grandineImage="$docker_image" + ;; + ethlambda) + export ethlambdaImage="$docker_image" ;; esac - echo " ✓ Using image for $client: $client_image" # get client specific cmd and its mode (docker, binary) sourceCmd="source client-cmds/$client-cmd.sh" diff --git a/user-config.yml.example b/user-config.yml.example index 70c237c..ae452d8 100644 --- a/user-config.yml.example +++ b/user-config.yml.example @@ -1,9 +1,11 @@ # User Configuration Example -# Copy this file to user-config.yml or .yml and customize your docker images -# Usage: ./spin-node.sh --config-file .yml --node zeam_0,ream_0 +# Copy this file to user-config.yml or .yml and customize +# Usage: ./spin-node.sh --configFile .yml --node zeam_0,ream_0 # -# Only specify nodes you want to override - others will use defaults -# from validator-config.yaml in your network directory +# Only specify nodes and fields you want to override - others will use defaults +# from validator-config.yaml in your network directory. +# +# Overrides are merged into deploy-validator-config.yaml before deployment. nodes: - name: zeam_0 @@ -24,3 +26,6 @@ nodes: # # - name: grandine_0 # image: sifrai/lean:custom-tag + # + # - name: ethlambda_0 + # image: ghcr.io/lambdaclass/ethlambda:custom-tag From d069ba008d569894a65a6ef9e9c00c99fabef033 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Fri, 6 Feb 2026 16:26:43 +0100 Subject: [PATCH 04/15] fix: change node to validator in user-config.yml --- README.md | 12 +++++------ ansible-devnet/genesis/validator-config.yaml | 14 ++++++------- merge-config.sh | 6 +++--- user-config.yml.example | 22 ++++++++++---------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index 26dce97..bf5e6ad 100644 --- a/README.md +++ b/README.md @@ -88,25 +88,25 @@ NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --configFil **Example config file (user-config.yml):** ```yaml -nodes: - - name: zeam_0 +validators: + - name: "zeam_0" image: blockblaz/zeam:feature-branch - - name: ream_0 + - name: "ream_0" image: ghcr.io/reamlabs/ream:v2.0 ``` **Testing a specific client build:** ```sh # Create custom config file for zeam /my-zeam-config.yml -nodes: - - name: zeam_0 +validators: + - name: "zeam_0" image: blockblaz/zeam:custom-tag # Run with custom zeam image NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --configFile /my-zeam-config.yml ``` -Only specify nodes you want to override - others will use their defaults from `validator-config.yaml`. +Only specify validators you want to override - others will use their defaults from `validator-config.yaml`. ## Args diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index 2b13eb1..fb6255d 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -8,7 +8,7 @@ validators: - name: "zeam_0" # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f - image: blockblaz/zeam:devnet2 + image: blockblaz/zeam:latest privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: # verify /ip4/127.0.0.1/udp/9000/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f @@ -20,7 +20,7 @@ validators: - name: "ream_0" # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 - image: ghcr.io/reamlabs/ream:latest-devnet2 + image: ghcr.io/reamlabs/ream:latest privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -32,7 +32,7 @@ validators: - name: "qlean_0" # TODO: add a third entry in nodes.yaml corresponding to this - image: qdrvm/qlean-mini:devnet-2 + image: qdrvm/qlean-mini:latest privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -66,19 +66,19 @@ validators: count: 1 - name: "grandine_0" - image: sifrai/lean:devnet-2 + image: sifrai/lean:latest privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" enrFields: - ip: "46.62.163.74" + ip: "37.27.250.20" quic: 9001 metricsPort: 9095 count: 1 - name: "ethlambda_0" - image: ghcr.io/lambdaclass/ethlambda:devnet2 + image: ghcr.io/lambdaclass/ethlambda:latest privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" enrFields: - ip: "91.99.52.24" + ip: "78.47.44.215" quic: 9001 metricsPort: 9095 count: 1 \ No newline at end of file diff --git a/merge-config.sh b/merge-config.sh index 902380a..3daaaed 100755 --- a/merge-config.sh +++ b/merge-config.sh @@ -43,10 +43,10 @@ fi echo "Merging user config overrides from $user_config..." # Get all node names from user config -node_names=$(yq eval '.nodes[].name' "$user_config" 2>/dev/null) +node_names=$(yq eval '.validators[].name' "$user_config" 2>/dev/null) if [ -z "$node_names" ]; then - echo "✓ Created deploy-validator-config.yaml (no nodes in user config)" + echo "✓ Created deploy-validator-config.yaml (no validators in user config)" exit 0 fi @@ -61,7 +61,7 @@ for node in $node_names; do fi # Merge image if specified - image=$(yq eval ".nodes[] | select(.name == \"$node\") | .image // \"\"" "$user_config" 2>/dev/null) + image=$(yq eval ".validators[] | select(.name == \"$node\") | .image // \"\"" "$user_config" 2>/dev/null) if [ -n "$image" ] && [ "$image" != "null" ] && [ "$image" != "" ]; then yq eval -i "(.validators[] | select(.name == \"$node\")).image = \"$image\"" "$output_config" echo " ✓ $node: image = $image" diff --git a/user-config.yml.example b/user-config.yml.example index ae452d8..c01533a 100644 --- a/user-config.yml.example +++ b/user-config.yml.example @@ -1,31 +1,31 @@ # User Configuration Example # Copy this file to user-config.yml or .yml and customize -# Usage: ./spin-node.sh --configFile .yml --node zeam_0,ream_0 +# Usage: ./spin-node.sh --configFile .yml # -# Only specify nodes and fields you want to override - others will use defaults +# Only specify validators and fields you want to override - others will use defaults # from validator-config.yaml in your network directory. # # Overrides are merged into deploy-validator-config.yaml before deployment. -nodes: - - name: zeam_0 +validators: + - name: "zeam_0" image: blockblaz/zeam:custom-tag - - name: ream_0 + - name: "ream_0" image: ghcr.io/reamlabs/ream:my-branch - # Add more nodes as needed: - # - name: qlean_0 + # Add more validators as needed: + # - name: "qlean_0" # image: qdrvm/qlean-mini:custom-tag # - # - name: lantern_0 + # - name: "lantern_0" # image: piertwo/lantern:custom-tag # - # - name: lighthouse_0 + # - name: "lighthouse_0" # image: hopinheimer/lighthouse:custom-tag # - # - name: grandine_0 + # - name: "grandine_0" # image: sifrai/lean:custom-tag # - # - name: ethlambda_0 + # - name: "ethlambda_0" # image: ghcr.io/lambdaclass/ethlambda:custom-tag From 95b5242ee87979b7e7d8a9271640fa1865a889bf Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Fri, 6 Feb 2026 18:15:59 +0100 Subject: [PATCH 05/15] fix: grandine flags --- ansible/roles/grandine/tasks/main.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index 97f8a3c..cfc22f4 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -17,9 +17,11 @@ - name: Set node configuration set_fact: - grandine_quic_port: "{{ grandine_node_config.results[0].stdout }}" - grandine_metrics_port: "{{ grandine_node_config.results[1].stdout }}" - grandine_privkey: "{{ grandine_node_config.results[2].stdout }}" + grandine_docker_image: "{{ grandine_node_config.results[0].stdout | trim | default('sifrai/lean:unstable') }}" + grandine_quic_port: "{{ grandine_node_config.results[1].stdout | trim }}" + grandine_metrics_port: "{{ grandine_node_config.results[2].stdout | trim }}" + grandine_privkey: "{{ grandine_node_config.results[3].stdout | trim }}" + deployment_mode: "{{ deployment_mode | default('docker') }}" when: grandine_node_config is defined - name: Ensure node key file exists From 7584e767c5a5f9e3e848e72b86e89bef27713069 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Mon, 9 Feb 2026 14:53:55 +0100 Subject: [PATCH 06/15] fix: update comments --- parse-vc.sh | 6 +++--- run-ansible.sh | 4 ++-- spin-node.sh | 12 ++++++------ 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/parse-vc.sh b/parse-vc.sh index dd376dd..78e7749 100644 --- a/parse-vc.sh +++ b/parse-vc.sh @@ -18,10 +18,10 @@ if ! command -v yq &> /dev/null; then exit 1 fi -# Use resolved config (has user overrides merged) +# Use deploy-validator-config.yaml (has user overrides merged) deploy_config_file="$configDir/deploy-validator-config.yaml" if [ ! -f "$deploy_config_file" ]; then - echo "Error: Resolved validator config file not found at $deploy_config_file" + echo "Error: deploy-validator-config.yaml not found at $deploy_config_file" echo "This file should have been created by merge-config.sh" exit 1 fi @@ -94,7 +94,7 @@ if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$ha export HASH_SIG_KEY_INDEX="$hashSigKeyIndex" fi -# Load docker image for this node (already merged in resolved config) +# Load docker image for this node (already merged in deploy-validator-config.yaml) docker_image=$(yq eval ".validators[] | select(.name == \"$item\") | .image" "$deploy_config_file" 2>/dev/null) if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then echo "Warning: No docker image found for $item" diff --git a/run-ansible.sh b/run-ansible.sh index b7d92f6..cbcf2ee 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -49,7 +49,7 @@ echo "SSH user for remote connections: $sshUser" ANSIBLE_DIR="$scriptDir/ansible" INVENTORY_FILE="$ANSIBLE_DIR/inventory/hosts.yml" -# Generate inventory if it doesn't exist or if resolved config is newer +# Generate inventory if it doesn't exist or if deploy-validator-config.yaml is newer if [ ! -f "$INVENTORY_FILE" ] || [ "$deploy_config_file" -nt "$INVENTORY_FILE" ]; then echo "Generating Ansible inventory from deploy-validator-config.yaml..." "$scriptDir/generate-ansible-inventory.sh" "$deploy_config_file" "$INVENTORY_FILE" @@ -183,7 +183,7 @@ if [ $EXIT_CODE -eq 0 ]; then fi printf "%-15s | %-10s | %s\n" "$node_name" "binary" "${binary_path:-unknown}" else - # Get image from resolved config (overrides already merged) + # Get image from deploy-validator-config.yaml (overrides already merged) docker_image=$(yq eval ".validators[] | select(.name == \"$node_name\") | .image" "$deploy_config_file" 2>/dev/null) printf "%-15s | %-10s | %s\n" "$node_name" "docker" "${docker_image:-unknown}" fi diff --git a/spin-node.sh b/spin-node.sh index 25bf0f6..3a0ad47 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -58,15 +58,15 @@ source "$(dirname $0)/set-up.sh" # 2. Create deploy-validator-config.yaml (merges user config overrides if provided) echo "" -echo "Creating resolved validator config..." +echo "Creating deploy-validator-config.yaml..." "$scriptDir/merge-config.sh" "$validator_config_file" "$configFile" deploy_config_file="$configDir/deploy-validator-config.yaml" # 3. collect the nodes that the user has asked us to spin and perform setup -# Load nodes from resolved validator config file +# Load nodes from deploy-validator-config.yaml if [ -f "$deploy_config_file" ]; then - # Use yq to extract node names from resolved config + # Use yq to extract node names from deploy-validator-config.yaml nodes=($(yq eval '.validators[].name' "$deploy_config_file")) # Validate that we found nodes @@ -75,7 +75,7 @@ if [ -f "$deploy_config_file" ]; then exit 1 fi else - echo "Error: Resolved validator config file not found at $deploy_config_file" + echo "Error: deploy-validator-config.yaml not found at $deploy_config_file" echo "This file should have been created by merge-config.sh" nodes=() exit 1 @@ -171,11 +171,11 @@ fi if [ -n "$stopNodes" ] && [ "$stopNodes" == "true" ]; then echo "Stopping local nodes..." - # Load nodes from resolved validator config file + # Load nodes from deploy-validator-config.yaml if [ -f "$deploy_config_file" ]; then nodes=($(yq eval '.validators[].name' "$deploy_config_file")) else - echo "Error: Resolved validator config file not found at $deploy_config_file" + echo "Error: deploy-validator-config.yaml not found at $deploy_config_file" exit 1 fi From bb1e9a2dfebba64159d578a29800625152d432c5 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Mon, 9 Feb 2026 16:22:07 +0100 Subject: [PATCH 07/15] fix: revert deleting annotated_validators.yaml for the separate pr --- .gitignore | 1 + ansible/playbooks/copy-genesis.yml | 2 + ansible/playbooks/generate-genesis.yml | 1 + generate-genesis.sh | 97 ++++++++++++++++++++++++++ 4 files changed, 101 insertions(+) diff --git a/.gitignore b/.gitignore index f61eae5..1ccc164 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ validators.yaml nodes.yaml genesis.json genesis.ssz +annotated_validators.yaml # Deploy config (created by merge-config.sh, merges validator-config + user overrides) deploy-validator-config.yaml diff --git a/ansible/playbooks/copy-genesis.yml b/ansible/playbooks/copy-genesis.yml index c8718ac..76a25f8 100644 --- a/ansible/playbooks/copy-genesis.yml +++ b/ansible/playbooks/copy-genesis.yml @@ -47,6 +47,7 @@ loop: - config.yaml - validators.yaml + - annotated_validators.yaml - nodes.yaml - genesis.json - genesis.ssz @@ -136,6 +137,7 @@ loop: - config.yaml - validators.yaml + - annotated_validators.yaml - nodes.yaml - genesis.json - genesis.ssz diff --git a/ansible/playbooks/generate-genesis.yml b/ansible/playbooks/generate-genesis.yml index fde89b9..786bf25 100644 --- a/ansible/playbooks/generate-genesis.yml +++ b/ansible/playbooks/generate-genesis.yml @@ -78,6 +78,7 @@ Generated files in {{ genesis_dir }}: - config.yaml (with GENESIS_VALIDATORS) - validators.yaml + - annotated_validators.yaml - nodes.yaml - genesis.json - genesis.ssz diff --git a/generate-genesis.sh b/generate-genesis.sh index d05ad6c..cd29b23 100755 --- a/generate-genesis.sh +++ b/generate-genesis.sh @@ -521,6 +521,103 @@ rm -f "$GENESIS_VALIDATORS_TMP" echo "" +# ======================================== +# Generate annotated_validators.yaml with key metadata +# ======================================== +echo "🔧 Generating annotated_validators.yaml..." + +VALIDATORS_OUTPUT_FILE="$GENESIS_DIR/validators.yaml" +ANNOTATED_VALIDATORS_FILE="$GENESIS_DIR/annotated_validators.yaml" + +if [ ! -f "$VALIDATORS_OUTPUT_FILE" ]; then + echo " ❌ Error: validators.yaml not found at $VALIDATORS_OUTPUT_FILE" + exit 1 +fi + +ASSIGNMENT_HAS_WRAPPER=$(yq eval 'has("validators")' "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) +if [ "$ASSIGNMENT_HAS_WRAPPER" != "true" ]; then + ASSIGNMENT_HAS_WRAPPER="false" +fi + +ASSIGNMENT_NODE_NAMES=() +if [ "$ASSIGNMENT_HAS_WRAPPER" = "true" ]; then + while IFS= read -r node_name; do + if [ -n "$node_name" ]; then + ASSIGNMENT_NODE_NAMES+=("$node_name") + fi + done < <(yq eval '.validators | keys | .[]' "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) +else + while IFS= read -r node_name; do + if [ -n "$node_name" ]; then + ASSIGNMENT_NODE_NAMES+=("$node_name") + fi + done < <(yq eval 'keys | .[]' "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) +fi + +if [ ${#ASSIGNMENT_NODE_NAMES[@]} -eq 0 ]; then + echo " ❌ Error: No validator assignments found in validators.yaml" + exit 1 +fi + +NODE_ASSIGNMENTS_TMP=$(mktemp) + +for idx in "${!ASSIGNMENT_NODE_NAMES[@]}"; do + node=${ASSIGNMENT_NODE_NAMES[$idx]} + + if [ "$ASSIGNMENT_HAS_WRAPPER" = "true" ]; then + INDEX_QUERY=".validators.\"$node\"[]" + else + INDEX_QUERY=".\"$node\"[]" + fi + + echo "$node:" >> "$NODE_ASSIGNMENTS_TMP" + + ENTRY_FOUND=false + + while IFS= read -r raw_index; do + # Trim whitespace using bash parameter expansion (avoids xargs which can fail in sandboxed environments) + raw_index="${raw_index#"${raw_index%%[![:space:]]*}"}" + raw_index="${raw_index%"${raw_index##*[![:space:]]}"}" + if [ -z "$raw_index" ] || [ "$raw_index" == "null" ]; then + continue + fi + + ENTRY_FOUND=true + + PUBKEY_HEX_VALUE=$(yq eval ".validators[$raw_index].$PUBKEY_FIELD" "$MANIFEST_FILE" 2>/dev/null) + + if [ -z "$PUBKEY_HEX_VALUE" ] || [ "$PUBKEY_HEX_VALUE" == "null" ]; then + echo " ❌ Error: Missing pubkey for validator index $raw_index in manifest" + rm -f "$NODE_ASSIGNMENTS_TMP" + exit 1 + fi + + PUBKEY_HEX_NO_PREFIX="${PUBKEY_HEX_VALUE#0x}" + PRIVKEY_FILENAME="validator_${raw_index}_sk.ssz" + + cat << EOF >> "$NODE_ASSIGNMENTS_TMP" + - index: $raw_index + pubkey_hex: $PUBKEY_HEX_NO_PREFIX + privkey_file: $PRIVKEY_FILENAME +EOF + done < <(yq eval "$INDEX_QUERY" "$VALIDATORS_OUTPUT_FILE" 2>/dev/null) + + if [ "$ENTRY_FOUND" = false ]; then + echo " []" >> "$NODE_ASSIGNMENTS_TMP" + fi + + if [ "$idx" -lt $(( ${#ASSIGNMENT_NODE_NAMES[@]} - 1 )) ]; then + echo "" >> "$NODE_ASSIGNMENTS_TMP" + fi +done + +cat "$NODE_ASSIGNMENTS_TMP" > "$ANNOTATED_VALIDATORS_FILE" +rm -f "$NODE_ASSIGNMENTS_TMP" + +echo " ✅ Generated annotated_validators.yaml with pubkey and privkey metadata" + +echo "" + # ======================================== # Step 4: Generate Private Key Files # ======================================== From 7e94c954110e63a9f4976ef76f2188aee13615d4 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Mon, 9 Feb 2026 16:23:07 +0100 Subject: [PATCH 08/15] fix: restore README for annotated validators --- README.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/README.md b/README.md index bf5e6ad..3730051 100644 --- a/README.md +++ b/README.md @@ -328,6 +328,31 @@ qlean_0: - 2 ``` +**Recommended:** `annotated_validators.yaml` is also generated and should be preferred by client software as it includes public keys and private key file references directly, eliminating the need for clients to derive key filenames from validator indices: + +```yaml +zeam_0: + - index: 0 + pubkey_hex: 4b3c31094bcc9b45446b2028eae5ad192b2df16778837b10230af102255c9c5f72d7ba43eae30b2c6a779f47367ebf5a42f6c959 + privkey_file: validator_0_sk.json + - index: 3 + pubkey_hex: 8df32a54d2fbdf3a88035b2fe3931320cb900d364d6e7c56b19c0f3c6006ce5b3ebe802a65fe1b420183f62e830a953cb33b7804 + privkey_file: validator_3_sk.json + +ream_0: + - index: 1 + pubkey_hex: 5b15f72f90bd655b039f9839c36951454b89c605f8c334581cfa832bdd0c994a1350094f7e22617d77607b067b0aa2439e0ead7d + privkey_file: validator_1_sk.json + - index: 4 + pubkey_hex: 71bf8f73980591574de34a0db471da74f5cfd84d4731d53f47bf3023b26c2638ac5bd24993ea71492fedbd6c4afe5c299213b76b + privkey_file: validator_4_sk.json + +qlean_0: + - index: 2 + pubkey_hex: b87e69568a347d1aa811cc158634fb1f4e247c5509ad2b1652a8d758ec0ab0796954e307b97dd6284fbb30088c2e595546fdf663 + privkey_file: validator_2_sk.json +``` + `nodes.yaml` provide enrs of all the nodes so that clients don't have to run a discovery protocol: ```yaml From c6b5cbf82eea255148bc282f8faa41b0d2f57f22 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Mon, 9 Feb 2026 16:24:44 +0100 Subject: [PATCH 09/15] fix: fix deployed nodes summary, add scripts folder --- merge-config.sh => scripts/merge-config.sh | 0 spin-node.sh | 22 ++++------------------ 2 files changed, 4 insertions(+), 18 deletions(-) rename merge-config.sh => scripts/merge-config.sh (100%) diff --git a/merge-config.sh b/scripts/merge-config.sh similarity index 100% rename from merge-config.sh rename to scripts/merge-config.sh diff --git a/spin-node.sh b/spin-node.sh index 3a0ad47..c4a47de 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -59,7 +59,7 @@ source "$(dirname $0)/set-up.sh" # 2. Create deploy-validator-config.yaml (merges user config overrides if provided) echo "" echo "Creating deploy-validator-config.yaml..." -"$scriptDir/merge-config.sh" "$validator_config_file" "$configFile" +"$scriptDir/scripts/merge-config.sh" "$validator_config_file" "$configFile" deploy_config_file="$configDir/deploy-validator-config.yaml" # 3. collect the nodes that the user has asked us to spin and perform setup @@ -233,8 +233,6 @@ elif [[ "$OSTYPE" == "linux"* ]]; then done fi spinned_pids=() -declare -A node_images -declare -A node_modes for item in "${spin_nodes[@]}"; do echo -e "\n\nspining $item: client=$client (mode=$node_setup)" @@ -291,13 +289,6 @@ for item in "${spin_nodes[@]}"; do echo "$sourceCmd" eval $sourceCmd - # Store the final image for display - if [ "$node_setup" == "docker" ]; then - node_images["$item"]=$(echo "$node_docker" | grep -oE '[^ ]+:[^ ]+' | head -1) - fi - - # Store node mode - node_modes["$item"]="$node_setup" # spin nodes if [ "$node_setup" == "binary" ] @@ -356,7 +347,7 @@ done; container_names="${spin_nodes[*]}" process_ids="${spinned_pids[*]}" -# Display summary table +# Display summary table (read directly from deploy-validator-config.yaml like ansible does) echo "" echo "==================================================" echo "Deployed Nodes Summary:" @@ -364,13 +355,8 @@ echo "==================================================" printf "%-15s | %-10s | %s\n" "Node" "Mode" "Docker Image" echo "--------------------------------------------------" for node in "${spin_nodes[@]}"; do - mode="${node_modes[$node]}" - if [ "$mode" == "docker" ]; then - image="${node_images[$node]}" - printf "%-15s | %-10s | %s\n" "$node" "$mode" "$image" - else - printf "%-15s | %-10s | %s\n" "$node" "$mode" "N/A (binary mode)" - fi + image=$(yq eval ".validators[] | select(.name == \"$node\") | .image" "$deploy_config_file" 2>/dev/null) + printf "%-15s | %-10s | %s\n" "$node" "docker" "$image" done echo "==================================================" echo "" From 6f606b1f4b0d7aca073661cfb23b6ee7ad903794 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Mon, 9 Feb 2026 22:01:39 +0100 Subject: [PATCH 10/15] fix: delete binary check from run-ansible --- ansible/playbooks/generate-genesis.yml | 2 +- parse-vc.sh | 2 +- run-ansible.sh | 35 +++++++------------------- 3 files changed, 11 insertions(+), 28 deletions(-) diff --git a/ansible/playbooks/generate-genesis.yml b/ansible/playbooks/generate-genesis.yml index 786bf25..7006d9c 100644 --- a/ansible/playbooks/generate-genesis.yml +++ b/ansible/playbooks/generate-genesis.yml @@ -78,10 +78,10 @@ Generated files in {{ genesis_dir }}: - config.yaml (with GENESIS_VALIDATORS) - validators.yaml - - annotated_validators.yaml - nodes.yaml - genesis.json - genesis.ssz + - annotated_validators.yaml {% for node in node_names_list %} - {{ node }}.key {% endfor %} diff --git a/parse-vc.sh b/parse-vc.sh index 78e7749..7e42350 100644 --- a/parse-vc.sh +++ b/parse-vc.sh @@ -102,6 +102,7 @@ if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then fi echo "Node: $item" +echo "Docker Image: ${docker_image:-}" echo "QUIC Port: $quicPort" echo "Metrics Port: $metricsPort" echo "Devnet: ${devnet:-}" @@ -112,4 +113,3 @@ if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$ha echo "Hash-Sig Public Key: $hashSigPkPath" echo "Hash-Sig Secret Key: $hashSigSkPath" fi -echo "Docker Image: ${docker_image:-}" diff --git a/run-ansible.sh b/run-ansible.sh index cbcf2ee..aa7e4c8 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -153,40 +153,23 @@ if [ $EXIT_CODE -eq 0 ]; then echo "==================================================" echo "Deployed Nodes Summary:" echo "==================================================" - printf "%-15s | %-10s | %s\n" "Node" "Mode" "Image / Binary Path" + printf "%-15s | %-10s | %s\n" "Node" "Mode" "Docker Image" echo "--------------------------------------------------" - # Parse node names (comma or space separated) - if [[ "$node" == *","* ]]; then + # Get node names for summary + if [[ "$node" == "all" ]]; then + # Expand "all" to actual node names from config + deployed_nodes=($(yq eval '.validators[].name' "$deploy_config_file" 2>/dev/null)) + elif [[ "$node" == *","* ]]; then IFS=',' read -r -a deployed_nodes <<< "$node" else IFS=' ' read -r -a deployed_nodes <<< "$node" fi for node_name in "${deployed_nodes[@]}"; do - # Extract client type from node name (e.g., zeam_0 -> zeam) - client_type=$(echo "$node_name" | sed 's/_[0-9]*$//') - - if [ "$DEFAULT_DEPLOYMENT_MODE" == "binary" ]; then - # Extract binary path from client-cmd.sh - client_cmd_file="$scriptDir/client-cmds/${client_type}-cmd.sh" - binary_path="" - if [ -f "$client_cmd_file" ]; then - # Extract the first word from node_binary line (the actual binary path) - binary_path=$(grep -E '^node_binary=' "$client_cmd_file" | head -1 | sed -E 's/node_binary="([^ "\\]+).*/\1/') - # Resolve $scriptDir to actual path - binary_path=$(echo "$binary_path" | sed "s|\\\$scriptDir|$scriptDir|g") - # For unresolved variables (like $grandine_bin), show as custom binary path - if [[ "$binary_path" == \$* ]]; then - binary_path="" - fi - fi - printf "%-15s | %-10s | %s\n" "$node_name" "binary" "${binary_path:-unknown}" - else - # Get image from deploy-validator-config.yaml (overrides already merged) - docker_image=$(yq eval ".validators[] | select(.name == \"$node_name\") | .image" "$deploy_config_file" 2>/dev/null) - printf "%-15s | %-10s | %s\n" "$node_name" "docker" "${docker_image:-unknown}" - fi + # Get image from deploy-validator-config.yaml (overrides already merged) + docker_image=$(yq eval ".validators[] | select(.name == \"$node_name\") | .image" "$deploy_config_file" 2>/dev/null) + printf "%-15s | %-10s | %s\n" "$node_name" "docker" "${docker_image:-unknown}" done echo "==================================================" echo "" From 24af2e52e20e5a021ca002b26a3a68f04932df41 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Mon, 9 Feb 2026 22:30:57 +0100 Subject: [PATCH 11/15] feat: replace validator-config.yaml with deploy-validator-config.yaml --- .gitignore | 2 +- README.md | 33 ++++++++++++------- ansible/README.md | 6 ++-- ansible/inventory/group_vars/all.yml | 2 +- ansible/playbooks/clean-node-data.yml | 16 ++++----- ansible/playbooks/copy-genesis.yml | 10 +++--- ansible/playbooks/deploy-nodes.yml | 24 +++++++------- .../playbooks/helpers/deploy-single-node.yml | 10 +++--- ansible/playbooks/stop-nodes.yml | 16 ++++----- ansible/roles/ethlambda/tasks/main.yml | 6 ++-- ansible/roles/grandine/tasks/main.yml | 6 ++-- ansible/roles/lantern/tasks/main.yml | 8 ++--- ansible/roles/lighthouse/tasks/main.yml | 8 ++--- ansible/roles/qlean/tasks/main.yml | 6 ++-- ansible/roles/ream/tasks/main.yml | 6 ++-- ansible/roles/zeam/tasks/main.yml | 6 ++-- client-cmds/ethlambda-cmd.sh | 2 +- client-cmds/grandine-cmd.sh | 2 +- client-cmds/lantern-cmd.sh | 6 ++-- client-cmds/lighthouse-cmd.sh | 6 ++-- client-cmds/qlean-cmd.sh | 2 +- client-cmds/ream-cmd.sh | 2 +- client-cmds/zeam-cmd.sh | 2 +- generate-ansible-inventory.sh | 22 ++++++------- parse-vc.sh | 30 ++++++++--------- run-ansible.sh | 14 ++++---- spin-node.sh | 22 ++++++------- 27 files changed, 143 insertions(+), 132 deletions(-) diff --git a/.gitignore b/.gitignore index 1ccc164..a393734 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,7 @@ deploy-validator-config.yaml # Hash-sig validator keys (contains sensitive secret keys) hash-sig-keys/ -# Auto-generated Ansible inventory (generated from validator-config.yaml) +# Auto-generated Ansible inventory (generated from deploy-validator-config.yaml) hosts.yml # Temporary cache files diff --git a/README.md b/README.md index 3730051..817a9e4 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ You can override default Docker images using the `--configFile` flag. This is us **Basic usage (without custom images):** ```sh -# Uses default images from validator-config.yaml in your network directory +# Uses default images from validator-config.yaml (merged into deploy-validator-config.yaml) NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis ``` @@ -106,7 +106,7 @@ validators: NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --configFile /my-zeam-config.yml ``` -Only specify validators you want to override - others will use their defaults from `validator-config.yaml`. +Only specify validators you want to override - others will use their defaults from `validator-config.yaml`. Overrides are merged into `deploy-validator-config.yaml` which is used during deployment. ## Args @@ -174,6 +174,7 @@ Current following clients are supported: 4. Lantern 5. Lighthouse 6. Grandine +7. EthLambda However adding a lean client to this setup is very easy. Feel free to do the PR or reach out to the maintainers. @@ -201,20 +202,30 @@ lean-quickstart/ ├── user-config.yml # Your custom image overrides (gitignored) ├── local-devnet/ # Local development │ ├── genesis/ -│ │ └── validator-config.yaml # Local IPs (127.0.0.1) +│ │ ├── validator-config.yaml # Source config - Local IPs (127.0.0.1) +│ │ └── deploy-validator-config.yaml # Generated - merged config for deployment (gitignored) │ └── data/ # Node data directories │ └── ansible-devnet/ # Ansible/remote deployment ├── genesis/ - │ └── validator-config.yaml # Remote IPs (your server IPs) + │ ├── validator-config.yaml # Source config - Remote IPs (your server IPs) + │ └── deploy-validator-config.yaml # Generated - merged config for deployment (gitignored) └── data/ # Node data directories ``` **Automatic Directory Selection:** + - When `deployment_mode: ansible` is set (in config or via `--deploymentMode ansible`), the script automatically uses `ansible-devnet/genesis/validator-config.yaml` - This keeps local and remote configurations completely separate - Genesis files are generated in the appropriate directory based on deployment mode +**Config File Flow:** + +- `validator-config.yaml` - Default source config file (node IPs, ports, validator counts) +- `user-config.yml` - Optional overrides (currently custom Docker images) +- `deploy-validator-config.yaml` - Generated by merging the above two files, recreated on each deployment +- All deployment operations (local and ansible) read from `deploy-validator-config.yaml` + ### Configuration The `validator-config.yaml` file defines the shuffle algorithm, active epoch configuration, and validator nodes specifications: @@ -367,13 +378,13 @@ Post genesis generation, the quickstarts loads and calls the appropriate node's **Client Integration:** Your client implementation should read these environment variables and use the hash-sig keys for validator operations. - - `$item` - the node name for which this cmd is being executed, index into `validator-config.yaml` for its configuration + - `$item` - the node name for which this cmd is being executed, index into `deploy-validator-config.yaml` for its configuration - `$configDir` - the abs folder housing `genesis` configuration (same as `NETWORK_DIR` env variable provided while executing shell command), already mapped to `/config` in the docker mode - A generic data folder is created inside config folder accessible as `$dataDir` with `$dataDir/$item` to be used as the data dir for a particular node to be used for binary format, already mapped to `/data` in the docker mode - - Variables read and available from `validator-config.yaml` (use them or directly read configuration from the `validator-config.yaml` using `$item` as the index into `validators` section) + - Variables read and available from `deploy-validator-config.yaml` (use them or directly read configuration from the `deploy-validator-config.yaml` using `$item` as the index into `validators` section) - `$metricsPort` - - `$quicPort` - - `$item.key` filename of the p2p `privkey` read and dumped into file from `validator-config.yaml` inside config dir (so `$configDir/$item.key` or `/config/$item.key`) + - `$quicPort` + - `$item.key` filename of the p2p `privkey` read and dumped into file from `deploy-validator-config.yaml` inside config dir (so `$configDir/$item.key` or `/config/$item.key`) Here is an example client cmd: ```bash @@ -529,7 +540,7 @@ This quickstart includes automated configuration parsing: - **Official Genesis Generation**: Uses PK's `eth-beacon-genesis` docker tool from [PR #36](https://github.com/ethpandaops/eth-beacon-genesis/pull/36) - **Complete File Set**: Generates `validators.yaml`, `nodes.yaml`, `genesis.json`, `genesis.ssz`, and `.key` files -- **QUIC Port Detection**: Automatically extracts QUIC ports from `validator-config.yaml` using `yq` +- **QUIC Port Detection**: Automatically extracts QUIC ports from `deploy-validator-config.yaml` using `yq` - **Node Detection**: Dynamically discovers available nodes from the validator configuration - **Private Key Management**: Automatically extracts and creates `.key` files for each node - **Error Handling**: Provides clear error messages when nodes or ports are not found @@ -697,7 +708,7 @@ ansible/ ### Remote Deployment -The Ansible inventory is **automatically generated** from `validator-config.yaml`. +The Ansible inventory is **automatically generated** from `deploy-validator-config.yaml`. **Configuration Setup:** @@ -806,7 +817,7 @@ Both deployment modes use the same `spin-node.sh` entry point, controlled by `de | **Multi-Host** | No | Yes | | **Rollback** | Manual | Built-in capabilities | | **Entry Point** | `spin-node.sh` | `spin-node.sh` (same command) | -| **Inventory** | N/A | Auto-generated from validator-config.yaml | +| **Inventory** | N/A | Auto-generated from deploy-validator-config.yaml | **Recommendation:** - Use `deployment_mode: local` for local development and quick testing diff --git a/ansible/README.md b/ansible/README.md index 0d41329..8628e5f 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -409,11 +409,11 @@ Or use sudo (not recommended): ./ansible-deploy.sh --node zeam_0 --network-dir local-devnet --docker-with-sudo ``` -#### 5. "Node not found in validator-config.yaml" +#### 5. "Node not found in deploy-validator-config.yaml" -Ensure node name matches exactly in `validator-config.yaml`: +Ensure node name matches exactly in `deploy-validator-config.yaml`: ```sh -yq eval '.validators[].name' local-devnet/genesis/validator-config.yaml +yq eval '.validators[].name' local-devnet/genesis/deploy-validator-config.yaml ``` #### 6. Container starts but immediately exits diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index 418ef92..df2f8f3 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -5,7 +5,7 @@ network_dir: "{{ playbook_dir }}/../local-devnet" genesis_dir: "{{ network_dir }}/genesis" data_dir: "{{ network_dir }}/data" -validator_config_file: "{{ genesis_dir }}/validator-config.yaml" +deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" # Remote paths (for remote deployments) # Use standard paths on remote hosts diff --git a/ansible/playbooks/clean-node-data.yml b/ansible/playbooks/clean-node-data.yml index a583782..733ff3d 100644 --- a/ansible/playbooks/clean-node-data.yml +++ b/ansible/playbooks/clean-node-data.yml @@ -6,7 +6,7 @@ connection: local gather_facts: no vars: - validator_config_file: "{{ genesis_dir }}/validator-config.yaml" + deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" tags: - zeam - ream @@ -18,15 +18,15 @@ - deploy tasks: - - name: Validate validator-config.yaml exists + - name: Validate deploy-validator-config.yaml exists stat: - path: "{{ validator_config_file }}" - register: validator_config_stat + path: "{{ deploy_validator_config_file }}" + register: deploy_config_stat - - name: Fail if validator-config.yaml missing + - name: Fail if deploy-validator-config.yaml missing fail: - msg: "validator-config.yaml not found at {{ validator_config_file }}" - when: not validator_config_stat.stat.exists + msg: "deploy-validator-config.yaml not found at {{ deploy_validator_config_file }}" + when: not deploy_config_stat.stat.exists - name: Verify yq is available command: yq --version @@ -42,7 +42,7 @@ - name: Extract all node names shell: | - yq eval '.validators[].name' {{ validator_config_file }} + yq eval '.validators[].name' {{ deploy_validator_config_file }} register: all_node_names_raw changed_when: false diff --git a/ansible/playbooks/copy-genesis.yml b/ansible/playbooks/copy-genesis.yml index 76a25f8..9b7f0a1 100644 --- a/ansible/playbooks/copy-genesis.yml +++ b/ansible/playbooks/copy-genesis.yml @@ -12,7 +12,7 @@ hosts: all:!local gather_facts: yes vars: - validator_config_file: "{{ genesis_dir }}/validator-config.yaml" + deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" # Use remote_genesis_dir from group_vars, or default to standard remote path actual_remote_genesis_dir: "{{ remote_genesis_dir | default('/opt/lean-quickstart/genesis') }}" @@ -51,7 +51,7 @@ - nodes.yaml - genesis.json - genesis.ssz - - validator-config.yaml + - deploy-validator-config.yaml - name: Fail if required genesis files are missing fail: @@ -68,9 +68,9 @@ delegate_to: localhost run_once: true - - name: Extract all node names from validator-config.yaml + - name: Extract all node names from deploy-validator-config.yaml shell: | - yq eval '.validators[].name' "{{ genesis_dir }}/validator-config.yaml" + yq eval '.validators[].name' "{{ genesis_dir }}/deploy-validator-config.yaml" register: all_node_names_raw delegate_to: localhost run_once: true @@ -141,7 +141,7 @@ - nodes.yaml - genesis.json - genesis.ssz - - validator-config.yaml + - deploy-validator-config.yaml - name: Copy node private key files to remote host copy: diff --git a/ansible/playbooks/deploy-nodes.yml b/ansible/playbooks/deploy-nodes.yml index 891b66f..1d543ba 100644 --- a/ansible/playbooks/deploy-nodes.yml +++ b/ansible/playbooks/deploy-nodes.yml @@ -4,7 +4,7 @@ # This playbook runs on localhost to parse node names, then deploys to remote hosts # # Before deployment, it syncs essential config files from local to remote: -# - validator-config.yaml +# - deploy-validator-config.yaml (validator config merged with user overrides) # - node key files (*.key) # - config.yaml, validators.yaml, nodes.yaml # - genesis.ssz, genesis.json @@ -15,22 +15,22 @@ connection: local gather_facts: yes vars: - validator_config_file: "{{ genesis_dir }}/validator-config.yaml" + deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" tasks: - - name: Validate validator-config.yaml exists + - name: Validate deploy-validator-config.yaml exists stat: - path: "{{ validator_config_file }}" - register: validator_config_stat + path: "{{ deploy_validator_config_file }}" + register: deploy_config_stat - - name: Fail if validator-config.yaml missing + - name: Fail if deploy-validator-config.yaml missing fail: - msg: "validator-config.yaml not found at {{ validator_config_file }}" - when: not validator_config_stat.stat.exists + msg: "deploy-validator-config.yaml not found at {{ deploy_validator_config_file }}" + when: not deploy_config_stat.stat.exists - name: Extract all node names shell: | - yq eval '.validators[].name' {{ validator_config_file }} + yq eval '.validators[].name' {{ deploy_validator_config_file }} register: all_node_names_raw changed_when: false @@ -101,7 +101,7 @@ # Use remote paths on remote hosts genesis_dir: "{{ remote_genesis_dir | default('/opt/lean-quickstart/genesis') }}" data_dir: "{{ remote_data_dir | default('/opt/lean-quickstart/data') }}" - validator_config_file: "{{ genesis_dir }}/validator-config.yaml" + deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" # node_name is set to inventory_hostname (which matches the node name) node_name: "{{ inventory_hostname }}" # Local genesis directory for syncing config files (from localhost play) @@ -117,10 +117,10 @@ - deploy - sync - - name: Sync deploy-validator-config.yaml to remote host (as validator-config.yaml) + - name: Sync deploy-validator-config.yaml to remote host copy: src: "{{ local_genesis_dir }}/deploy-validator-config.yaml" - dest: "{{ genesis_dir }}/validator-config.yaml" + dest: "{{ genesis_dir }}/deploy-validator-config.yaml" mode: '0644' force: yes tags: diff --git a/ansible/playbooks/helpers/deploy-single-node.yml b/ansible/playbooks/helpers/deploy-single-node.yml index 34f17f7..48a3c5e 100644 --- a/ansible/playbooks/helpers/deploy-single-node.yml +++ b/ansible/playbooks/helpers/deploy-single-node.yml @@ -6,14 +6,14 @@ set_fact: client_type: "{{ node_name.split('_')[0] }}" -- name: Set validator config file paths +- name: Set deploy config file paths set_fact: - actual_validator_config_file: "{{ genesis_dir }}/validator-config.yaml" - local_validator_config_file: "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" + actual_deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" + local_deploy_validator_config_file: "{{ hostvars['localhost']['local_genesis_dir_path'] }}/deploy-validator-config.yaml" - name: Extract node configuration (from local config) shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .name" {{ local_validator_config_file }} + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .name" {{ local_deploy_validator_config_file }} register: node_check changed_when: false failed_when: false @@ -21,7 +21,7 @@ - name: Fail if node not found in config fail: - msg: "Node '{{ node_name }}' not found in {{ local_validator_config_file }}" + msg: "Node '{{ node_name }}' not found in {{ local_deploy_validator_config_file }}" when: node_check.stdout == "" or node_check.rc != 0 - name: Set zeam validator config diff --git a/ansible/playbooks/stop-nodes.yml b/ansible/playbooks/stop-nodes.yml index af706a3..3a6fc46 100644 --- a/ansible/playbooks/stop-nodes.yml +++ b/ansible/playbooks/stop-nodes.yml @@ -7,22 +7,22 @@ connection: local gather_facts: yes vars: - validator_config_file: "{{ genesis_dir }}/validator-config.yaml" + deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" tasks: - - name: Validate validator-config.yaml exists + - name: Validate deploy-validator-config.yaml exists stat: - path: "{{ validator_config_file }}" - register: validator_config_stat + path: "{{ deploy_validator_config_file }}" + register: deploy_config_stat - - name: Fail if validator-config.yaml missing + - name: Fail if deploy-validator-config.yaml missing fail: - msg: "validator-config.yaml not found at {{ validator_config_file }}" - when: not validator_config_stat.stat.exists + msg: "deploy-validator-config.yaml not found at {{ deploy_validator_config_file }}" + when: not deploy_config_stat.stat.exists - name: Extract all node names shell: | - yq eval '.validators[].name' {{ validator_config_file }} + yq eval '.validators[].name' {{ deploy_validator_config_file }} register: all_node_names_raw changed_when: false diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index 2509552..7a0fe48 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -1,11 +1,11 @@ --- # Ethlambda role: Deploy and manage Ethlambda nodes # Converts client-cmds/ethlambda-cmd.sh logic to Ansible tasks -# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) +# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from validator-config.yaml +- name: Extract node configuration from deploy-validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" register: ethlambda_node_config changed_when: false loop: diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index cfc22f4..99438a9 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -1,11 +1,11 @@ --- # Grandine role: Deploy and manage Grandine nodes # Converts client-cmds/grandine-cmd.sh logic to Ansible tasks -# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) +# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from validator-config.yaml +- name: Extract node configuration from deploy-validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" register: grandine_node_config changed_when: false loop: diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index 54e3829..50cd134 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -1,11 +1,11 @@ --- # Lantern role: Deploy and manage Lantern nodes # Converts client-cmds/lantern-cmd.sh logic to Ansible tasks -# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) +# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from validator-config.yaml +- name: Extract node configuration from deploy-validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" register: lantern_node_config changed_when: false loop: @@ -64,7 +64,7 @@ --genesis-config /config/config.yaml --validator-registry-path /config/validators.yaml --genesis-state /config/genesis.ssz - --validator-config /config/validator-config.yaml + --validator-config /config/deploy-validator-config.yaml --nodes-path /config/nodes.yaml --node-id {{ node_name }} --node-key-path /config/{{ node_name }}.key diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index db66fd0..e220f61 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -1,11 +1,11 @@ --- # Lighthouse role: Deploy and manage Lighthouse nodes # Converts client-cmds/lighthouse-cmd.sh logic to Ansible tasks -# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) +# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from validator-config.yaml +- name: Extract node configuration from deploy-validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" register: lighthouse_node_config changed_when: false loop: @@ -67,7 +67,7 @@ lighthouse lean_node --datadir /data --config /config/config.yaml - --validators /config/validator-config.yaml + --validators /config/deploy-validator-config.yaml --nodes /config/nodes.yaml --node-id {{ node_name }} --private-key /config/{{ node_name }}.key diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index 2d7afa3..d9b0af2 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -1,11 +1,11 @@ --- # Qlean role: Deploy and manage Qlean nodes # Converts client-cmds/qlean-cmd.sh logic to Ansible tasks -# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) +# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from validator-config.yaml +- name: Extract node configuration from deploy-validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" register: qlean_node_config changed_when: false loop: diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index a535a92..cb970f4 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -1,11 +1,11 @@ --- # Ream role: Deploy and manage Ream nodes # Converts client-cmds/ream-cmd.sh logic to Ansible tasks -# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) +# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from validator-config.yaml +- name: Extract node configuration from deploy-validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" register: ream_node_config changed_when: false loop: diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index 4e14784..019ebfb 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -1,11 +1,11 @@ --- # Zeam role: Deploy and manage Zeam nodes # Converts client-cmds/zeam-cmd.sh logic to Ansible tasks -# Note: validator-config.yaml on remote already has user overrides merged (from deploy-validator-config.yaml) +# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from validator-config.yaml +- name: Extract node configuration from deploy-validator-config.yaml shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/validator-config.yaml" + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" register: zeam_node_config changed_when: false loop: diff --git a/client-cmds/ethlambda-cmd.sh b/client-cmds/ethlambda-cmd.sh index 0b7b9c5..7ef0e23 100644 --- a/client-cmds/ethlambda-cmd.sh +++ b/client-cmds/ethlambda-cmd.sh @@ -2,7 +2,7 @@ #-----------------------ethlambda setup---------------------- -# Docker image (set from validator-config.yaml or user config via --configFile) +# Docker image (set from deploy-validator-config.yaml, merged from validator-config.yaml + user config) # ethlambdaImage is exported by spin-node.sh before sourcing this file binary_path="$scriptDir/../ethlambda/target/release/ethlambda" diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index 746af1e..d62bdc8 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Docker image (set from validator-config.yaml or user config via --configFile) +# Docker image (set from deploy-validator-config.yaml, merged from validator-config.yaml + user config) # grandineImage is exported by spin-node.sh before sourcing this file node_binary="$grandine_bin \ diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index 6e78723..84f37e4 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -1,7 +1,7 @@ #!/bin/bash #-----------------------lantern setup---------------------- -# Docker image (set from validator-config.yaml or user config via --configFile) +# Docker image (set from deploy-validator-config.yaml or user config via --configFile) # lanternImage is exported by spin-node.sh before sourcing this file devnet_flag="" @@ -15,7 +15,7 @@ node_binary="$scriptDir/lantern/build/lantern_cli \ --genesis-config $configDir/config.yaml \ --validator-registry-path $configDir/validators.yaml \ --genesis-state $configDir/genesis.ssz \ - --validator-config $configDir/validator-config.yaml \ + --validator-config $configDir/deploy-validator-config.yaml \ $devnet_flag \ --nodes-path $configDir/nodes.yaml \ --node-id $item --node-key-path $configDir/$privKeyPath \ @@ -29,7 +29,7 @@ node_docker="$lanternImage --data-dir /data \ --genesis-config /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --genesis-state /config/genesis.ssz \ - --validator-config /config/validator-config.yaml \ + --validator-config /config/deploy-validator-config.yaml \ $devnet_flag \ --nodes-path /config/nodes.yaml \ --node-id $item --node-key-path /config/$privKeyPath \ diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index ef2c2f7..2122438 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -3,13 +3,13 @@ # Metrics enabled by default metrics_flag="--metrics" -# Docker image (set from validator-config.yaml or user config via --configFile) +# Docker image (set from deploy-validator-config.yaml or user config via --configFile) # lighthouseImage is exported by spin-node.sh before sourcing this file node_binary="$lighthouse_bin lean_node \ --datadir \"$dataDir/$item\" \ --config \"$configDir/config.yaml\" \ - --validators \"$configDir/validator-config.yaml\" \ + --validators \"$configDir/deploy-validator-config.yaml\" \ --nodes \"$configDir/nodes.yaml\" \ --node-id \"$item\" \ --private-key \"$configDir/$privKeyPath\" \ @@ -22,7 +22,7 @@ node_binary="$lighthouse_bin lean_node \ node_docker="$lighthouseImage lighthouse lean_node \ --datadir /data \ --config /config/config.yaml \ - --validators /config/validator-config.yaml \ + --validators /config/deploy-validator-config.yaml \ --nodes /config/nodes.yaml \ --node-id $item \ --private-key /config/$privKeyPath \ diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index ff1613f..1bc9108 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -4,7 +4,7 @@ # expects "qlean" submodule or symlink inside "lean-quickstart" root directory # https://github.com/qdrvm/qlean-mini -# Docker image (set from validator-config.yaml or user config via --configFile) +# Docker image (set from deploy-validator-config.yaml, merged from validator-config.yaml + user config) # qleanImage is exported by spin-node.sh before sourcing this file node_binary="$scriptDir/qlean/build/src/executable/qlean \ diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index c188703..d6606a0 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -4,7 +4,7 @@ # Metrics enabled by default metrics_flag="--metrics" -# Docker image (set from validator-config.yaml or user config via --configFile) +# Docker image (set from deploy-validator-config.yaml, merged from validator-config.yaml + user config) # reamImage is exported by spin-node.sh before sourcing this file # modify the path to the ream binary as per your system diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 17755a6..c37b722 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -6,7 +6,7 @@ # Metrics enabled by default metrics_flag="--metrics_enable" -# Docker image (set from validator-config.yaml or user config via --configFile) +# Docker image (set from deploy-validator-config.yaml, merged from validator-config.yaml + user config) # zeamImage is exported by spin-node.sh before sourcing this file node_binary="$scriptDir/../zig-out/bin/zeam node \ diff --git a/generate-ansible-inventory.sh b/generate-ansible-inventory.sh index e36044e..b60357c 100755 --- a/generate-ansible-inventory.sh +++ b/generate-ansible-inventory.sh @@ -1,15 +1,15 @@ #!/bin/bash -# Generate Ansible inventory from validator-config.yaml -# This script reads validator-config.yaml and generates hosts.yml for Ansible +# Generate Ansible inventory from deploy-validator-config.yaml +# This script reads deploy-validator-config.yaml and generates hosts.yml for Ansible set -e if [ $# -lt 2 ]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi -VALIDATOR_CONFIG="$1" +DEPLOY_CONFIG="$1" OUTPUT_FILE="$2" # Check if yq is installed @@ -21,8 +21,8 @@ if ! command -v yq &> /dev/null; then fi # Check if validator config exists -if [ ! -f "$VALIDATOR_CONFIG" ]; then - echo "Error: Validator config file not found: $VALIDATOR_CONFIG" +if [ ! -f "$DEPLOY_CONFIG" ]; then + echo "Error: Deploy config file not found: $DEPLOY_CONFIG" exit 1 fi @@ -34,7 +34,7 @@ mkdir -p "$OUTPUT_DIR" cat > "$OUTPUT_FILE" << 'EOF' --- # Ansible Inventory for Lean Quickstart -# Auto-generated from validator-config.yaml +# Auto-generated from deploy-validator-config.yaml # DO NOT EDIT MANUALLY - This file is auto-generated all: @@ -62,8 +62,8 @@ all: hosts: {} EOF -# Extract node information from validator-config.yaml -nodes=($(yq eval '.validators[].name' "$VALIDATOR_CONFIG")) +# Extract node information from deploy-validator-config.yaml +nodes=($(yq eval '.validators[].name' "$DEPLOY_CONFIG")) # Process each node and generate inventory entries for node_name in "${nodes[@]}"; do @@ -73,8 +73,8 @@ for node_name in "${nodes[@]}"; do group_name="${client_type}_nodes" # Extract node-specific information - node_ip=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.ip // \"127.0.0.1\"" "$VALIDATOR_CONFIG") - node_quic=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.quic // \"9000\"" "$VALIDATOR_CONFIG") + node_ip=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.ip // \"127.0.0.1\"" "$DEPLOY_CONFIG") + node_quic=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.quic // \"9000\"" "$DEPLOY_CONFIG") # Check if this is a remote deployment (IP is not localhost/127.0.0.1) is_remote=false diff --git a/parse-vc.sh b/parse-vc.sh index 7e42350..fb69744 100644 --- a/parse-vc.sh +++ b/parse-vc.sh @@ -19,47 +19,47 @@ if ! command -v yq &> /dev/null; then fi # Use deploy-validator-config.yaml (has user overrides merged) -deploy_config_file="$configDir/deploy-validator-config.yaml" -if [ ! -f "$deploy_config_file" ]; then - echo "Error: deploy-validator-config.yaml not found at $deploy_config_file" +deploy_validator_config_file="$configDir/deploy-validator-config.yaml" +if [ ! -f "$deploy_validator_config_file" ]; then + echo "Error: deploy-validator-config.yaml not found at $deploy_validator_config_file" echo "This file should have been created by merge-config.sh" exit 1 fi # Automatically extract QUIC port using yq -quicPort=$(yq eval ".validators[] | select(.name == \"$item\") | .enrFields.quic" "$deploy_config_file") +quicPort=$(yq eval ".validators[] | select(.name == \"$item\") | .enrFields.quic" "$deploy_validator_config_file") # Validate that we found a QUIC port for this node if [ -z "$quicPort" ] || [ "$quicPort" == "null" ]; then - echo "Error: No QUIC port found for node '$item' in $deploy_config_file" + echo "Error: No QUIC port found for node '$item' in $deploy_validator_config_file" echo "Available nodes:" - yq eval '.validators[].name' "$deploy_config_file" + yq eval '.validators[].name' "$deploy_validator_config_file" exit 1 fi # Automatically extract metrics port using yq -metricsPort=$(yq eval ".validators[] | select(.name == \"$item\") | .metricsPort" "$deploy_config_file") +metricsPort=$(yq eval ".validators[] | select(.name == \"$item\") | .metricsPort" "$deploy_validator_config_file") # Validate that we found a metrics port for this node if [ -z "$metricsPort" ] || [ "$metricsPort" == "null" ]; then - echo "Error: No metrics port found for node '$item' in $deploy_config_file" + echo "Error: No metrics port found for node '$item' in $deploy_validator_config_file" echo "Available nodes:" - yq eval '.validators[].name' "$deploy_config_file" + yq eval '.validators[].name' "$deploy_validator_config_file" exit 1 fi # Automatically extract devnet using yq (optional - only ream uses it) -devnet=$(yq eval ".validators[] | select(.name == \"$item\") | .devnet" "$deploy_config_file") +devnet=$(yq eval ".validators[] | select(.name == \"$item\") | .devnet" "$deploy_validator_config_file") if [ -z "$devnet" ] || [ "$devnet" == "null" ]; then devnet="" fi # Automatically extract private key using yq -privKey=$(yq eval ".validators[] | select(.name == \"$item\") | .privkey" "$deploy_config_file") +privKey=$(yq eval ".validators[] | select(.name == \"$item\") | .privkey" "$deploy_validator_config_file") # Validate that we found a private key for this node if [ -z "$privKey" ] || [ "$privKey" == "null" ]; then - echo "Error: No private key found for node '$item' in $deploy_config_file" + echo "Error: No private key found for node '$item' in $deploy_validator_config_file" exit 1 fi @@ -68,8 +68,8 @@ privKeyPath="$item.key" echo "$privKey" > "$configDir/$privKeyPath" # Extract hash-sig key configuration from top-level config -keyType=$(yq eval ".config.keyType" "$deploy_config_file") -hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name == \"$item\") | .key" "$deploy_config_file") +keyType=$(yq eval ".config.keyType" "$deploy_validator_config_file") +hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name == \"$item\") | .key" "$deploy_validator_config_file") # Load hash-sig keys if configured if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$hashSigKeyIndex" ]; then @@ -95,7 +95,7 @@ if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$ha fi # Load docker image for this node (already merged in deploy-validator-config.yaml) -docker_image=$(yq eval ".validators[] | select(.name == \"$item\") | .image" "$deploy_config_file" 2>/dev/null) +docker_image=$(yq eval ".validators[] | select(.name == \"$item\") | .image" "$deploy_validator_config_file" 2>/dev/null) if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then echo "Warning: No docker image found for $item" docker_image="" diff --git a/run-ansible.sh b/run-ansible.sh index aa7e4c8..8a3d8b2 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -22,7 +22,7 @@ configDir="$1" node="$2" cleanData="$3" validatorConfig="$4" -deploy_config_file="$5" # deploy-validator-config.yaml (with user overrides merged) +deploy_validator_config_file="$5" # deploy-validator-config.yaml (with user overrides merged) sshKeyFile="$6" useRoot="$7" # Flag to use root user (defaults to current user) action="$8" # Action: "stop" to stop nodes, otherwise deploy @@ -35,9 +35,9 @@ else fi # Validate required arguments -if [ -z "$configDir" ] || [ -z "$deploy_config_file" ]; then +if [ -z "$configDir" ] || [ -z "$deploy_validator_config_file" ]; then echo "Error: Missing required arguments" - echo "Usage: $0 [sshKeyFile] [useRoot] [action]" + echo "Usage: $0 [sshKeyFile] [useRoot] [action]" exit 1 fi @@ -50,9 +50,9 @@ ANSIBLE_DIR="$scriptDir/ansible" INVENTORY_FILE="$ANSIBLE_DIR/inventory/hosts.yml" # Generate inventory if it doesn't exist or if deploy-validator-config.yaml is newer -if [ ! -f "$INVENTORY_FILE" ] || [ "$deploy_config_file" -nt "$INVENTORY_FILE" ]; then +if [ ! -f "$INVENTORY_FILE" ] || [ "$deploy_validator_config_file" -nt "$INVENTORY_FILE" ]; then echo "Generating Ansible inventory from deploy-validator-config.yaml..." - "$scriptDir/generate-ansible-inventory.sh" "$deploy_config_file" "$INVENTORY_FILE" + "$scriptDir/generate-ansible-inventory.sh" "$deploy_validator_config_file" "$INVENTORY_FILE" fi # Update inventory with SSH key file and user if provided @@ -159,7 +159,7 @@ if [ $EXIT_CODE -eq 0 ]; then # Get node names for summary if [[ "$node" == "all" ]]; then # Expand "all" to actual node names from config - deployed_nodes=($(yq eval '.validators[].name' "$deploy_config_file" 2>/dev/null)) + deployed_nodes=($(yq eval '.validators[].name' "$deploy_validator_config_file" 2>/dev/null)) elif [[ "$node" == *","* ]]; then IFS=',' read -r -a deployed_nodes <<< "$node" else @@ -168,7 +168,7 @@ if [ $EXIT_CODE -eq 0 ]; then for node_name in "${deployed_nodes[@]}"; do # Get image from deploy-validator-config.yaml (overrides already merged) - docker_image=$(yq eval ".validators[] | select(.name == \"$node_name\") | .image" "$deploy_config_file" 2>/dev/null) + docker_image=$(yq eval ".validators[] | select(.name == \"$node_name\") | .image" "$deploy_validator_config_file" 2>/dev/null) printf "%-15s | %-10s | %s\n" "$node_name" "docker" "${docker_image:-unknown}" done echo "==================================================" diff --git a/spin-node.sh b/spin-node.sh index c4a47de..a6e70d5 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -60,14 +60,14 @@ source "$(dirname $0)/set-up.sh" echo "" echo "Creating deploy-validator-config.yaml..." "$scriptDir/scripts/merge-config.sh" "$validator_config_file" "$configFile" -deploy_config_file="$configDir/deploy-validator-config.yaml" +deploy_validator_config_file="$configDir/deploy-validator-config.yaml" # 3. collect the nodes that the user has asked us to spin and perform setup # Load nodes from deploy-validator-config.yaml -if [ -f "$deploy_config_file" ]; then +if [ -f "$deploy_validator_config_file" ]; then # Use yq to extract node names from deploy-validator-config.yaml - nodes=($(yq eval '.validators[].name' "$deploy_config_file")) + nodes=($(yq eval '.validators[].name' "$deploy_validator_config_file")) # Validate that we found nodes if [ ${#nodes[@]} -eq 0 ]; then @@ -75,7 +75,7 @@ if [ -f "$deploy_config_file" ]; then exit 1 fi else - echo "Error: deploy-validator-config.yaml not found at $deploy_config_file" + echo "Error: deploy-validator-config.yaml not found at $deploy_validator_config_file" echo "This file should have been created by merge-config.sh" nodes=() exit 1 @@ -149,7 +149,7 @@ if [ "$deployment_mode" == "ansible" ]; then # Handle stop action if [ -n "$stopNodes" ] && [ "$stopNodes" == "true" ]; then echo "Stopping nodes via Ansible..." - if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$deploy_config_file" "$sshKeyFile" "$useRoot" "stop"; then + if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$deploy_validator_config_file" "$sshKeyFile" "$useRoot" "stop"; then echo "❌ Ansible stop operation failed. Exiting." exit 1 fi @@ -158,7 +158,7 @@ if [ "$deployment_mode" == "ansible" ]; then # Call separate Ansible execution script # If Ansible deployment fails, exit immediately (don't fall through to local deployment) - if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$deploy_config_file" "$sshKeyFile" "$useRoot" ""; then + if ! "$scriptDir/run-ansible.sh" "$configDir" "$node" "$cleanData" "$validatorConfig" "$deploy_validator_config_file" "$sshKeyFile" "$useRoot" ""; then echo "❌ Ansible deployment failed. Exiting." exit 1 fi @@ -172,10 +172,10 @@ if [ -n "$stopNodes" ] && [ "$stopNodes" == "true" ]; then echo "Stopping local nodes..." # Load nodes from deploy-validator-config.yaml - if [ -f "$deploy_config_file" ]; then - nodes=($(yq eval '.validators[].name' "$deploy_config_file")) + if [ -f "$deploy_validator_config_file" ]; then + nodes=($(yq eval '.validators[].name' "$deploy_validator_config_file")) else - echo "Error: deploy-validator-config.yaml not found at $deploy_config_file" + echo "Error: deploy-validator-config.yaml not found at $deploy_validator_config_file" exit 1 fi @@ -251,7 +251,7 @@ for item in "${spin_nodes[@]}"; do eval "$cmd" fi - # parse validator-config.yaml for $item to load args values (including docker_image) + # parse deploy-validator-config.yaml for $item to load args values (including docker_image) source parse-vc.sh # extract client config @@ -355,7 +355,7 @@ echo "==================================================" printf "%-15s | %-10s | %s\n" "Node" "Mode" "Docker Image" echo "--------------------------------------------------" for node in "${spin_nodes[@]}"; do - image=$(yq eval ".validators[] | select(.name == \"$node\") | .image" "$deploy_config_file" 2>/dev/null) + image=$(yq eval ".validators[] | select(.name == \"$node\") | .image" "$deploy_validator_config_file" 2>/dev/null) printf "%-15s | %-10s | %s\n" "$node" "docker" "$image" done echo "==================================================" From 90ca5fa2c77d807ef26ddc4d59ad3a3d03d6bbf8 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Mon, 9 Feb 2026 23:51:08 +0100 Subject: [PATCH 12/15] fix: use deploy_validator_config_file for prometheus --- README.md | 2 +- spin-node.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e88e6b5..cc56a1e 100644 --- a/README.md +++ b/README.md @@ -170,7 +170,7 @@ Only specify validators you want to override - others will use their defaults fr - The script will automatically pull the specified Docker images before running containers - Example: `--tag devnet0` or `--tag devnet1` 11. `--metrics` starts a Prometheus + Grafana metrics stack alongside the devnet (local deployments only). When specified: - - Generates `metrics/prometheus/prometheus.yml` from `validator-config.yaml` with scrape targets for all configured nodes + - Generates `metrics/prometheus/prometheus.yml` from `deploy-validator-config.yaml` with scrape targets for all configured nodes - Starts Prometheus (http://localhost:9090) and Grafana (http://localhost:3000) via Docker Compose - Grafana is pre-provisioned with Lean Ethereum dashboards (no login required) - On `--stop --metrics`, the metrics stack is also torn down diff --git a/spin-node.sh b/spin-node.sh index 81296b2..fcd8a73 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -363,8 +363,8 @@ if [ -n "$enableMetrics" ] && [ "$enableMetrics" == "true" ]; then metricsDir="$scriptDir/metrics" - # Generate prometheus.yml from validator-config.yaml - "$scriptDir/generate-prometheus-config.sh" "$validator_config_file" "$metricsDir/prometheus" + # Generate prometheus.yml from deploy-validator-config.yaml + "$scriptDir/generate-prometheus-config.sh" "$deploy_validator_config_file" "$metricsDir/prometheus" # Pull and start metrics containers if [ -n "$dockerWithSudo" ]; then From 36fa6e9a78bb3b72a4106f99e0371abb1431921b Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Tue, 10 Feb 2026 16:11:46 +0100 Subject: [PATCH 13/15] fix: rename variable --- generate-ansible-inventory.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/generate-ansible-inventory.sh b/generate-ansible-inventory.sh index b60357c..a385d75 100755 --- a/generate-ansible-inventory.sh +++ b/generate-ansible-inventory.sh @@ -9,7 +9,7 @@ if [ $# -lt 2 ]; then exit 1 fi -DEPLOY_CONFIG="$1" +DEPLOY_VALIDATOR_CONFIG="$1" OUTPUT_FILE="$2" # Check if yq is installed @@ -21,8 +21,8 @@ if ! command -v yq &> /dev/null; then fi # Check if validator config exists -if [ ! -f "$DEPLOY_CONFIG" ]; then - echo "Error: Deploy config file not found: $DEPLOY_CONFIG" +if [ ! -f "$DEPLOY_VALIDATOR_CONFIG" ]; then + echo "Error: Deploy config file not found: $DEPLOY_VALIDATOR_CONFIG" exit 1 fi @@ -63,7 +63,7 @@ all: EOF # Extract node information from deploy-validator-config.yaml -nodes=($(yq eval '.validators[].name' "$DEPLOY_CONFIG")) +nodes=($(yq eval '.validators[].name' "$DEPLOY_VALIDATOR_CONFIG")) # Process each node and generate inventory entries for node_name in "${nodes[@]}"; do @@ -73,8 +73,8 @@ for node_name in "${nodes[@]}"; do group_name="${client_type}_nodes" # Extract node-specific information - node_ip=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.ip // \"127.0.0.1\"" "$DEPLOY_CONFIG") - node_quic=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.quic // \"9000\"" "$DEPLOY_CONFIG") + node_ip=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.ip // \"127.0.0.1\"" "$DEPLOY_VALIDATOR_CONFIG") + node_quic=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.quic // \"9000\"" "$DEPLOY_VALIDATOR_CONFIG") # Check if this is a remote deployment (IP is not localhost/127.0.0.1) is_remote=false From 2c2c398c47865bc01279c168de863dba4927d711 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Tue, 17 Feb 2026 17:50:34 +0100 Subject: [PATCH 14/15] fix: add deploy-validator-config to genesis generator --- generate-genesis.sh | 25 ++++++++++++++++++------- spin-node.sh | 13 +++++++------ 2 files changed, 25 insertions(+), 13 deletions(-) diff --git a/generate-genesis.sh b/generate-genesis.sh index cd29b23..5736f80 100755 --- a/generate-genesis.sh +++ b/generate-genesis.sh @@ -22,8 +22,9 @@ Generates: config.yaml, validators.yaml, nodes.yaml, genesis.json, genesis.ssz, Arguments: genesis-directory Path to the genesis directory containing: - - validator-config.yaml (with node configurations and individual counts) - - validator-config.yaml must include key: config.activeEpoch (positive integer) + - deploy-validator-config.yaml (preferred, merged config with user overrides) + - validator-config.yaml (fallback if deploy-validator-config.yaml not found) + - config must include key: config.activeEpoch (positive integer) Options: --mode Deployment mode: 'local' or 'ansible' (default: local) @@ -47,13 +48,13 @@ Generated Files: How It Works: 1. Calculates GENESIS_TIME based on --mode (local: +30s, ansible: +360s) or --offset if provided - 2. Reads individual validator 'count' fields from validator-config.yaml - 3. Reads config.activeEpoch from validator-config.yaml (required) + 2. Reads individual validator 'count' fields from deploy-validator-config.yaml (or validator-config.yaml as fallback) + 3. Reads config.activeEpoch from the validator config file (required) 4. Automatically sums them to calculate total VALIDATOR_COUNT 5. Generates config.yaml from scratch with calculated values including config.activeEpoch 6. Runs PK's genesis generator with correct parameters -Note: config.yaml is a generated file - only edit validator-config.yaml +Note: config.yaml is a generated file - only edit validator-config.yaml (or user-config.yml for overrides) Requirements: - Docker (to run PK's eth-beacon-genesis tool) @@ -83,7 +84,17 @@ fi GENESIS_DIR="$1" CONFIG_FILE="$GENESIS_DIR/config.yaml" -VALIDATOR_CONFIG_FILE="$GENESIS_DIR/validator-config.yaml" + +# Prefer deploy-validator-config.yaml (merged config with user overrides) if available, +# otherwise fall back to validator-config.yaml for standalone usage +DEPLOY_CONFIG="$GENESIS_DIR/deploy-validator-config.yaml" +BASE_CONFIG="$GENESIS_DIR/validator-config.yaml" +if [ -f "$DEPLOY_CONFIG" ]; then + VALIDATOR_CONFIG_FILE="$DEPLOY_CONFIG" +else + echo " deploy-validator-config.yaml not found, falling back to validator-config.yaml" + VALIDATOR_CONFIG_FILE="$BASE_CONFIG" +fi # Parse optional flags SKIP_KEY_GEN="true" @@ -425,7 +436,7 @@ docker run --rm --pull=never \ "$PK_DOCKER_IMAGE" \ leanchain \ --config "/data/genesis/config.yaml" \ - --mass-validators "/data/genesis/validator-config.yaml" \ + --mass-validators "/data/genesis/$(basename "$VALIDATOR_CONFIG_FILE")" \ --state-output "/data/genesis/genesis.ssz" \ --json-output "/data/genesis/genesis.json" \ --nodes-output "/data/genesis/nodes.yaml" \ diff --git a/spin-node.sh b/spin-node.sh index fcd8a73..9b71810 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -51,17 +51,18 @@ if [ "$deployment_mode" == "ansible" ] && ([ "$validatorConfig" == "genesis_boot echo "Using Ansible deployment: configDir=$configDir, validator config=$validator_config_file" fi -#1. setup genesis params and run genesis generator -source "$(dirname $0)/set-up.sh" -# ✅ Genesis generator implemented using PK's eth-beacon-genesis tool -# Generates: validators.yaml, nodes.yaml, genesis.json, genesis.ssz, and .key files - -# 2. Create deploy-validator-config.yaml (merges user config overrides if provided) +# 1. Create deploy-validator-config.yaml (merges user config overrides if provided) +# This must run BEFORE genesis generation so that generate-genesis.sh can read the merged config echo "" echo "Creating deploy-validator-config.yaml..." "$scriptDir/scripts/merge-config.sh" "$validator_config_file" "$configFile" deploy_validator_config_file="$configDir/deploy-validator-config.yaml" +# 2. Setup genesis params and run genesis generator +source "$(dirname $0)/set-up.sh" +# ✅ Genesis generator implemented using PK's eth-beacon-genesis tool +# Generates: validators.yaml, nodes.yaml, genesis.json, genesis.ssz, and .key files + # 3. collect the nodes that the user has asked us to spin and perform setup # Load nodes from deploy-validator-config.yaml From 4450189f4b57d4c31a83bc2508b9f45339f93c64 Mon Sep 17 00:00:00 2001 From: Katya Ryazantseva Date: Thu, 19 Feb 2026 14:22:05 +0100 Subject: [PATCH 15/15] feat: add overrides for docker_image and run_mode --- .gitignore | 5 +- README.md | 115 +++++++--------- ansible-devnet/genesis/validator-config.yaml | 7 - ansible/README.md | 6 +- ansible/inventory/group_vars/all.yml | 2 +- ansible/playbooks/clean-node-data.yml | 16 +-- ansible/playbooks/copy-genesis.yml | 10 +- ansible/playbooks/deploy-nodes.yml | 26 ++-- .../playbooks/helpers/deploy-single-node.yml | 10 +- ansible/playbooks/stop-nodes.yml | 16 +-- ansible/roles/ethlambda/tasks/main.yml | 42 ++++-- ansible/roles/grandine/tasks/main.yml | 44 ++++-- ansible/roles/lantern/tasks/main.yml | 45 +++++-- ansible/roles/lighthouse/tasks/main.yml | 45 +++++-- ansible/roles/qlean/tasks/main.yml | 52 ++++++-- ansible/roles/ream/tasks/main.yml | 43 ++++-- ansible/roles/zeam/tasks/main.yml | 52 ++++++-- client-cmds/ethlambda-cmd.sh | 5 +- client-cmds/grandine-cmd.sh | 2 +- client-cmds/lantern-cmd.sh | 9 +- client-cmds/lighthouse-cmd.sh | 6 +- client-cmds/qlean-cmd.sh | 2 +- client-cmds/ream-cmd.sh | 2 +- client-cmds/zeam-cmd.sh | 2 +- generate-ansible-inventory.sh | 22 +-- generate-genesis.sh | 25 +--- local-devnet/genesis/validator-config.yaml | 7 - parse-env.sh | 6 - parse-vc.sh | 65 ++++----- run-ansible.sh | 37 +---- scripts/merge-config.sh | 74 ---------- spin-node.sh | 126 +++++++----------- user-config.yml.example | 41 ++---- 33 files changed, 465 insertions(+), 502 deletions(-) delete mode 100755 scripts/merge-config.sh diff --git a/.gitignore b/.gitignore index 9baba83..914a1e7 100644 --- a/.gitignore +++ b/.gitignore @@ -15,13 +15,10 @@ genesis.json genesis.ssz annotated_validators.yaml -# Deploy config (created by merge-config.sh, merges validator-config + user overrides) -deploy-validator-config.yaml - # Hash-sig validator keys (contains sensitive secret keys) hash-sig-keys/ -# Auto-generated Ansible inventory (generated from deploy-validator-config.yaml) +# Auto-generated Ansible inventory (generated from validator-config.yaml) hosts.yml # Temporary cache files diff --git a/README.md b/README.md index 12c34e6..cf0b00f 100644 --- a/README.md +++ b/README.md @@ -82,44 +82,6 @@ Grafana is started with the two pre-provisioned dashboards from [leanMetrics](ht > **Note:** The `--metrics` flag only affects local deployments. When using Ansible deployment mode, this flag is ignored. Metrics ports are always exposed by clients regardless of this flag. -### Using custom Docker images - -You can override default Docker images using the `--configFile` flag. This is useful for testing custom builds or using specific versions without modifying the codebase. - -**Basic usage (without custom images):** -```sh -# Uses default images from validator-config.yaml (merged into deploy-validator-config.yaml) -NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis -``` - -**With custom config file:** -```sh -# Override specific node images -NETWORK_DIR=local-devnet ./spin-node.sh --node all --generateGenesis --configFile user-config.yml -``` - -**Example config file (user-config.yml):** -```yaml -validators: - - name: "zeam_0" - image: blockblaz/zeam:feature-branch - - name: "ream_0" - image: ghcr.io/reamlabs/ream:v2.0 -``` - -**Testing a specific client build:** -```sh -# Create custom config file for zeam /my-zeam-config.yml -validators: - - name: "zeam_0" - image: blockblaz/zeam:custom-tag - -# Run with custom zeam image -NETWORK_DIR=local-devnet ./spin-node.sh --node zeam_0 --configFile /my-zeam-config.yml -``` - -Only specify validators you want to override - others will use their defaults from `validator-config.yaml`. Overrides are merged into `deploy-validator-config.yaml` which is used during deployment. - ## Args 1. `NETWORK_DIR` is an env to specify the network directory. Should have a `genesis` directory with genesis config. A `data` folder will be created inside this `NETWORK_DIR` if not already there. @@ -170,11 +132,12 @@ Only specify validators you want to override - others will use their defaults fr - The script will automatically pull the specified Docker images before running containers - Example: `--tag devnet0` or `--tag devnet1` 11. `--metrics` starts a Prometheus + Grafana metrics stack alongside the devnet (local deployments only). When specified: - - Generates `metrics/prometheus/prometheus.yml` from `deploy-validator-config.yaml` with scrape targets for all configured nodes + - Generates `metrics/prometheus/prometheus.yml` from `validator-config.yaml` with scrape targets for all configured nodes - Starts Prometheus (http://localhost:9090) and Grafana (http://localhost:3000) via Docker Compose - Grafana is pre-provisioned with Lean Ethereum dashboards (no login required) - On `--stop --metrics`, the metrics stack is also torn down - On Ctrl+C cleanup, the metrics stack is stopped automatically + Note: Client metrics endpoints are always enabled regardless of this flag. 12. `--checkpoint-sync-url` specifies the URL to fetch finalized checkpoint state from for checkpoint sync. Default: `https://leanpoint.leanroadmap.org/lean/v0/states/finalized`. Only used when `--restart-client` is specified. 13. `--restart-client` comma-separated list of client node names (e.g., `zeam_0,ream_0`). When specified, those clients are stopped, their data cleared, and restarted using checkpoint sync. Genesis is skipped. Use with `--checkpoint-sync-url` to override the default URL. @@ -225,7 +188,6 @@ Current following clients are supported: 4. Lantern 5. Lighthouse 6. Grandine -7. EthLambda However adding a lean client to this setup is very easy. Feel free to do the PR or reach out to the maintainers. @@ -245,38 +207,22 @@ The quickstart uses separate directories for local and Ansible deployments: ``` lean-quickstart/ -├── client-cmds/ # Client command scripts -│ ├── zeam-cmd.sh -│ ├── ream-cmd.sh -│ └── ... -├── user-config.yml.example # Example custom config (copy to user-config.yml) -├── user-config.yml # Your custom image overrides (gitignored) -├── local-devnet/ # Local development +├── local-devnet/ # Local development │ ├── genesis/ -│ │ ├── validator-config.yaml # Source config - Local IPs (127.0.0.1) -│ │ └── deploy-validator-config.yaml # Generated - merged config for deployment (gitignored) +│ │ └── validator-config.yaml # Local IPs (127.0.0.1) │ └── data/ # Node data directories │ -└── ansible-devnet/ # Ansible/remote deployment +└── ansible-devnet/ # Ansible/remote deployment ├── genesis/ - │ ├── validator-config.yaml # Source config - Remote IPs (your server IPs) - │ └── deploy-validator-config.yaml # Generated - merged config for deployment (gitignored) + │ └── validator-config.yaml # Remote IPs (your server IPs) └── data/ # Node data directories ``` **Automatic Directory Selection:** - - When `deployment_mode: ansible` is set (in config or via `--deploymentMode ansible`), the script automatically uses `ansible-devnet/genesis/validator-config.yaml` - This keeps local and remote configurations completely separate - Genesis files are generated in the appropriate directory based on deployment mode -**Config File Flow:** - -- `validator-config.yaml` - Default source config file (node IPs, ports, validator counts) -- `user-config.yml` - Optional overrides (currently custom Docker images) -- `deploy-validator-config.yaml` - Generated by merging the above two files, recreated on each deployment -- All deployment operations (local and ansible) read from `deploy-validator-config.yaml` - ### Configuration The `validator-config.yaml` file defines the shuffle algorithm, active epoch configuration, and validator nodes specifications: @@ -429,13 +375,13 @@ Post genesis generation, the quickstarts loads and calls the appropriate node's **Client Integration:** Your client implementation should read these environment variables and use the hash-sig keys for validator operations. - - `$item` - the node name for which this cmd is being executed, index into `deploy-validator-config.yaml` for its configuration + - `$item` - the node name for which this cmd is being executed, index into `validator-config.yaml` for its configuration - `$configDir` - the abs folder housing `genesis` configuration (same as `NETWORK_DIR` env variable provided while executing shell command), already mapped to `/config` in the docker mode - A generic data folder is created inside config folder accessible as `$dataDir` with `$dataDir/$item` to be used as the data dir for a particular node to be used for binary format, already mapped to `/data` in the docker mode - - Variables read and available from `deploy-validator-config.yaml` (use them or directly read configuration from the `deploy-validator-config.yaml` using `$item` as the index into `validators` section) + - Variables read and available from `validator-config.yaml` (use them or directly read configuration from the `validator-config.yaml` using `$item` as the index into `validators` section) - `$metricsPort` - - `$quicPort` - - `$item.key` filename of the p2p `privkey` read and dumped into file from `deploy-validator-config.yaml` inside config dir (so `$configDir/$item.key` or `/config/$item.key`) + - `$quicPort` + - `$item.key` filename of the p2p `privkey` read and dumped into file from `validator-config.yaml` inside config dir (so `$configDir/$item.key` or `/config/$item.key`) Here is an example client cmd: ```bash @@ -467,6 +413,41 @@ node_docker="--platform linux/amd64 qdrvm/qlean-mini:dd67521 \ node_setup="docker" ``` +### User Configuration Overrides (`user-config.yml`) + +You can override the Docker image or run mode (docker/binary) for individual nodes without modifying any tracked files. This is useful for testing custom builds or switching a node to binary mode for local debugging. + +**Setup:** +```sh +# Copy the example file +cp user-config.yml.example user-config.yml + +# Edit to override specific nodes +``` + +**Format** — specify only the nodes you want to override: +```yaml +zeam_0: + run_mode: docker + docker_image: blockblaz/zeam:custom-tag + +ream_0: + run_mode: binary +``` + +**Fields:** +| Field | Description | +|-------|-------------| +| `run_mode` | `"docker"` or `"binary"` — overrides the default set in the client-cmd script | +| `docker_image` | Docker image to use (only applies when `run_mode` is `docker`) | + +**How it works:** +- `user-config.yml` is auto-detected from the project root during `spin-node.sh` execution +- Overrides are applied per-node after sourcing the client-cmd script, right before spinning the node +- If `run_mode` is set to `binary`, `docker_image` is ignored +- Nodes not listed in the file use their defaults from `client-cmds/-cmd.sh` +- The file is gitignored so your local overrides won't affect others + ## Key Management ### Key Lifetime @@ -591,7 +572,7 @@ This quickstart includes automated configuration parsing: - **Official Genesis Generation**: Uses PK's `eth-beacon-genesis` docker tool from [PR #36](https://github.com/ethpandaops/eth-beacon-genesis/pull/36) - **Complete File Set**: Generates `validators.yaml`, `nodes.yaml`, `genesis.json`, `genesis.ssz`, and `.key` files -- **QUIC Port Detection**: Automatically extracts QUIC ports from `deploy-validator-config.yaml` using `yq` +- **QUIC Port Detection**: Automatically extracts QUIC ports from `validator-config.yaml` using `yq` - **Node Detection**: Dynamically discovers available nodes from the validator configuration - **Private Key Management**: Automatically extracts and creates `.key` files for each node - **Error Handling**: Provides clear error messages when nodes or ports are not found @@ -759,7 +740,7 @@ ansible/ ### Remote Deployment -The Ansible inventory is **automatically generated** from `deploy-validator-config.yaml`. +The Ansible inventory is **automatically generated** from `validator-config.yaml`. **Configuration Setup:** @@ -868,7 +849,7 @@ Both deployment modes use the same `spin-node.sh` entry point, controlled by `de | **Multi-Host** | No | Yes | | **Rollback** | Manual | Built-in capabilities | | **Entry Point** | `spin-node.sh` | `spin-node.sh` (same command) | -| **Inventory** | N/A | Auto-generated from deploy-validator-config.yaml | +| **Inventory** | N/A | Auto-generated from validator-config.yaml | **Recommendation:** - Use `deployment_mode: local` for local development and quick testing diff --git a/ansible-devnet/genesis/validator-config.yaml b/ansible-devnet/genesis/validator-config.yaml index b5512f2..091275a 100644 --- a/ansible-devnet/genesis/validator-config.yaml +++ b/ansible-devnet/genesis/validator-config.yaml @@ -8,7 +8,6 @@ validators: - name: "zeam_0" # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f - image: blockblaz/zeam:latest privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: # verify /ip4/127.0.0.1/udp/9000/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f @@ -21,7 +20,6 @@ validators: - name: "ream_0" # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 - image: ghcr.io/reamlabs/ream:latest privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -34,7 +32,6 @@ validators: - name: "qlean_0" # TODO: add a third entry in nodes.yaml corresponding to this - image: qdrvm/qlean-mini:latest privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -47,7 +44,6 @@ validators: - name: "lantern_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - image: piertwo/lantern:v0.0.2 privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -60,7 +56,6 @@ validators: - name: "lighthouse_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - image: hopinheimer/lighthouse:latest privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -71,7 +66,6 @@ validators: count: 1 - name: "grandine_0" - image: sifrai/lean:latest privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" enrFields: ip: "37.27.250.20" @@ -81,7 +75,6 @@ validators: count: 1 - name: "ethlambda_0" - image: ghcr.io/lambdaclass/ethlambda:latest privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" enrFields: ip: "78.47.44.215" diff --git a/ansible/README.md b/ansible/README.md index 8628e5f..0d41329 100644 --- a/ansible/README.md +++ b/ansible/README.md @@ -409,11 +409,11 @@ Or use sudo (not recommended): ./ansible-deploy.sh --node zeam_0 --network-dir local-devnet --docker-with-sudo ``` -#### 5. "Node not found in deploy-validator-config.yaml" +#### 5. "Node not found in validator-config.yaml" -Ensure node name matches exactly in `deploy-validator-config.yaml`: +Ensure node name matches exactly in `validator-config.yaml`: ```sh -yq eval '.validators[].name' local-devnet/genesis/deploy-validator-config.yaml +yq eval '.validators[].name' local-devnet/genesis/validator-config.yaml ``` #### 6. Container starts but immediately exits diff --git a/ansible/inventory/group_vars/all.yml b/ansible/inventory/group_vars/all.yml index df2f8f3..418ef92 100644 --- a/ansible/inventory/group_vars/all.yml +++ b/ansible/inventory/group_vars/all.yml @@ -5,7 +5,7 @@ network_dir: "{{ playbook_dir }}/../local-devnet" genesis_dir: "{{ network_dir }}/genesis" data_dir: "{{ network_dir }}/data" -deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" +validator_config_file: "{{ genesis_dir }}/validator-config.yaml" # Remote paths (for remote deployments) # Use standard paths on remote hosts diff --git a/ansible/playbooks/clean-node-data.yml b/ansible/playbooks/clean-node-data.yml index 733ff3d..a583782 100644 --- a/ansible/playbooks/clean-node-data.yml +++ b/ansible/playbooks/clean-node-data.yml @@ -6,7 +6,7 @@ connection: local gather_facts: no vars: - deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" + validator_config_file: "{{ genesis_dir }}/validator-config.yaml" tags: - zeam - ream @@ -18,15 +18,15 @@ - deploy tasks: - - name: Validate deploy-validator-config.yaml exists + - name: Validate validator-config.yaml exists stat: - path: "{{ deploy_validator_config_file }}" - register: deploy_config_stat + path: "{{ validator_config_file }}" + register: validator_config_stat - - name: Fail if deploy-validator-config.yaml missing + - name: Fail if validator-config.yaml missing fail: - msg: "deploy-validator-config.yaml not found at {{ deploy_validator_config_file }}" - when: not deploy_config_stat.stat.exists + msg: "validator-config.yaml not found at {{ validator_config_file }}" + when: not validator_config_stat.stat.exists - name: Verify yq is available command: yq --version @@ -42,7 +42,7 @@ - name: Extract all node names shell: | - yq eval '.validators[].name' {{ deploy_validator_config_file }} + yq eval '.validators[].name' {{ validator_config_file }} register: all_node_names_raw changed_when: false diff --git a/ansible/playbooks/copy-genesis.yml b/ansible/playbooks/copy-genesis.yml index 9b7f0a1..76a25f8 100644 --- a/ansible/playbooks/copy-genesis.yml +++ b/ansible/playbooks/copy-genesis.yml @@ -12,7 +12,7 @@ hosts: all:!local gather_facts: yes vars: - deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" + validator_config_file: "{{ genesis_dir }}/validator-config.yaml" # Use remote_genesis_dir from group_vars, or default to standard remote path actual_remote_genesis_dir: "{{ remote_genesis_dir | default('/opt/lean-quickstart/genesis') }}" @@ -51,7 +51,7 @@ - nodes.yaml - genesis.json - genesis.ssz - - deploy-validator-config.yaml + - validator-config.yaml - name: Fail if required genesis files are missing fail: @@ -68,9 +68,9 @@ delegate_to: localhost run_once: true - - name: Extract all node names from deploy-validator-config.yaml + - name: Extract all node names from validator-config.yaml shell: | - yq eval '.validators[].name' "{{ genesis_dir }}/deploy-validator-config.yaml" + yq eval '.validators[].name' "{{ genesis_dir }}/validator-config.yaml" register: all_node_names_raw delegate_to: localhost run_once: true @@ -141,7 +141,7 @@ - nodes.yaml - genesis.json - genesis.ssz - - deploy-validator-config.yaml + - validator-config.yaml - name: Copy node private key files to remote host copy: diff --git a/ansible/playbooks/deploy-nodes.yml b/ansible/playbooks/deploy-nodes.yml index 1d543ba..f89610a 100644 --- a/ansible/playbooks/deploy-nodes.yml +++ b/ansible/playbooks/deploy-nodes.yml @@ -4,7 +4,7 @@ # This playbook runs on localhost to parse node names, then deploys to remote hosts # # Before deployment, it syncs essential config files from local to remote: -# - deploy-validator-config.yaml (validator config merged with user overrides) +# - validator-config.yaml # - node key files (*.key) # - config.yaml, validators.yaml, nodes.yaml # - genesis.ssz, genesis.json @@ -15,22 +15,22 @@ connection: local gather_facts: yes vars: - deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" + validator_config_file: "{{ genesis_dir }}/validator-config.yaml" tasks: - - name: Validate deploy-validator-config.yaml exists + - name: Validate validator-config.yaml exists stat: - path: "{{ deploy_validator_config_file }}" - register: deploy_config_stat + path: "{{ validator_config_file }}" + register: validator_config_stat - - name: Fail if deploy-validator-config.yaml missing + - name: Fail if validator-config.yaml missing fail: - msg: "deploy-validator-config.yaml not found at {{ deploy_validator_config_file }}" - when: not deploy_config_stat.stat.exists + msg: "validator-config.yaml not found at {{ validator_config_file }}" + when: not validator_config_stat.stat.exists - name: Extract all node names shell: | - yq eval '.validators[].name' {{ deploy_validator_config_file }} + yq eval '.validators[].name' {{ validator_config_file }} register: all_node_names_raw changed_when: false @@ -101,7 +101,7 @@ # Use remote paths on remote hosts genesis_dir: "{{ remote_genesis_dir | default('/opt/lean-quickstart/genesis') }}" data_dir: "{{ remote_data_dir | default('/opt/lean-quickstart/data') }}" - deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" + validator_config_file: "{{ genesis_dir }}/validator-config.yaml" # node_name is set to inventory_hostname (which matches the node name) node_name: "{{ inventory_hostname }}" # Local genesis directory for syncing config files (from localhost play) @@ -117,10 +117,10 @@ - deploy - sync - - name: Sync deploy-validator-config.yaml to remote host + - name: Sync validator-config.yaml to remote host copy: - src: "{{ local_genesis_dir }}/deploy-validator-config.yaml" - dest: "{{ genesis_dir }}/deploy-validator-config.yaml" + src: "{{ local_genesis_dir }}/validator-config.yaml" + dest: "{{ genesis_dir }}/validator-config.yaml" mode: '0644' force: yes tags: diff --git a/ansible/playbooks/helpers/deploy-single-node.yml b/ansible/playbooks/helpers/deploy-single-node.yml index 48a3c5e..34f17f7 100644 --- a/ansible/playbooks/helpers/deploy-single-node.yml +++ b/ansible/playbooks/helpers/deploy-single-node.yml @@ -6,14 +6,14 @@ set_fact: client_type: "{{ node_name.split('_')[0] }}" -- name: Set deploy config file paths +- name: Set validator config file paths set_fact: - actual_deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" - local_deploy_validator_config_file: "{{ hostvars['localhost']['local_genesis_dir_path'] }}/deploy-validator-config.yaml" + actual_validator_config_file: "{{ genesis_dir }}/validator-config.yaml" + local_validator_config_file: "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" - name: Extract node configuration (from local config) shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .name" {{ local_deploy_validator_config_file }} + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .name" {{ local_validator_config_file }} register: node_check changed_when: false failed_when: false @@ -21,7 +21,7 @@ - name: Fail if node not found in config fail: - msg: "Node '{{ node_name }}' not found in {{ local_deploy_validator_config_file }}" + msg: "Node '{{ node_name }}' not found in {{ local_validator_config_file }}" when: node_check.stdout == "" or node_check.rc != 0 - name: Set zeam validator config diff --git a/ansible/playbooks/stop-nodes.yml b/ansible/playbooks/stop-nodes.yml index 3a6fc46..af706a3 100644 --- a/ansible/playbooks/stop-nodes.yml +++ b/ansible/playbooks/stop-nodes.yml @@ -7,22 +7,22 @@ connection: local gather_facts: yes vars: - deploy_validator_config_file: "{{ genesis_dir }}/deploy-validator-config.yaml" + validator_config_file: "{{ genesis_dir }}/validator-config.yaml" tasks: - - name: Validate deploy-validator-config.yaml exists + - name: Validate validator-config.yaml exists stat: - path: "{{ deploy_validator_config_file }}" - register: deploy_config_stat + path: "{{ validator_config_file }}" + register: validator_config_stat - - name: Fail if deploy-validator-config.yaml missing + - name: Fail if validator-config.yaml missing fail: - msg: "deploy-validator-config.yaml not found at {{ deploy_validator_config_file }}" - when: not deploy_config_stat.stat.exists + msg: "validator-config.yaml not found at {{ validator_config_file }}" + when: not validator_config_stat.stat.exists - name: Extract all node names shell: | - yq eval '.validators[].name' {{ deploy_validator_config_file }} + yq eval '.validators[].name' {{ validator_config_file }} register: all_node_names_raw changed_when: false diff --git a/ansible/roles/ethlambda/tasks/main.yml b/ansible/roles/ethlambda/tasks/main.yml index b084031..d8ff446 100644 --- a/ansible/roles/ethlambda/tasks/main.yml +++ b/ansible/roles/ethlambda/tasks/main.yml @@ -1,25 +1,49 @@ --- # Ethlambda role: Deploy and manage Ethlambda nodes # Converts client-cmds/ethlambda-cmd.sh logic to Ansible tasks -# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from deploy-validator-config.yaml +- name: Extract docker image from client-cmd.sh shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" + # Extract the first word (docker image) from node_docker line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_docker=' "$project_root/client-cmds/ethlambda-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + register: ethlambda_docker_image_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Extract deployment mode from client-cmd.sh + shell: | + # Extract the value from node_setup line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_setup=' "$project_root/client-cmds/ethlambda-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' + register: ethlambda_deployment_mode_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set docker image and deployment mode from client-cmd.sh + set_fact: + ethlambda_docker_image: "{{ ethlambda_docker_image_raw.stdout | trim | default('ghcr.io/lambdaclass/ethlambda:devnet1') }}" + deployment_mode: "{{ ethlambda_deployment_mode_raw.stdout | trim | default('docker') }}" + +- name: Extract node configuration from validator-config.yaml + shell: | + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" register: ethlambda_node_config changed_when: false + delegate_to: localhost loop: - - image - enrFields.quic - metricsPort when: node_name is defined -- name: Set node configuration +- name: Set node ports set_fact: - ethlambda_docker_image: "{{ ethlambda_node_config.results[0].stdout | trim | default('ghcr.io/lambdaclass/ethlambda:latest') }}" - ethlambda_quic_port: "{{ ethlambda_node_config.results[1].stdout | trim }}" - ethlambda_metrics_port: "{{ ethlambda_node_config.results[2].stdout | trim }}" - deployment_mode: "{{ deployment_mode | default('docker') }}" + ethlambda_quic_port: "{{ ethlambda_node_config.results[0].stdout }}" + ethlambda_metrics_port: "{{ ethlambda_node_config.results[1].stdout }}" when: ethlambda_node_config is defined - name: Ensure node key file exists diff --git a/ansible/roles/grandine/tasks/main.yml b/ansible/roles/grandine/tasks/main.yml index d6b9830..1fd9558 100644 --- a/ansible/roles/grandine/tasks/main.yml +++ b/ansible/roles/grandine/tasks/main.yml @@ -1,27 +1,51 @@ --- # Grandine role: Deploy and manage Grandine nodes # Converts client-cmds/grandine-cmd.sh logic to Ansible tasks -# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from deploy-validator-config.yaml +- name: Extract docker image from client-cmd.sh shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" + # Extract the first word (docker image) from node_docker line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_docker=' "$project_root/client-cmds/grandine-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + register: grandine_docker_image_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Extract deployment mode from client-cmd.sh + shell: | + # Extract the value from node_setup line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_setup=' "$project_root/client-cmds/grandine-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' + register: grandine_deployment_mode_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set docker image and deployment mode from client-cmd.sh + set_fact: + grandine_docker_image: "{{ grandine_docker_image_raw.stdout | trim | default('sifrai/lean:unstable') }}" + deployment_mode: "{{ grandine_deployment_mode_raw.stdout | trim | default('docker') }}" + +- name: Extract node configuration from validator-config.yaml + shell: | + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" register: grandine_node_config changed_when: false + delegate_to: localhost loop: - - image - enrFields.quic - metricsPort - privkey when: node_name is defined -- name: Set node configuration +- name: Set node ports set_fact: - grandine_docker_image: "{{ grandine_node_config.results[0].stdout | trim | default('sifrai/lean:unstable') }}" - grandine_quic_port: "{{ grandine_node_config.results[1].stdout | trim }}" - grandine_metrics_port: "{{ grandine_node_config.results[2].stdout | trim }}" - grandine_privkey: "{{ grandine_node_config.results[3].stdout | trim }}" - deployment_mode: "{{ deployment_mode | default('docker') }}" + grandine_quic_port: "{{ grandine_node_config.results[0].stdout }}" + grandine_metrics_port: "{{ grandine_node_config.results[1].stdout }}" + grandine_privkey: "{{ grandine_node_config.results[2].stdout }}" when: grandine_node_config is defined - name: Ensure node key file exists diff --git a/ansible/roles/lantern/tasks/main.yml b/ansible/roles/lantern/tasks/main.yml index 14319ba..23b9246 100644 --- a/ansible/roles/lantern/tasks/main.yml +++ b/ansible/roles/lantern/tasks/main.yml @@ -1,25 +1,50 @@ --- # Lantern role: Deploy and manage Lantern nodes # Converts client-cmds/lantern-cmd.sh logic to Ansible tasks -# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from deploy-validator-config.yaml +- name: Extract docker image from client-cmd.sh shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" + # Extract the docker image from LANTERN_IMAGE line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^LANTERN_IMAGE=' "$project_root/client-cmds/lantern-cmd.sh" | head -1 | sed -E 's/.*LANTERN_IMAGE="([^"]+)".*/\1/' + register: lantern_docker_image_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Extract deployment mode from client-cmd.sh + shell: | + # Extract the value from node_setup line + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_setup=' "$project_root/client-cmds/lantern-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' + register: lantern_deployment_mode_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set docker image and deployment mode from client-cmd.sh + set_fact: + lantern_docker_image: "{{ lantern_docker_image_raw.stdout | trim | default('piertwo/lantern:v0.0.1') }}" + deployment_mode: "{{ lantern_deployment_mode_raw.stdout | trim | default('docker') }}" + +- name: Extract node configuration from validator-config.yaml + shell: | + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" register: lantern_node_config changed_when: false + delegate_to: localhost loop: - - image - enrFields.quic - metricsPort + - privkey when: node_name is defined -- name: Set node configuration +- name: Set node ports set_fact: - lantern_docker_image: "{{ lantern_node_config.results[0].stdout | trim | default('piertwo/lantern:latest') }}" - lantern_quic_port: "{{ lantern_node_config.results[1].stdout | trim }}" - lantern_metrics_port: "{{ lantern_node_config.results[2].stdout | trim }}" - deployment_mode: "{{ deployment_mode | default('docker') }}" + lantern_quic_port: "{{ lantern_node_config.results[0].stdout }}" + lantern_metrics_port: "{{ lantern_node_config.results[1].stdout }}" + lantern_privkey: "{{ lantern_node_config.results[2].stdout }}" when: lantern_node_config is defined - name: Ensure node key file exists @@ -65,7 +90,7 @@ --genesis-config /config/config.yaml --validator-registry-path /config/validators.yaml --genesis-state /config/genesis.ssz - --validator-config /config/deploy-validator-config.yaml + --validator-config /config/validator-config.yaml --nodes-path /config/nodes.yaml --node-id {{ node_name }} --node-key-path /config/{{ node_name }}.key diff --git a/ansible/roles/lighthouse/tasks/main.yml b/ansible/roles/lighthouse/tasks/main.yml index 648e357..d87c831 100644 --- a/ansible/roles/lighthouse/tasks/main.yml +++ b/ansible/roles/lighthouse/tasks/main.yml @@ -1,25 +1,50 @@ --- # Lighthouse role: Deploy and manage Lighthouse nodes # Converts client-cmds/lighthouse-cmd.sh logic to Ansible tasks -# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from deploy-validator-config.yaml +- name: Extract docker image from client-cmd.sh shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" + # Extract the first word (docker image) from node_docker line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_docker=' "$project_root/client-cmds/lighthouse-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + register: lighthouse_docker_image_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Extract deployment mode from client-cmd.sh + shell: | + # Extract the value from node_setup line + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_setup=' "$project_root/client-cmds/lighthouse-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' + register: lighthouse_deployment_mode_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set docker image and deployment mode from client-cmd.sh + set_fact: + lighthouse_docker_image: "{{ lighthouse_docker_image_raw.stdout | trim | default('hopinheimer/lighthouse:latest') }}" + deployment_mode: "{{ lighthouse_deployment_mode_raw.stdout | trim | default('docker') }}" + +- name: Extract node configuration from validator-config.yaml + shell: | + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" register: lighthouse_node_config changed_when: false + delegate_to: localhost loop: - - image - enrFields.quic - metricsPort + - privkey when: node_name is defined -- name: Set node configuration +- name: Set node ports set_fact: - lighthouse_docker_image: "{{ lighthouse_node_config.results[0].stdout | trim | default('hopinheimer/lighthouse:latest') }}" - lighthouse_quic_port: "{{ lighthouse_node_config.results[1].stdout | trim }}" - lighthouse_metrics_port: "{{ lighthouse_node_config.results[2].stdout | trim }}" - deployment_mode: "{{ deployment_mode | default('docker') }}" + lighthouse_quic_port: "{{ lighthouse_node_config.results[0].stdout }}" + lighthouse_metrics_port: "{{ lighthouse_node_config.results[1].stdout }}" + lighthouse_privkey: "{{ lighthouse_node_config.results[2].stdout }}" when: lighthouse_node_config is defined - name: Ensure node key file exists @@ -68,7 +93,7 @@ lighthouse lean_node --datadir /data --config /config/config.yaml - --validators /config/deploy-validator-config.yaml + --validators /config/validator-config.yaml --nodes /config/nodes.yaml --node-id {{ node_name }} --private-key /config/{{ node_name }}.key diff --git a/ansible/roles/qlean/tasks/main.yml b/ansible/roles/qlean/tasks/main.yml index e1bfd4a..1cb505c 100644 --- a/ansible/roles/qlean/tasks/main.yml +++ b/ansible/roles/qlean/tasks/main.yml @@ -1,38 +1,64 @@ --- # Qlean role: Deploy and manage Qlean nodes # Converts client-cmds/qlean-cmd.sh logic to Ansible tasks -# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from deploy-validator-config.yaml +- name: Extract docker image from client-cmd.sh shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" + # Extract the first word (docker image) from node_docker line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_docker=' "$project_root/client-cmds/qlean-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + register: qlean_docker_image_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Extract deployment mode from client-cmd.sh + shell: | + # Extract the value from node_setup line + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_setup=' "$project_root/client-cmds/qlean-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' + register: qlean_deployment_mode_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set docker image and deployment mode from client-cmd.sh + set_fact: + qlean_docker_image: "{{ qlean_docker_image_raw.stdout | trim | default('qdrvm/qlean-mini:dd67521') }}" + deployment_mode: "{{ qlean_deployment_mode_raw.stdout | trim | default('docker') }}" + +- name: Extract node configuration from validator-config.yaml + shell: | + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" register: qlean_node_config changed_when: false + delegate_to: localhost loop: - - image - enrFields.quic - metricsPort + - privkey when: node_name is defined -- name: Set node configuration +- name: Set node ports set_fact: - qlean_docker_image: "{{ qlean_node_config.results[0].stdout | trim | default('qdrvm/qlean-mini:latest') }}" - qlean_quic_port: "{{ qlean_node_config.results[1].stdout | trim }}" - qlean_metrics_port: "{{ qlean_node_config.results[2].stdout | trim }}" - deployment_mode: "{{ deployment_mode | default('docker') }}" + qlean_quic_port: "{{ qlean_node_config.results[0].stdout }}" + qlean_metrics_port: "{{ qlean_node_config.results[1].stdout }}" + qlean_privkey: "{{ qlean_node_config.results[2].stdout }}" when: qlean_node_config is defined - name: Extract validator index from validators.yaml shell: | - yq eval '."{{ node_name }}" | .[0]' "{{ genesis_dir }}/validators.yaml" - register: qlean_validator_index_raw + yq eval '."{{ node_name }}" | .[0]' "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validators.yaml" + register: qlean_validator_index changed_when: false failed_when: false + delegate_to: localhost - name: Set validator index set_fact: - qlean_validator_index: "{{ qlean_validator_index_raw.stdout | trim | default('0') }}" - when: qlean_validator_index_raw is defined + qlean_validator_index: "{{ qlean_validator_index.stdout | trim | default('0') }}" + when: qlean_validator_index is defined - name: Ensure node key file exists stat: diff --git a/ansible/roles/ream/tasks/main.yml b/ansible/roles/ream/tasks/main.yml index 278c630..8008fae 100644 --- a/ansible/roles/ream/tasks/main.yml +++ b/ansible/roles/ream/tasks/main.yml @@ -1,25 +1,50 @@ --- # Ream role: Deploy and manage Ream nodes # Converts client-cmds/ream-cmd.sh logic to Ansible tasks -# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from deploy-validator-config.yaml +- name: Extract docker image from client-cmd.sh shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" + # Extract the first word (docker image) from node_docker line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_docker=' "$project_root/client-cmds/ream-cmd.sh" | head -1 | sed -E 's/.*node_docker="([^ "]+).*/\1/' + register: ream_docker_image_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Extract deployment mode from client-cmd.sh + shell: | + # Extract the value from node_setup line + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_setup=' "$project_root/client-cmds/ream-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' + register: ream_deployment_mode_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set docker image and deployment mode from client-cmd.sh + set_fact: + ream_docker_image: "{{ ream_docker_image_raw.stdout | trim | default('ghcr.io/reamlabs/ream:latest') }}" + deployment_mode: "{{ ream_deployment_mode_raw.stdout | trim | default('docker') }}" + +- name: Extract node configuration from validator-config.yaml + shell: | + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" register: ream_node_config changed_when: false + delegate_to: localhost loop: - - image - enrFields.quic - metricsPort + - privkey when: node_name is defined -- name: Set node configuration +- name: Set node ports set_fact: - ream_docker_image: "{{ ream_node_config.results[0].stdout | trim | default('ghcr.io/reamlabs/ream:latest') }}" - ream_quic_port: "{{ ream_node_config.results[1].stdout | trim }}" - ream_metrics_port: "{{ ream_node_config.results[2].stdout | trim }}" - deployment_mode: "{{ deployment_mode | default('docker') }}" + ream_quic_port: "{{ ream_node_config.results[0].stdout }}" + ream_metrics_port: "{{ ream_node_config.results[1].stdout }}" + ream_privkey: "{{ ream_node_config.results[2].stdout }}" when: ream_node_config is defined - name: Ensure node key file exists diff --git a/ansible/roles/zeam/tasks/main.yml b/ansible/roles/zeam/tasks/main.yml index 5c0fa61..5026ed6 100644 --- a/ansible/roles/zeam/tasks/main.yml +++ b/ansible/roles/zeam/tasks/main.yml @@ -1,24 +1,56 @@ --- # Zeam role: Deploy and manage Zeam nodes # Converts client-cmds/zeam-cmd.sh logic to Ansible tasks -# Note: deploy-validator-config.yaml has user overrides merged (from merge-config.sh) -- name: Extract node configuration from deploy-validator-config.yaml +- name: Extract docker image from client-cmd.sh shell: | - yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ genesis_dir }}/deploy-validator-config.yaml" - register: zeam_node_config + # Extract the docker image from node_docker line (find word containing / and :) + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + script_path="$project_root/client-cmds/zeam-cmd.sh" + if [ ! -f "$script_path" ]; then + echo "ERROR: Script not found at $script_path (project_root=$project_root, playbook_dir={{ playbook_dir }})" >&2 + exit 1 + fi + grep -E '^node_docker=' "$script_path" | head -1 | grep -oE '[a-zA-Z0-9._-]+/[a-zA-Z0-9._-]+:[a-zA-Z0-9._-]+' | head -1 + register: zeam_docker_image_raw changed_when: false + delegate_to: localhost + run_once: true + +- name: Extract deployment mode from client-cmd.sh + shell: | + # Extract the value from node_setup line + # playbook_dir points to ansible/playbooks, go up two levels to reach project root + project_root="$(cd '{{ playbook_dir }}/../..' && pwd)" + grep -E '^node_setup=' "$project_root/client-cmds/zeam-cmd.sh" | head -1 | sed -E 's/.*node_setup="([^"]+)".*/\1/' + register: zeam_deployment_mode_raw + changed_when: false + delegate_to: localhost + run_once: true + +- name: Set docker image and deployment mode from client-cmd.sh + set_fact: + zeam_docker_image: "{{ zeam_docker_image_raw.stdout | trim | default('blockblaz/zeam:latest') }}" + deployment_mode: "{{ zeam_deployment_mode_raw.stdout | trim | default('docker') }}" + delegate_to: localhost + run_once: true + +- name: Extract node configuration from validator-config.yaml + shell: | + yq eval ".validators[] | select(.name == \"{{ node_name }}\") | .{{ item }}" "{{ hostvars['localhost']['local_genesis_dir_path'] }}/validator-config.yaml" + register: node_config + changed_when: false + delegate_to: localhost loop: - - image - metricsPort + - privkey when: node_name is defined -- name: Set node configuration +- name: Set node metrics port set_fact: - zeam_docker_image: "{{ zeam_node_config.results[0].stdout | trim | default('blockblaz/zeam:latest') }}" - zeam_metrics_port: "{{ zeam_node_config.results[1].stdout | trim }}" - deployment_mode: "{{ deployment_mode | default('docker') }}" - when: zeam_node_config is defined + zeam_metrics_port: "{{ node_config.results[0].stdout }}" + when: node_config is defined - name: Ensure node key file exists stat: diff --git a/client-cmds/ethlambda-cmd.sh b/client-cmds/ethlambda-cmd.sh index eb79253..a71ac71 100644 --- a/client-cmds/ethlambda-cmd.sh +++ b/client-cmds/ethlambda-cmd.sh @@ -2,9 +2,6 @@ #-----------------------ethlambda setup---------------------- -# Docker image (set from deploy-validator-config.yaml, merged from validator-config.yaml + user config) -# ethlambdaImage is exported by spin-node.sh before sourcing this file - binary_path="$scriptDir/../ethlambda/target/release/ethlambda" # Set aggregator flag based on isAggregator value @@ -31,7 +28,7 @@ node_binary="$binary_path \ $checkpoint_sync_flag" # Command when running as docker container -node_docker="$ethlambdaImage \ +node_docker="ghcr.io/lambdaclass/ethlambda:devnet2 \ --custom-network-config-dir /config \ --gossipsub-port $quicPort \ --node-id $item \ diff --git a/client-cmds/grandine-cmd.sh b/client-cmds/grandine-cmd.sh index 8b8f8c0..d1d550a 100644 --- a/client-cmds/grandine-cmd.sh +++ b/client-cmds/grandine-cmd.sh @@ -27,7 +27,7 @@ node_binary="$grandine_bin \ $aggregator_flag \ $checkpoint_sync_flag" -node_docker="$grandineImage \ +node_docker="sifrai/lean:devnet-2 \ --genesis /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --bootnodes /config/nodes.yaml \ diff --git a/client-cmds/lantern-cmd.sh b/client-cmds/lantern-cmd.sh index a33849f..7d22960 100755 --- a/client-cmds/lantern-cmd.sh +++ b/client-cmds/lantern-cmd.sh @@ -1,8 +1,7 @@ #!/bin/bash #-----------------------lantern setup---------------------- -# Docker image (set from deploy-validator-config.yaml or user config via --configFile) -# lanternImage is exported by spin-node.sh before sourcing this file +LANTERN_IMAGE="piertwo/lantern:v0.0.2" devnet_flag="" if [ -n "$devnet" ]; then @@ -27,7 +26,7 @@ node_binary="$scriptDir/lantern/build/lantern_cli \ --genesis-config $configDir/config.yaml \ --validator-registry-path $configDir/validators.yaml \ --genesis-state $configDir/genesis.ssz \ - --validator-config $configDir/deploy-validator-config.yaml \ + --validator-config $configDir/validator-config.yaml \ $devnet_flag \ --nodes-path $configDir/nodes.yaml \ --node-id $item --node-key-path $configDir/$privKeyPath \ @@ -39,11 +38,11 @@ node_binary="$scriptDir/lantern/build/lantern_cli \ $aggregator_flag \ $checkpoint_sync_flag" -node_docker="$lanternImage --data-dir /data \ +node_docker="$LANTERN_IMAGE --data-dir /data \ --genesis-config /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --genesis-state /config/genesis.ssz \ - --validator-config /config/deploy-validator-config.yaml \ + --validator-config /config/validator-config.yaml \ $devnet_flag \ --nodes-path /config/nodes.yaml \ --node-id $item --node-key-path /config/$privKeyPath \ diff --git a/client-cmds/lighthouse-cmd.sh b/client-cmds/lighthouse-cmd.sh index 6181785..030b79c 100644 --- a/client-cmds/lighthouse-cmd.sh +++ b/client-cmds/lighthouse-cmd.sh @@ -18,7 +18,7 @@ fi node_binary="$lighthouse_bin lean_node \ --datadir \"$dataDir/$item\" \ --config \"$configDir/config.yaml\" \ - --validators \"$configDir/deploy-validator-config.yaml\" \ + --validators \"$configDir/validator-config.yaml\" \ --nodes \"$configDir/nodes.yaml\" \ --node-id \"$item\" \ --private-key \"$configDir/$privKeyPath\" \ @@ -30,10 +30,10 @@ node_binary="$lighthouse_bin lean_node \ $aggregator_flag \ $checkpoint_sync_flag" -node_docker="$lighthouseImage lighthouse lean_node \ +node_docker="hopinheimer/lighthouse:latest lighthouse lean_node \ --datadir /data \ --config /config/config.yaml \ - --validators /config/deploy-validator-config.yaml \ + --validators /config/validator-config.yaml \ --nodes /config/nodes.yaml \ --node-id $item \ --private-key /config/$privKeyPath \ diff --git a/client-cmds/qlean-cmd.sh b/client-cmds/qlean-cmd.sh index 57a4bf1..0df9e18 100644 --- a/client-cmds/qlean-cmd.sh +++ b/client-cmds/qlean-cmd.sh @@ -33,7 +33,7 @@ node_binary="$scriptDir/qlean/build/src/executable/qlean \ -ldebug \ -ltrace" -node_docker="$qleanImage \ +node_docker="qdrvm/qlean-mini:devnet-2 \ --genesis /config/config.yaml \ --validator-registry-path /config/validators.yaml \ --validator-keys-manifest /config/hash-sig-keys/validator-keys-manifest.yaml \ diff --git a/client-cmds/ream-cmd.sh b/client-cmds/ream-cmd.sh index 2fd468f..3585f59 100755 --- a/client-cmds/ream-cmd.sh +++ b/client-cmds/ream-cmd.sh @@ -31,7 +31,7 @@ node_binary="$scriptDir/../ream/target/release/ream --data-dir $dataDir/$item \ $aggregator_flag \ $checkpoint_sync_flag" -node_docker="$reamImage --data-dir /data \ +node_docker="ghcr.io/reamlabs/ream:latest-devnet2 --data-dir /data \ lean_node \ --network /config/config.yaml \ --validator-registry-path /config/validators.yaml \ diff --git a/client-cmds/zeam-cmd.sh b/client-cmds/zeam-cmd.sh index 5f75e38..6f693d4 100644 --- a/client-cmds/zeam-cmd.sh +++ b/client-cmds/zeam-cmd.sh @@ -28,7 +28,7 @@ node_binary="$scriptDir/../zig-out/bin/zeam node \ $aggregator_flag \ $checkpoint_sync_flag" -node_docker="--security-opt seccomp=unconfined $zeamImage node \ +node_docker="--security-opt seccomp=unconfined blockblaz/zeam:devnet2 node \ --custom_genesis /config \ --validator_config $validatorConfig \ --data-dir /data \ diff --git a/generate-ansible-inventory.sh b/generate-ansible-inventory.sh index a385d75..e36044e 100755 --- a/generate-ansible-inventory.sh +++ b/generate-ansible-inventory.sh @@ -1,15 +1,15 @@ #!/bin/bash -# Generate Ansible inventory from deploy-validator-config.yaml -# This script reads deploy-validator-config.yaml and generates hosts.yml for Ansible +# Generate Ansible inventory from validator-config.yaml +# This script reads validator-config.yaml and generates hosts.yml for Ansible set -e if [ $# -lt 2 ]; then - echo "Usage: $0 " + echo "Usage: $0 " exit 1 fi -DEPLOY_VALIDATOR_CONFIG="$1" +VALIDATOR_CONFIG="$1" OUTPUT_FILE="$2" # Check if yq is installed @@ -21,8 +21,8 @@ if ! command -v yq &> /dev/null; then fi # Check if validator config exists -if [ ! -f "$DEPLOY_VALIDATOR_CONFIG" ]; then - echo "Error: Deploy config file not found: $DEPLOY_VALIDATOR_CONFIG" +if [ ! -f "$VALIDATOR_CONFIG" ]; then + echo "Error: Validator config file not found: $VALIDATOR_CONFIG" exit 1 fi @@ -34,7 +34,7 @@ mkdir -p "$OUTPUT_DIR" cat > "$OUTPUT_FILE" << 'EOF' --- # Ansible Inventory for Lean Quickstart -# Auto-generated from deploy-validator-config.yaml +# Auto-generated from validator-config.yaml # DO NOT EDIT MANUALLY - This file is auto-generated all: @@ -62,8 +62,8 @@ all: hosts: {} EOF -# Extract node information from deploy-validator-config.yaml -nodes=($(yq eval '.validators[].name' "$DEPLOY_VALIDATOR_CONFIG")) +# Extract node information from validator-config.yaml +nodes=($(yq eval '.validators[].name' "$VALIDATOR_CONFIG")) # Process each node and generate inventory entries for node_name in "${nodes[@]}"; do @@ -73,8 +73,8 @@ for node_name in "${nodes[@]}"; do group_name="${client_type}_nodes" # Extract node-specific information - node_ip=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.ip // \"127.0.0.1\"" "$DEPLOY_VALIDATOR_CONFIG") - node_quic=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.quic // \"9000\"" "$DEPLOY_VALIDATOR_CONFIG") + node_ip=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.ip // \"127.0.0.1\"" "$VALIDATOR_CONFIG") + node_quic=$(yq eval ".validators[] | select(.name == \"$node_name\") | .enrFields.quic // \"9000\"" "$VALIDATOR_CONFIG") # Check if this is a remote deployment (IP is not localhost/127.0.0.1) is_remote=false diff --git a/generate-genesis.sh b/generate-genesis.sh index 5736f80..cd29b23 100755 --- a/generate-genesis.sh +++ b/generate-genesis.sh @@ -22,9 +22,8 @@ Generates: config.yaml, validators.yaml, nodes.yaml, genesis.json, genesis.ssz, Arguments: genesis-directory Path to the genesis directory containing: - - deploy-validator-config.yaml (preferred, merged config with user overrides) - - validator-config.yaml (fallback if deploy-validator-config.yaml not found) - - config must include key: config.activeEpoch (positive integer) + - validator-config.yaml (with node configurations and individual counts) + - validator-config.yaml must include key: config.activeEpoch (positive integer) Options: --mode Deployment mode: 'local' or 'ansible' (default: local) @@ -48,13 +47,13 @@ Generated Files: How It Works: 1. Calculates GENESIS_TIME based on --mode (local: +30s, ansible: +360s) or --offset if provided - 2. Reads individual validator 'count' fields from deploy-validator-config.yaml (or validator-config.yaml as fallback) - 3. Reads config.activeEpoch from the validator config file (required) + 2. Reads individual validator 'count' fields from validator-config.yaml + 3. Reads config.activeEpoch from validator-config.yaml (required) 4. Automatically sums them to calculate total VALIDATOR_COUNT 5. Generates config.yaml from scratch with calculated values including config.activeEpoch 6. Runs PK's genesis generator with correct parameters -Note: config.yaml is a generated file - only edit validator-config.yaml (or user-config.yml for overrides) +Note: config.yaml is a generated file - only edit validator-config.yaml Requirements: - Docker (to run PK's eth-beacon-genesis tool) @@ -84,17 +83,7 @@ fi GENESIS_DIR="$1" CONFIG_FILE="$GENESIS_DIR/config.yaml" - -# Prefer deploy-validator-config.yaml (merged config with user overrides) if available, -# otherwise fall back to validator-config.yaml for standalone usage -DEPLOY_CONFIG="$GENESIS_DIR/deploy-validator-config.yaml" -BASE_CONFIG="$GENESIS_DIR/validator-config.yaml" -if [ -f "$DEPLOY_CONFIG" ]; then - VALIDATOR_CONFIG_FILE="$DEPLOY_CONFIG" -else - echo " deploy-validator-config.yaml not found, falling back to validator-config.yaml" - VALIDATOR_CONFIG_FILE="$BASE_CONFIG" -fi +VALIDATOR_CONFIG_FILE="$GENESIS_DIR/validator-config.yaml" # Parse optional flags SKIP_KEY_GEN="true" @@ -436,7 +425,7 @@ docker run --rm --pull=never \ "$PK_DOCKER_IMAGE" \ leanchain \ --config "/data/genesis/config.yaml" \ - --mass-validators "/data/genesis/$(basename "$VALIDATOR_CONFIG_FILE")" \ + --mass-validators "/data/genesis/validator-config.yaml" \ --state-output "/data/genesis/genesis.ssz" \ --json-output "/data/genesis/genesis.json" \ --nodes-output "/data/genesis/nodes.yaml" \ diff --git a/local-devnet/genesis/validator-config.yaml b/local-devnet/genesis/validator-config.yaml index b9ded01..72d3ad3 100644 --- a/local-devnet/genesis/validator-config.yaml +++ b/local-devnet/genesis/validator-config.yaml @@ -8,7 +8,6 @@ validators: - name: "zeam_0" # node id 7d0904dc6d8d7130e0e68d5d3175d0c3cf470f8725f67bd8320882f5b9753cc0 # peer id 16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f - image: blockblaz/zeam:latest privkey: "bdf953adc161873ba026330c56450453f582e3c4ee6cb713644794bcfdd85fe5" enrFields: # verify /ip4/127.0.0.1/udp/9000/quic-v1/p2p/16Uiu2HAkvi2sxT75Bpq1c7yV2FjnSQJJ432d6jeshbmfdJss1i6f @@ -21,7 +20,6 @@ validators: - name: "ream_0" # node id bc531fc1a99a896acb45603f28a32f81ae607480af46435009de4609370cb7bb # peer id 16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 - image: ghcr.io/reamlabs/ream:latest privkey: "af27950128b49cda7e7bc9fcb7b0270f7a3945aa7543326f3bfdbd57d2a97a32" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -34,7 +32,6 @@ validators: - name: "qlean_0" # TODO: add a third entry in nodes.yaml corresponding to this - image: qdrvm/qlean-mini:latest privkey: "c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9d4b8b6b8c2bbdac5e876b3e9" enrFields: #verify /ip4/127.0.0.1/udp/9001/quic-v1/p2p/16Uiu2HAmPQhkD6Zg5Co2ee8ShshkiY4tDePKFARPpCS2oKSLj1E1 @@ -47,7 +44,6 @@ validators: - name: "lantern_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - image: piertwo/lantern:v0.0.2 privkey: "d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2c3d4e5" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -60,7 +56,6 @@ validators: - name: "lighthouse_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv - image: hopinheimer/lighthouse:latest privkey: "4fd22cf461fbeae4947a3fdaef8d533fc7fd1ef1ce4cd98e993210c18234df3f" # verify /ip4/127.0.0.1/udp/9004/quic-v1/p2p/16Uiu2HAm7TYVs6qvDKnrovd9m4vvRikc4HPXm1WyLumKSe5fHxBv enrFields: @@ -71,7 +66,6 @@ validators: count: 1 - name: "grandine_0" - image: sifrai/lean:unstable privkey: "64a7f5ab53907966374ca23af36392910af682eec82c12e3abbb6c2ccdf39a72" enrFields: ip: "127.0.0.1" @@ -83,7 +77,6 @@ validators: - name: "ethlambda_0" # node id a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3f4a5b6c7d8e9f0a1b2 # peer id 16Uiu2HAmPV5jU62WtmDkCEmfq1jzbBDkGbHNsDN78gJyvmv2TuC5 - image: ghcr.io/lambdaclass/ethlambda:latest privkey: "299550529a79bc2dce003747c52fb0639465c893e00b0440ac66144d625e066a" # verify /ip4/127.0.0.1/udp/9007/quic-v1/p2p/16Uiu2HAmPV5jU62WtmDkCEmfq1jzbBDkGbHNsDN78gJyvmv2TuC5 enrFields: diff --git a/parse-env.sh b/parse-env.sh index 2b7d713..c3b1028 100755 --- a/parse-env.sh +++ b/parse-env.sh @@ -76,11 +76,6 @@ while [[ $# -gt 0 ]]; do shift # past argument shift # past value ;; - --configFile) - configFile="$2" - shift # past argument - shift # past value - ;; --stop) stopNodes=true shift @@ -134,7 +129,6 @@ echo "generateGenesis = $generateGenesis" echo "cleanData = $cleanData" echo "popupTerminal = $popupTerminal" echo "dockerTag = ${dockerTag:-latest}" -echo "configFile = ${configFile:-none}" echo "enableMetrics = $enableMetrics" echo "coreDumps = ${coreDumps:-disabled}" echo "checkpointSyncUrl = ${checkpointSyncUrl:-}" diff --git a/parse-vc.sh b/parse-vc.sh index 0cbb43d..629f4e1 100644 --- a/parse-vc.sh +++ b/parse-vc.sh @@ -1,14 +1,12 @@ #!/bin/bash -# parse deploy-validator-config.yaml to load values related to the $item -# needed for ream and qlean (or any other client), zeam picks directly from config +# parse validator-config to load values related to the $item +# needed for ream and qlean (or any other client), zeam picks directly from validator-config # 1. load quic port and export it in $quicPort # 2. private key and dump it into a file $client.key and export it in $privKeyPath # 3. devnet and export it in $devnet -# 4. docker image (already merged in deploy-validator-config.yaml) # $item, $configDir (genesis dir) is available here -# Note: deploy-validator-config.yaml already has user overrides merged (from merge-config.sh) # Check if yq is installed if ! command -v yq &> /dev/null; then @@ -18,38 +16,37 @@ if ! command -v yq &> /dev/null; then exit 1 fi -# Use deploy-validator-config.yaml (has user overrides merged) -deploy_validator_config_file="$configDir/deploy-validator-config.yaml" -if [ ! -f "$deploy_validator_config_file" ]; then - echo "Error: deploy-validator-config.yaml not found at $deploy_validator_config_file" - echo "This file should have been created by merge-config.sh" +# Validate that validator config file exists +validator_config_file="$configDir/validator-config.yaml" +if [ ! -f "$validator_config_file" ]; then + echo "Error: Validator config file not found at $validator_config_file" exit 1 fi # Automatically extract QUIC port using yq -quicPort=$(yq eval ".validators[] | select(.name == \"$item\") | .enrFields.quic" "$deploy_validator_config_file") +quicPort=$(yq eval ".validators[] | select(.name == \"$item\") | .enrFields.quic" "$validator_config_file") # Validate that we found a QUIC port for this node if [ -z "$quicPort" ] || [ "$quicPort" == "null" ]; then - echo "Error: No QUIC port found for node '$item' in $deploy_validator_config_file" + echo "Error: No QUIC port found for node '$item' in $validator_config_file" echo "Available nodes:" - yq eval '.validators[].name' "$deploy_validator_config_file" + yq eval '.validators[].name' "$validator_config_file" exit 1 fi # Automatically extract metrics port using yq -metricsPort=$(yq eval ".validators[] | select(.name == \"$item\") | .metricsPort" "$deploy_validator_config_file") +metricsPort=$(yq eval ".validators[] | select(.name == \"$item\") | .metricsPort" "$validator_config_file") # Validate that we found a metrics port for this node if [ -z "$metricsPort" ] || [ "$metricsPort" == "null" ]; then - echo "Error: No metrics port found for node '$item' in $deploy_validator_config_file" + echo "Error: No metrics port found for node '$item' in $validator_config_file" echo "Available nodes:" - yq eval '.validators[].name' "$deploy_validator_config_file" + yq eval '.validators[].name' "$validator_config_file" exit 1 fi # Automatically extract devnet using yq (optional - only ream uses it) -devnet=$(yq eval ".validators[] | select(.name == \"$item\") | .devnet" "$deploy_validator_config_file") +devnet=$(yq eval ".validators[] | select(.name == \"$item\") | .devnet" "$validator_config_file") if [ -z "$devnet" ] || [ "$devnet" == "null" ]; then devnet="" fi @@ -61,11 +58,11 @@ if [ -z "$isAggregator" ] || [ "$isAggregator" == "null" ]; then fi # Automatically extract private key using yq -privKey=$(yq eval ".validators[] | select(.name == \"$item\") | .privkey" "$deploy_validator_config_file") +privKey=$(yq eval ".validators[] | select(.name == \"$item\") | .privkey" "$validator_config_file") # Validate that we found a private key for this node if [ -z "$privKey" ] || [ "$privKey" == "null" ]; then - echo "Error: No private key found for node '$item' in $deploy_validator_config_file" + echo "Error: No private key found for node '$item' in $validator_config_file" exit 1 fi @@ -74,46 +71,36 @@ privKeyPath="$item.key" echo "$privKey" > "$configDir/$privKeyPath" # Extract hash-sig key configuration from top-level config -keyType=$(yq eval ".config.keyType" "$deploy_validator_config_file") -hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name == \"$item\") | .key" "$deploy_validator_config_file") +keyType=$(yq eval ".config.keyType" "$validator_config_file") +hashSigKeyIndex=$(yq eval ".validators | to_entries | .[] | select(.value.name == \"$item\") | .key" "$validator_config_file") # Load hash-sig keys if configured if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$hashSigKeyIndex" ]; then # Set hash-sig key paths hashSigPkPath="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_pk.json" hashSigSkPath="$configDir/hash-sig-keys/validator_${hashSigKeyIndex}_sk.json" - + # Validate that hash-sig keys exist if [ ! -f "$hashSigPkPath" ]; then echo "Warning: Hash-sig public key not found at $hashSigPkPath" echo "Run genesis generator to create hash-sig keys: ./generate-genesis.sh $configDir" fi - + if [ ! -f "$hashSigSkPath" ]; then echo "Warning: Hash-sig secret key not found at $hashSigSkPath" echo "Run genesis generator to create hash-sig keys: ./generate-genesis.sh $configDir" fi - + # Export hash-sig key paths for client use export HASH_SIG_PK_PATH="$hashSigPkPath" export HASH_SIG_SK_PATH="$hashSigSkPath" export HASH_SIG_KEY_INDEX="$hashSigKeyIndex" -fi - -# Load docker image for this node (already merged in deploy-validator-config.yaml) -docker_image=$(yq eval ".validators[] | select(.name == \"$item\") | .image" "$deploy_validator_config_file" 2>/dev/null) -if [ -z "$docker_image" ] || [ "$docker_image" == "null" ]; then - echo "Warning: No docker image found for $item" - docker_image="" -fi - -echo "Node: $item" -echo "Docker Image: ${docker_image:-}" -echo "QUIC Port: $quicPort" -echo "Metrics Port: $metricsPort" -echo "Devnet: ${devnet:-}" -echo "Private Key File: $privKeyPath" -if [ "$keyType" == "hash-sig" ] && [ "$hashSigKeyIndex" != "null" ] && [ -n "$hashSigKeyIndex" ]; then + + echo "Node: $item" + echo "QUIC Port: $quicPort" + echo "Metrics Port: $metricsPort" + echo "Devnet: ${devnet:-}" + echo "Private Key File: $privKeyPath" echo "Key Type: $keyType" echo "Hash-Sig Key Index: $hashSigKeyIndex" echo "Hash-Sig Public Key: $hashSigPkPath" diff --git a/run-ansible.sh b/run-ansible.sh index e38a6d1..1d82736 100755 --- a/run-ansible.sh +++ b/run-ansible.sh @@ -22,7 +22,7 @@ configDir="$1" node="$2" cleanData="$3" validatorConfig="$4" -deploy_validator_config_file="$5" # deploy-validator-config.yaml (with user overrides merged) +validator_config_file="$5" sshKeyFile="$6" useRoot="$7" # Flag to use root user (defaults to current user) action="$8" # Action: "stop" to stop nodes, otherwise deploy @@ -37,7 +37,7 @@ else fi # Validate required arguments -if [ -z "$configDir" ] || [ -z "$deploy_validator_config_file" ]; then +if [ -z "$configDir" ] || [ -z "$validator_config_file" ]; then echo "Error: Missing required arguments" echo "Usage: $0 [sshKeyFile] [useRoot] [action] [coreDumps]" exit 1 @@ -51,10 +51,10 @@ echo "SSH user for remote connections: $sshUser" ANSIBLE_DIR="$scriptDir/ansible" INVENTORY_FILE="$ANSIBLE_DIR/inventory/hosts.yml" -# Generate inventory if it doesn't exist or if deploy-validator-config.yaml is newer -if [ ! -f "$INVENTORY_FILE" ] || [ "$deploy_validator_config_file" -nt "$INVENTORY_FILE" ]; then - echo "Generating Ansible inventory from deploy-validator-config.yaml..." - "$scriptDir/generate-ansible-inventory.sh" "$deploy_validator_config_file" "$INVENTORY_FILE" +# Generate inventory if it doesn't exist or if validator config is newer +if [ ! -f "$INVENTORY_FILE" ] || [ "$validator_config_file" -nt "$INVENTORY_FILE" ]; then + echo "Generating Ansible inventory from validator-config.yaml..." + "$scriptDir/generate-ansible-inventory.sh" "$validator_config_file" "$INVENTORY_FILE" fi # Update inventory with SSH key file and user if provided @@ -158,31 +158,6 @@ if [ $EXIT_CODE -eq 0 ]; then if [ "$action" == "stop" ]; then echo "✅ Ansible stop operation completed successfully!" else - # Display summary of deployed nodes - echo "" - echo "==================================================" - echo "Deployed Nodes Summary:" - echo "==================================================" - printf "%-15s | %-10s | %s\n" "Node" "Mode" "Docker Image" - echo "--------------------------------------------------" - - # Get node names for summary - if [[ "$node" == "all" ]]; then - # Expand "all" to actual node names from config - deployed_nodes=($(yq eval '.validators[].name' "$deploy_validator_config_file" 2>/dev/null)) - elif [[ "$node" == *","* ]]; then - IFS=',' read -r -a deployed_nodes <<< "$node" - else - IFS=' ' read -r -a deployed_nodes <<< "$node" - fi - - for node_name in "${deployed_nodes[@]}"; do - # Get image from deploy-validator-config.yaml (overrides already merged) - docker_image=$(yq eval ".validators[] | select(.name == \"$node_name\") | .image" "$deploy_validator_config_file" 2>/dev/null) - printf "%-15s | %-10s | %s\n" "$node_name" "docker" "${docker_image:-unknown}" - done - echo "==================================================" - echo "" echo "✅ Ansible deployment completed successfully!" fi else diff --git a/scripts/merge-config.sh b/scripts/merge-config.sh deleted file mode 100755 index 3daaaed..0000000 --- a/scripts/merge-config.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash -# Merge user-config.yml overrides into validator-config.yaml -# Creates deploy-validator-config.yaml in the same directory as the base config -# -# Usage: ./merge-config.sh [user-config.yml] -# -# If user-config.yml is not provided or doesn't exist, the base config is copied as-is. - -set -e - -base_config="$1" # validator-config.yaml (required) -user_config="$2" # user-config.yml (optional) - -if [ -z "$base_config" ]; then - echo "Usage: $0 [user-config.yml]" - exit 1 -fi - -if [ ! -f "$base_config" ]; then - echo "Error: Base config not found: $base_config" - exit 1 -fi - -# Check if yq is installed -if ! command -v yq &> /dev/null; then - echo "Error: yq is required but not installed." - exit 1 -fi - -# Output file is in the same directory as base config -config_dir=$(dirname "$base_config") -output_config="$config_dir/deploy-validator-config.yaml" - -# Start with base config -cp "$base_config" "$output_config" - -# If no user config provided or doesn't exist, we're done -if [ -z "$user_config" ] || [ ! -f "$user_config" ]; then - echo "✓ Created deploy-validator-config.yaml (no user overrides)" - exit 0 -fi - -echo "Merging user config overrides from $user_config..." - -# Get all node names from user config -node_names=$(yq eval '.validators[].name' "$user_config" 2>/dev/null) - -if [ -z "$node_names" ]; then - echo "✓ Created deploy-validator-config.yaml (no validators in user config)" - exit 0 -fi - -override_count=0 - -for node in $node_names; do - # Check if this node exists in base config - node_exists=$(yq eval ".validators[] | select(.name == \"$node\") | .name" "$output_config" 2>/dev/null) - if [ -z "$node_exists" ] || [ "$node_exists" == "null" ]; then - echo " ⚠ Skipping $node: not found in validator-config.yaml" - continue - fi - - # Merge image if specified - image=$(yq eval ".validators[] | select(.name == \"$node\") | .image // \"\"" "$user_config" 2>/dev/null) - if [ -n "$image" ] && [ "$image" != "null" ] && [ "$image" != "" ]; then - yq eval -i "(.validators[] | select(.name == \"$node\")).image = \"$image\"" "$output_config" - echo " ✓ $node: image = $image" - ((override_count++)) - fi - - # Add more mergeable fields here as needed -done - -echo "✓ Created deploy-validator-config.yaml ($override_count override(s) applied)" diff --git a/spin-node.sh b/spin-node.sh index b6769dc..6e28a10 100755 --- a/spin-node.sh +++ b/spin-node.sh @@ -68,24 +68,17 @@ if [ "$deployment_mode" == "ansible" ] && ([ "$validatorConfig" == "genesis_boot echo "Using Ansible deployment: configDir=$configDir, validator config=$validator_config_file" fi -# 1. Create deploy-validator-config.yaml (merges user config overrides if provided) -# This must run BEFORE genesis generation so that generate-genesis.sh can read the merged config -echo "" -echo "Creating deploy-validator-config.yaml..." -"$scriptDir/scripts/merge-config.sh" "$validator_config_file" "$configFile" -deploy_validator_config_file="$configDir/deploy-validator-config.yaml" - -# 2. Setup genesis params and run genesis generator +#1. setup genesis params and run genesis generator source "$(dirname $0)/set-up.sh" # ✅ Genesis generator implemented using PK's eth-beacon-genesis tool # Generates: validators.yaml, nodes.yaml, genesis.json, genesis.ssz, and .key files -# 3. collect the nodes that the user has asked us to spin and perform setup +# 2. collect the nodes that the user has asked us to spin and perform setup -# Load nodes from deploy-validator-config.yaml -if [ -f "$deploy_validator_config_file" ]; then - # Use yq to extract node names from deploy-validator-config.yaml - nodes=($(yq eval '.validators[].name' "$deploy_validator_config_file")) +# Load nodes from validator config file +if [ -f "$validator_config_file" ]; then + # Use yq to extract node names from validator config + nodes=($(yq eval '.validators[].name' "$validator_config_file")) # Validate that we found nodes if [ ${#nodes[@]} -eq 0 ]; then @@ -93,8 +86,10 @@ if [ -f "$deploy_validator_config_file" ]; then exit 1 fi else - echo "Error: deploy-validator-config.yaml not found at $deploy_validator_config_file" - echo "This file should have been created by merge-config.sh" + echo "Error: Validator config file not found at $validator_config_file" + if [ "$deployment_mode" == "ansible" ]; then + echo "Please create ansible-devnet/genesis/validator-config.yaml for Ansible deployments" + fi nodes=() exit 1 fi @@ -212,7 +207,7 @@ if [ "$deployment_mode" == "ansible" ]; then fi exit 0 fi - + # Call separate Ansible execution script # If Ansible deployment fails, exit immediately (don't fall through to local deployment) if ! "$scriptDir/run-ansible.sh" "$configDir" "$ansible_node_arg" "$cleanData" "$validatorConfig" "$validator_config_file" "$sshKeyFile" "$useRoot" "" "$coreDumps" "$ansible_skip_genesis"; then @@ -227,12 +222,12 @@ fi # Handle stop action for local deployment if [ -n "$stopNodes" ] && [ "$stopNodes" == "true" ]; then echo "Stopping local nodes..." - - # Load nodes from deploy-validator-config.yaml - if [ -f "$deploy_validator_config_file" ]; then - nodes=($(yq eval '.validators[].name' "$deploy_validator_config_file")) + + # Load nodes from validator config file + if [ -f "$validator_config_file" ]; then + nodes=($(yq eval '.validators[].name' "$validator_config_file")) else - echo "Error: deploy-validator-config.yaml not found at $deploy_validator_config_file" + echo "Error: Validator config file not found at $validator_config_file" exit 1 fi @@ -301,7 +296,6 @@ elif [[ "$OSTYPE" == "linux"* ]]; then done fi spinned_pids=() - for item in "${spin_nodes[@]}"; do echo -e "\n\nspining $item: client=$client (mode=$node_setup)" printf '%*s' $(tput cols) | tr ' ' '-' @@ -329,7 +323,7 @@ for item in "${spin_nodes[@]}"; do eval "$cmd" fi - # parse deploy-validator-config.yaml for $item to load args values (including docker_image) + # parse validator-config.yaml for $item to load args values source parse-vc.sh # export checkpoint_sync_url for client-cmd scripts when restarting with checkpoint sync @@ -343,37 +337,38 @@ for item in "${spin_nodes[@]}"; do IFS='_' read -r -a elements <<< "$item" client="${elements[0]}" - # Export the image variable for this client (will be used by client-cmd.sh) - # docker_image is set by parse-vc.sh - case "$client" in - zeam) - export zeamImage="$docker_image" - ;; - ream) - export reamImage="$docker_image" - ;; - qlean) - export qleanImage="$docker_image" - ;; - lantern) - export lanternImage="$docker_image" - ;; - lighthouse) - export lighthouseImage="$docker_image" - ;; - grandine) - export grandineImage="$docker_image" - ;; - ethlambda) - export ethlambdaImage="$docker_image" - ;; - esac - # get client specific cmd and its mode (docker, binary) sourceCmd="source client-cmds/$client-cmd.sh" echo "$sourceCmd" eval $sourceCmd + # Apply user-config.yml overrides (run_mode, docker_image) if file exists + user_config="$scriptDir/user-config.yml" + if [ -f "$user_config" ]; then + # Override run_mode if specified for this node (check first since it affects docker_image) + user_run_mode=$(yq eval ".$item.run_mode // \"\"" "$user_config" 2>/dev/null) + if [ -n "$user_run_mode" ] && [ "$user_run_mode" != "null" ]; then + if [ "$user_run_mode" == "docker" ] || [ "$user_run_mode" == "binary" ]; then + node_setup="$user_run_mode" + echo " user-config.yml: overriding run_mode for $item: $user_run_mode" + else + echo " user-config.yml: ignoring unknown run_mode '$user_run_mode' for $item (expected: docker or binary)" + fi + fi + + # Override docker image if specified for this node (only applies in docker mode) + if [ "$node_setup" == "docker" ]; then + user_docker_image=$(yq eval ".$item.docker_image // \"\"" "$user_config" 2>/dev/null) + if [ -n "$user_docker_image" ] && [ "$user_docker_image" != "null" ]; then + # Replace the image in node_docker string (first word matching image:tag pattern) + original_image=$(echo "$node_docker" | grep -oE '[a-zA-Z0-9._/-]+:[a-zA-Z0-9._-]+' | head -1) + if [ -n "$original_image" ]; then + node_docker="${node_docker/$original_image/$user_docker_image}" + echo " user-config.yml: overriding docker_image for $item: $user_docker_image" + fi + fi + fi + fi # spin nodes if [ "$node_setup" == "binary" ] @@ -388,25 +383,12 @@ for item in "${spin_nodes[@]}"; do else # Extract image name from node_docker (find word containing ':' which is the image:tag) docker_image=$(echo "$node_docker" | grep -oE '[^ ]+:[^ ]+' | head -1) - # Pull image first + # Pull image first if [ -n "$dockerWithSudo" ]; then sudo docker pull "$docker_image" || true else docker pull "$docker_image" || true fi - - # Check if the image exists locally (either pulled successfully or was cached) - if [ -n "$dockerWithSudo" ]; then - image_exists=$(sudo docker image inspect "$docker_image" > /dev/null 2>&1 && echo "yes" || echo "no") - else - image_exists=$(docker image inspect "$docker_image" > /dev/null 2>&1 && echo "yes" || echo "no") - fi - - if [ "$image_exists" == "no" ]; then - echo "⚠️ Skipping $item: Docker image '$docker_image' does not exist and could not be pulled" - continue - fi - execCmd="docker run --rm --pull=never" if [ -n "$dockerWithSudo" ] then @@ -452,8 +434,8 @@ if [ -n "$enableMetrics" ] && [ "$enableMetrics" == "true" ]; then metricsDir="$scriptDir/metrics" - # Generate prometheus.yml from deploy-validator-config.yaml - "$scriptDir/generate-prometheus-config.sh" "$deploy_validator_config_file" "$metricsDir/prometheus" + # Generate prometheus.yml from validator-config.yaml + "$scriptDir/generate-prometheus-config.sh" "$validator_config_file" "$metricsDir/prometheus" # Pull and start metrics containers if [ -n "$dockerWithSudo" ]; then @@ -472,20 +454,6 @@ fi container_names="${spin_nodes[*]}" process_ids="${spinned_pids[*]}" -# Display summary table (read directly from deploy-validator-config.yaml like ansible does) -echo "" -echo "==================================================" -echo "Deployed Nodes Summary:" -echo "==================================================" -printf "%-15s | %-10s | %s\n" "Node" "Mode" "Docker Image" -echo "--------------------------------------------------" -for node in "${spin_nodes[@]}"; do - image=$(yq eval ".validators[] | select(.name == \"$node\") | .image" "$deploy_validator_config_file" 2>/dev/null) - printf "%-15s | %-10s | %s\n" "$node" "docker" "$image" -done -echo "==================================================" -echo "" - cleanup() { echo -e "\n\ncleaning up" printf '%*s' $(tput cols) | tr ' ' '-' diff --git a/user-config.yml.example b/user-config.yml.example index c01533a..05adf5e 100644 --- a/user-config.yml.example +++ b/user-config.yml.example @@ -1,31 +1,14 @@ -# User Configuration Example -# Copy this file to user-config.yml or .yml and customize -# Usage: ./spin-node.sh --configFile .yml +# User configuration overrides (auto-detected from project root) +# Copy this file to user-config.yml and customize. +# Only specify nodes you want to override — others use defaults from client-cmd scripts. # -# Only specify validators and fields you want to override - others will use defaults -# from validator-config.yaml in your network directory. -# -# Overrides are merged into deploy-validator-config.yaml before deployment. - -validators: - - name: "zeam_0" - image: blockblaz/zeam:custom-tag +# Fields: +# run_mode: "docker" or "binary" (default: whatever client-cmd.sh sets) +# docker_image: Override the Docker image for this node - - name: "ream_0" - image: ghcr.io/reamlabs/ream:my-branch - - # Add more validators as needed: - # - name: "qlean_0" - # image: qdrvm/qlean-mini:custom-tag - # - # - name: "lantern_0" - # image: piertwo/lantern:custom-tag - # - # - name: "lighthouse_0" - # image: hopinheimer/lighthouse:custom-tag - # - # - name: "grandine_0" - # image: sifrai/lean:custom-tag - # - # - name: "ethlambda_0" - # image: ghcr.io/lambdaclass/ethlambda:custom-tag +# zeam_0: +# run_mode: docker +# docker_image: blockblaz/zeam:custom-tag +# +# ream_0: +# run_mode: binary