From eca6a9214bf64596b0e7acd3ee5515617fed025f Mon Sep 17 00:00:00 2001 From: yorickdowne <71337066+yorickdowne@users.noreply.github.com> Date: Tue, 29 Oct 2024 05:32:31 -0400 Subject: [PATCH] MEV_RELAYS can now be multi-line (#1968) --- default.env | 6 +- ethd | 379 ++++++++++++++++++++++++++++++++++------------------ 2 files changed, 254 insertions(+), 131 deletions(-) diff --git a/default.env b/default.env index 8e3a7b71..592e659f 100644 --- a/default.env +++ b/default.env @@ -8,7 +8,11 @@ FEE_RECIPIENT= # If "true" and used with a CL, it also requires :mev-boost.yml in COMPOSE_FILE MEV_BOOST=false # For relay information, please see https://ethstaker.cc/mev-relay-list/ -MEV_RELAYS=https://0xafa4c6985aa049fb79dd37010438cfebeb0f2bd42b115b89dd678dab0670c1de38da0c4e9138c9290a398ecd9a0b3110@boost-relay-holesky.flashbots.net,https://0xaa58208899c6105603b74396734a6263cc7d947f444f396a90f7b7d3e65d102aec7e5e5291b27e08d02c50a050825c2f@holesky.titanrelay.xyz,https://0x821f2a65afb70e7f2e820a925a9b4c80a159620582c1766b1b09729fec178b11ea22abb3a51f07b288be815a1a2ff516@bloxroute.holesky.blxrbdn.com +MEV_RELAYS=" +https://0xafa4c6985aa049fb79dd37010438cfebeb0f2bd42b115b89dd678dab0670c1de38da0c4e9138c9290a398ecd9a0b3110@boost-relay-holesky.flashbots.net, +https://0xaa58208899c6105603b74396734a6263cc7d947f444f396a90f7b7d3e65d102aec7e5e5291b27e08d02c50a050825c2f@holesky.titanrelay.xyz, +https://0x821f2a65afb70e7f2e820a925a9b4c80a159620582c1766b1b09729fec178b11ea22abb3a51f07b288be815a1a2ff516@bloxroute.holesky.blxrbdn.com +" # Set a minimum MEV bid (e.g. 0.05), used by mev-boost.yml. If empty, no minimum is used. MEV_MIN_BID= # Graffiti to use for validator diff --git a/ethd b/ethd index 06952096..b779b714 100755 --- a/ethd +++ b/ethd @@ -469,11 +469,11 @@ __check_disk_space() { __get_docker_free_space __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") __var="AUTOPRUNE_NM" - __auto_prune=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __auto_prune=$(__get_value_from_env "${__var}" "${__env_file}") __var="NETWORK" - NETWORK=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + NETWORK=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then __min_free=314572800 @@ -529,7 +529,7 @@ Full\". Free space:" __source_build() { # Check whether there's a source-built client and if so, force it with --no-cache __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") case "${__value}" in *deposit-cli.yml* ) @@ -539,7 +539,7 @@ __source_build() { case "${__value}" in *mev-boost.yml* ) __var="MEV_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache mev-boost fi @@ -548,42 +548,42 @@ __source_build() { case "${__value}" in *reth.yml* ) __var="RETH_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache execution fi ;; *geth.yml* ) __var="GETH_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache execution fi ;; *besu.yml* ) __var="BESU_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache execution fi ;; *nethermind.yml* ) __var="NM_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache execution fi ;; *erigon.yml* ) __var="ERIGON_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache execution fi ;; *nimbus-el.yml* ) __var="NIMEL_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache execution fi @@ -592,42 +592,42 @@ __source_build() { case "${__value}" in *lighthouse.yml* | *lighthouse-cl-only.yml* ) __var="LH_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache consensus fi ;; *teku.yml* | *teku-allin1.yml* | *teku-cl-only.yml* ) __var="TEKU_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache consensus fi ;; *lodestar.yml* | *lodestar-cl-only.yml* ) __var="LS_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache consensus fi ;; *nimbus.yml* | *nimbus-allin1.yml* | *nimbus-cl-only.yml* ) __var="NIM_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache consensus fi ;; *prysm.yml* | *prysm-cl-only.yml* ) __var="PRYSM_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache consensus fi ;; *grandine.yml* | *grandine-allin1.yml* | *grandine-cl-only.yml* ) __var="GRANDINE_DOCKERFILE" - __build=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __build=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${__build}" = "Dockerfile.source" ]; then __docompose build --pull --no-cache consensus fi @@ -687,7 +687,7 @@ __ssv_switch() { echo "Backup copy blox-ssv-config.yaml.bak created" echo "Making changes to ssv-config/config.yaml" __var="NETWORK" - NETWORK=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + NETWORK=$(__get_value_from_env "${__var}" "${__env_file}") sed -i'.original' 's/blox-ssv2.yml/ssv.yml/' "${__env_file}".source if ! grep -q "LogFilePath:" ssv-config/config.yaml; then # macOS-isms: Newline for sed add @@ -727,7 +727,7 @@ MetricsAPIPort: 15000 __delete_reth() { # Check for Reth __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # I do mean to match literally # shellcheck disable=SC2076 if [[ ! "${__value}" =~ "reth.yml" ]]; then @@ -736,7 +736,7 @@ __delete_reth() { # Check Reth version, only continue if not on alpha __var="RETH_DOCKER_TAG" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # I do mean to match literally # shellcheck disable=SC2076 if [[ "${__value}" =~ "alpha" ]]; then @@ -785,7 +785,7 @@ __delete_reth() { __delete_erigon() { # Check for Erigon __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # I do mean to match literally # shellcheck disable=SC2076 if [[ ! "${__value}" =~ "erigon.yml" ]]; then @@ -794,9 +794,9 @@ __delete_erigon() { # Check Erigon version, only continue if v3 __var="ERIGON_DOCKER_TAG" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") __var="ERIGON_DOCKER_REPO" - __repo=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __repo=$(__get_value_from_env "${__var}" "${__env_file}") # I do mean to match literally # shellcheck disable=SC2076 if [[ ! ("${__value}" =~ "v3" || ( "${__value}" = "latest" && "${__repo}" =~ "thorax" ) || "${__value}" = "main-latest") ]]; then @@ -836,7 +836,7 @@ __delete_erigon() { __upgrade_postgres() { # Check for web3signer __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # I do mean to match literally # shellcheck disable=SC2076 if [[ ! "${__value}" =~ "web3signer.yml" ]]; then @@ -947,7 +947,7 @@ __upgrade_postgres() { # This gets used, but shellcheck doesn't recognize that # shellcheck disable=SC2034 PG_DOCKER_TAG=${__target_pg}-bookworm # To bookworm to avoid collation errors - also a faster PostgreSQL - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" echo "Web3signer has been stopped. You'll need to run \"$__me up\" to start it again." echo echo "A copy of your old slashing protection database is in the Docker volume ${__backup_vol}." @@ -957,10 +957,10 @@ __upgrade_postgres() { __lookup_cf_zone() { # Migrates traefik-cf setup to use Zone ID - __compose_ymls=$(sed -n -e "s/^COMPOSE_FILE=\(.*\)/\1/p" "${__env_file}.source" || true) - __dns_token=$(sed -n -e "s/^CF_DNS_API_TOKEN=\(.*\)/\1/p" "${__env_file}.source" || true) - __zone_token=$(sed -n -e "s/^CF_ZONE_API_TOKEN=\(.*\)/\1/p" "${__env_file}.source" || true) - __domain=$(sed -n -e "s/^DOMAIN=\(.*\)/\1/p" "${__env_file}.source" || true) + __compose_ymls=$(__get_value_from_env "COMPOSE_FILE" "${__env_file}.source") + __dns_token=$(__get_value_from_env "CF_DNS_API_TOKEN" "${__env_file}.source") + __zone_token=$(__get_value_from_env "CF_ZONE_API_TOKEN" "${__env_file}.source") + __domain=$(__get_value_from_env "DOMAIN" "${__env_file}.source") if [[ ! $__compose_ymls =~ traefik-cf.yml ]]; then __value="" return @@ -1002,7 +1002,7 @@ __enable_v6() { fi __var="IPV6" - IPV6=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + IPV6=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${IPV6}" = "true" ]; then return fi @@ -1021,12 +1021,140 @@ __enable_v6() { if [ "${__v6_works}" = "true" ]; then echo "Enabling IPv4/6 dual-stack for your Eth Docker setup" IPV6="true" - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __enabled_v6=1 fi } +__get_value_from_env() { + local __var_name="$1" + local __env_file="$2" + local __value + + __value=$(awk -v var="$__var_name" ' + BEGIN { __found = 0; __value = "" } + + # Skip empty lines and comments + /^#|^\s*$/ { + next + } + + # Match single-line unquoted value + $0 ~ "^[ \t]*"var"=[^\"].*$" { + gsub("^[ \t]*"var"=", "") + gsub(/^[ \t]*|[ \t]*$/, "", $0) + __value = $0 + __found = 1 + exit + } + + # Match a quoted single-line value + $0 ~ "^[ \t]*"var"=\"[^\"]*\"[ \t]*$" { + gsub("^[ \t]*"var"=\"", "") + gsub(/\"[ \t]*$/, "", $0) + __value = "\"" $0 "\"" + __found = 1 + exit + } + + # Match the start of a multi-line value (with opening quote) + $0 ~ "^[ \t]*"var"=\"[^\"]*$" { + gsub("^[ \t]*"var"=\"", "") + __value = "\"" $0 "\n" + __found = 1 + next + } + + # Continue collecting lines for a multi-line value + __found && !/\"[ \t]*$/ { + __value = __value $0 "\n" + next + } + + # End of a multi-line value (with closing quote) + __found && /\"[ \t]*$/ { + gsub(/[ \t]*\"[ \t]*$/, "") + __value = __value $0 "\"" + __found = 1 + exit + } + + END { + if (__found) { + # Print the value as is, including quotes for multi-line + print __value + } + } + ' "$__env_file") + + printf "%s" "$__value" +} + + +__update_value_in_env() { +# Call as __update_value_in_env "$__var" "$__value" "$__env_file" + local __var_name="$1" + local __new_value="$2" + local __env_file="$3" + + # Escape backslashes for safety + local __escaped_value + __escaped_value=$(printf '%s' "${__new_value}" | sed 's/\\/\\\\/g') + + # Check if the variable already exists in the .env file + if grep -q "^[ \t]*${__var_name}=" "${__env_file}"; then + # Variable exists, update it + awk -v var="$__var_name" -v new_value="$__escaped_value" ' + BEGIN { in_block = 0; multi_line = 0 } + + # Match the line that starts with the variable name + $0 ~ "^[ \t]*" var "=" { + # If the value starts with a quote, it may be a multi-line + if ($0 ~ "^[ \t]*" var "=\"") { + # Start of multi-line value + multi_line = 1 + # Print the variable name with the new value, replacing & safely + gsub(/&/, "\\&", new_value) + print var "=" new_value + } else { + # Single-line value + gsub(/&/, "\\&", new_value) + print var "=" new_value + } + # Set the flag to indicate we are processing the target variable block + in_block = 1 + next + } + + # If we encounter a new variable definition, stop skipping lines + /^[A-Z_][A-Z0-9_]*=/ && in_block { + in_block = 0 + multi_line = 0 + } + + # Continue to skip lines in a multi-line block if multi_line is true + multi_line && !/\"[ \t]*$/ { + next + } + + # If we reach the end of a multi-line value, reset flags + multi_line && /\"[ \t]*$/ { + in_block = 0 + multi_line = 0 + next + } + + # Print all lines if not in the target variable block + { print } + ' "$__env_file" > "${__env_file}.tmp" && mv "${__env_file}.tmp" "$__env_file" + else + # Variable does not exist, append it + printf "%s=%s\n" "$__var_name" "$__escaped_value" >> "$__env_file" + fi +} + + __env_migrate() { if [ ! -f "${__env_file}" ]; then return 0 @@ -1063,15 +1191,15 @@ __env_migrate() { # Always make sure we have a SIREN password __var="SIREN_PASSWORD" - SIREN_PASSWORD=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + SIREN_PASSWORD=$(__get_value_from_env "${__var}" "${__env_file}") if [ -z "${SIREN_PASSWORD}" ]; then SIREN_PASSWORD=$(head -c 8 /dev/urandom | od -A n -t u8 | tr -d '[:space:]' | sha256sum | head -c 32) - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" fi __var=ENV_VERSION - __target_ver=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "default.env" || true) - __source_ver=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __target_ver=$(__get_value_from_env "${__var}" "default.env") + __source_ver=$(__get_value_from_env "${__var}" "${__env_file}") # Aggressive prune to work around Docker grabbing old clients. Here so it doesn't get called during config if [[ "${__source_ver}" -lt "9" ]]; then __dodocker system prune --force -a @@ -1093,7 +1221,7 @@ __env_migrate() { ${__as_owner} cp default.env "${__env_file}" __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}.source" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}.source") # Literal match intended # shellcheck disable=SC2076 if [[ "${__value}" =~ "blox-ssv2.yml" ]]; then @@ -1102,7 +1230,7 @@ __env_migrate() { # Migrate over user settings for __var in "${__all_vars[@]}"; do - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}.source" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}.source") if [ -n "${__value}" ] || [ "${__var}" = "GRAFFITI" ] || [ "${__var}" = "MEV_RELAYS" ] \ || [ "${__var}" = "ETH_DOCKER_TAG" ] || [ "${__var}" = "RAPID_SYNC_URL" ] \ || [ "${__var}" = "OBOL_CHARON_CL_ENDPOINTS" ]; then @@ -1116,19 +1244,19 @@ __env_migrate() { fi fi if [[ "${__source_ver}" -lt "18" && "${__var}" = "OBOL_CHARON_CL_ENDPOINTS" ]]; then - __obol_cl_node=$(sed -n -e "s/^OBOL_CL_NODE=\(.*\)/\1/p" "${__env_file}.source" || true) + __obol_cl_node=$(__get_value_from_env "OBOL_CL_NODE" "${__env_file}.source") if [ -n "${__obol_cl_node}" ]; then __value="${__obol_cl_node}" echo "Set OBOL_CHARON_CL_ENDPOINTS to ${__value}" fi fi if [ "${__var}" = "CL_QUIC_PORT" ]; then - __cl_port=$(sed -n -e "s/^CL_P2P_PORT=\(.*\)/\1/p" "${__env_file}.source" || true) + __cl_port=$(__get_value_from_env "CL_P2P_PORT" "${__env_file}.source") if [ -n "${__cl_port}" ] && [ "${__cl_port}" = "${__value}" ]; then __value=$((__value + 1)) echo "Adjusted CL_QUIC_PORT to ${__value} so it does not conflict with CL_P2P_PORT" fi - __prysm_port=$(sed -n -e "s/^PRYSM_UDP_PORT=\(.*\)/\1/p" "${__env_file}.source" || true) + __prysm_port=$(__get_value_from_env "PRYSM_UDP_PORT" "${__env_file}.source") if [ -n "${__prysm_port}" ] && [ "${__prysm_port}" = "${__value}" ]; then # just in case this is one ahead __value=$((__value + 1)) echo "Adjusted CL_QUIC_PORT to ${__value} so it does not conflict with PRYSM_UDP_PORT" @@ -1145,13 +1273,13 @@ __env_migrate() { if [[ "${__var}" = "SHARE_IP" && "${__value: -1}" = ":" ]]; then __value="${__value%:}" # Undo Compose V1 accommodation fi - # Handle & in GRAFFITI gracefully - sed -i'.original' -e "s~^\(${__var}\s*=\s*\).*\$~\1${__value//&/\\&}~" "${__env_file}" + # Handle & in GRAFFITI gracefully, as well as multi-line + __update_value_in_env "${__var}" "$__value" "${__env_file}" else # empty __value if [ "${__var}" = "CF_ZONE_ID" ]; then __lookup_cf_zone if [ -n "${__value}" ]; then - sed -i'.original' -e "s~^\(${__var}\s*=\s*\).*\$~\1${__value//&/\\&}~" "${__env_file}" + __update_value_in_env "${__var}" "$__value" "${__env_file}" fi fi fi @@ -1159,7 +1287,7 @@ __env_migrate() { if [ "${__keep_targets}" -eq 1 ]; then # Migrate over build targets for __var in "${__target_vars[@]}"; do - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}.source" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}.source") if [ -n "${__value}" ]; then if [[ "${__var}" = "DDNS_TAG" && "${__source_ver}" -lt "8" ]]; then # Switch to ddns-updater __value="v2" @@ -1176,21 +1304,21 @@ __env_migrate() { if [[ "${__var}" = "SSV_NODE_REPO" && "${__value}" = "bloxstaking/ssv-node" ]]; then # SSV new repo __value="ssvlabs/ssv-node" fi - sed -i'.original' -e "s~^\(${__var}\s*=\s*\).*$~\1${__value}~" "${__env_file}" + __update_value_in_env "${__var}" "$__value" "${__env_file}" fi done fi # Move value from old variable name(s) to new one(s) for __index in "${!__old_vars[@]}"; do __var=${__old_vars[__index]} - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}.source" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}.source") if [ -n "${__value}" ]; then - sed -i'.original' -e "s~^\(${__new_vars[__index]}\s*=\s*\).*$~\1${__value}~" "${__env_file}" + __update_value_in_env "${__new_vars[__index]}" "$__value" "${__env_file}" fi done # Check whether we run a CL or VC, if so nag about FEE_RECIPIENT __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # It's CL&VC, CL-only, or VC-only # I do mean to match literally # shellcheck disable=SC2076 @@ -1199,7 +1327,7 @@ __env_migrate() { || "${__value}" =~ "-allin1.yml" || "${__value}" =~ "-vc-only.yml" ]]; then # Check for rewards __var="FEE_RECIPIENT" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") if [[ -z "${__value}" || ${__value} != 0x* || ${#__value} -ne 42 ]]; then if [ "${__non_interactive:-0}" -eq 0 ]; then whiptail --msgbox "A fee recipient ETH wallet address is required in order to start the client. This is \ @@ -1207,7 +1335,7 @@ __env_migrate() { Eth Docker docs (https://ethdocker.com/About/Rewards) for more information.\n\nCAUTION: \"$__me up\" will fail if no \ valid address is set" 12 75 __query_coinbase - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" else echo "A fee recipient ETH wallet address is required in order to start the client. Please set one in \".env\"." echo "CAUTION: \"$__me up\" will fail if no valid address is set." @@ -1217,15 +1345,14 @@ __env_migrate() { # User signals it's a distributed setup and not to nag __var="DISTRIBUTED" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") if [[ "${__value}" = "true" || "${__non_interactive:-0}" -eq 1 ]]; then - ${__as_owner} rm "${__env_file}".original __during_migrate=0 return 0 fi # Check for CL and EL, nag if we have only one without the other __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # Case 1 ... CL, do we have an EL? # I do mean to match literally # shellcheck disable=SC2076 @@ -1251,7 +1378,6 @@ ${__env_file}" 12 75 fi fi - ${__as_owner} rm "${__env_file}".original __during_migrate=0 echo "${__env_file} updated successfully" } @@ -1326,7 +1452,7 @@ update() { set +e ${__as_owner} git config pull.rebase false __var="ETH_DOCKER_TAG" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") if [ -z "${__value}" ] || [ "${__value}" = "latest" ]; then export ETHDPINNED="" __branch=$(git rev-parse --abbrev-ref HEAD) @@ -1469,7 +1595,7 @@ reset to defaults." resync-execution() { # Check for EL client __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") case "${__value}" in *erigon.yml* ) __el_volume='erigon-el-data'; __el_client="erigon";; @@ -1521,7 +1647,7 @@ resync-execution() { resync-consensus() { # Check for CL client __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") case "${__value}" in *lighthouse.yml* | *lighthouse-cl-only.yml* ) __cl_volume='lhconsensus-data'; __cl_client="lighthouse";; @@ -1544,7 +1670,7 @@ resync-consensus() { # Can we checkpoint sync? __var="RAPID_SYNC_URL" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") echo "This will stop ${__cl_client} and delete its database to force a resync." if [ -z "${__value}" ]; then read -rp "WARNING - RAPID_SYNC_URL not set, resync may take days. Do you wish to continue? (No/yes) " __yn @@ -1639,7 +1765,7 @@ prune-besu() { # Check for archive node __var="ARCHIVE_NODE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") if [[ "${__value}" = "true" ]]; then echo "Besu is an archive node: Aborting." exit 1 @@ -1732,7 +1858,7 @@ prune-reth() { # Check for archive node __var="ARCHIVE_NODE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") if [[ "${__value}" = "true" ]]; then echo "Reth is an archive node: Aborting." exit 1 @@ -1825,7 +1951,7 @@ prune-nethermind() { # Check for archive node __var="ARCHIVE_NODE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") if [[ "${__value}" = "true" ]]; then echo "Nethermind is an archive node: Aborting." exit 1 @@ -1834,7 +1960,7 @@ prune-nethermind() { __get_docker_free_space __var="NETWORK" - NETWORK=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + NETWORK=$(__get_value_from_env "${__var}" "${__env_file}") if [ "${NETWORK}" = "mainnet" ] || [ "${NETWORK}" = "gnosis" ]; then __min_free=262144000 @@ -1883,7 +2009,7 @@ prune-nethermind() { fi __var="AUTOPRUNE_NM" - __auto_prune=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __auto_prune=$(__get_value_from_env "${__var}" "${__env_file}") if [ $__non_interactive = 0 ]; then while true; do @@ -1965,7 +2091,7 @@ prune-lighthouse() { fi __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # Literal match intended # shellcheck disable=SC2076 if [[ ! "${__value}" =~ "lighthouse.yml" && ! "${__value}" =~ "lighthouse-cl-only.yml" ]]; then @@ -1975,7 +2101,7 @@ prune-lighthouse() { # Check for archive node __var="ARCHIVE_NODE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") if [[ "${__value}" = "true" ]]; then echo "Lighthouse is an archive node: Aborting." exit 1 @@ -2040,7 +2166,7 @@ __prep-keyimport() { fi __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # Literal match intended # shellcheck disable=SC2076 if [[ ! "${__value}" =~ "prysm.yml" ]] && [[ ! "${__value}" =~ "lighthouse.yml" ]] && [[ ! "${__value}" =~ "teku.yml" ]] \ @@ -2123,7 +2249,7 @@ __i_haz_ethdo() { exit 0 fi __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # Literal match intended # shellcheck disable=SC2076 if [[ ! "${__value}" =~ "ethdo.yml" ]]; then @@ -2142,7 +2268,7 @@ __i_haz_ethdo() { COMPOSE_FILE="ethdo.yml" echo "You do not have a CL in ${__project_name}. Please make sure CL_NODE in ${__env_file} points at an available one" fi - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" echo "Your COMPOSE_FILE now reads ${COMPOSE_FILE}" fi } @@ -2155,13 +2281,13 @@ __i_haz_web3signer() { fi __var="WEB3SIGNER" - __w3s=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __w3s=$(__get_value_from_env "${__var}" "${__env_file}") if [ ! "${__w3s}" = "true" ]; then return 0 fi __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # Literal match intended # shellcheck disable=SC2076 if [[ ! "${__value}" =~ "web3signer.yml" ]]; then @@ -2181,7 +2307,7 @@ __i_haz_web3signer() { echo "You do not have a validator client in ${__project_name}. web3signer cannot be used without one." exit 1 fi - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" echo "Your COMPOSE_FILE now reads ${COMPOSE_FILE}" fi } @@ -2292,7 +2418,7 @@ keys() { __docompose run --rm -e OWNER_UID="${__owner_uid}" validator-keys import "${__args}" elif [ "${1:-}" = "create-prysm-wallet" ]; then __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # Literal match intended # shellcheck disable=SC2076 if [[ ! "${__value}" =~ "prysm.yml" ]] && [[ ! "${__value}" =~ "prysm-vc-only.yml" ]]; then @@ -2305,8 +2431,8 @@ keys() { up fi elif [ "${1:-}" = "create-for-csm" ]; then - var="NETWORK" - NETWORK=$(sed -n -e "s/^${var}=\(.*\)/\1/p" "${ENV_FILE}" || true) + __var="NETWORK" + NETWORK=$(__get_value_from_env "${__var}" "${__env_file}") __query_lido_keys_generation elif [ "${1:-}" = "prepare-address-change" ]; then __i_haz_ethdo @@ -2465,7 +2591,7 @@ keys() { #elif [ "${1:-}" = "send-exit" ] && ! __i_haz_keys_service silent; then elif [ "${1:-}" = "send-exit" ]; then __var="CL_NODE" - CL_NODE=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + CL_NODE=$(__get_value_from_env "${__var}" "${__env_file}") CL_NODE="${CL_NODE%%,*}" __network_name="$(__docompose config | awk ' BEGIN { @@ -2509,7 +2635,7 @@ keys() { __docompose run --rm -e OWNER_UID="${__owner_uid}" validator-keys "$@" fi __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") } @@ -2588,7 +2714,7 @@ terminate() { __query_network() { __var="NETWORK" - __prev_network=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __prev_network=$(__get_value_from_env "${__var}" "${__env_file}") NETWORK=$(whiptail --notags --title "Select Network" --menu \ "Which network do you want to run on?" 13 65 6 \ "holesky" "Holešovice Testnet" \ @@ -2807,9 +2933,9 @@ __query_custom_execution_client() { JWT_SECRET="" else __var="EL_NODE" - EL_CUSTOM_NODE=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + EL_CUSTOM_NODE=$(__get_value_from_env "${__var}" "${__env_file}") __var="JWT_SECRET" - JWT_SECRET=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + JWT_SECRET=$(__get_value_from_env "${__var}" "${__env_file}") fi EL_CUSTOM_NODE=$(whiptail --title "Configure custom execution client" --inputbox "What is the URL for your custom \ execution client? (right-click to paste)" 10 65 "${EL_CUSTOM_NODE}" 3>&1 1>&2 2>&3) @@ -2909,7 +3035,7 @@ __query_remote_beacon() { fi else __var="CL_NODE" - REMOTE_BEACON=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + REMOTE_BEACON=$(__get_value_from_env "${__var}" "${__env_file}") fi REMOTE_BEACON=$(whiptail --title "Configure remote consensus client" --inputbox "What is the URL for your remote \ consensus client? (right-click to paste)" 10 60 "${REMOTE_BEACON}" 3>&1 1>&2 2>&3) @@ -2923,7 +3049,7 @@ __query_checkpoint_beacon() { RAPID_SYNC_URL="" else __var="RAPID_SYNC_URL" - RAPID_SYNC_URL=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + RAPID_SYNC_URL=$(__get_value_from_env "${__var}" "${__env_file}") fi if [ -z "${RAPID_SYNC_URL}" ]; then case "${NETWORK}" in @@ -2954,7 +3080,7 @@ checkpoint sync provider? (right-click to paste)" 10 65 "${RAPID_SYNC_URL}" 3>&1 __query_graffiti() { __var="GRAFFITI" - GRAFFITI=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + GRAFFITI=$(__get_value_from_env "${__var}" "${__env_file}") while true; do GRAFFITI=$(whiptail --title "Configure Graffiti" --inputbox "What Graffiti do you want to send with your blocks? \ @@ -2982,7 +3108,7 @@ __query_rapid_sync() { __query_coinbase() { __var="FEE_RECIPIENT" - FEE_RECIPIENT=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + FEE_RECIPIENT=$(__get_value_from_env "${__var}" "${__env_file}") while true; do set +e # Can't rely on the error handler here because of the special-casing below for update() @@ -3061,7 +3187,7 @@ https://0x98650451ba02064f7b000f5768cf0cf4d4e492317d82871bdc87ef841a0743f69f0f1e return 0 fi __var="MEV_BOOST" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # I do mean to match literally # shellcheck disable=SC2076 if [[ "${CONSENSUS_CLIENT}" =~ "-vc-only.yml" ]]; then @@ -3161,7 +3287,7 @@ want to use MEV Boost?" 10 65); then MEV_BOOST="true" if [ "${__value}" = "true" ]; then __var="MEV_RELAYS" - MEV_RELAYS=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + MEV_RELAYS=$(__get_value_from_env "${__var}" "${__env_file}") else case "${NETWORK}" in "sepolia") @@ -3188,8 +3314,14 @@ https://0x8c7d33605ecef85403f8b7289c8058f440cbb6bf72b055dfe2f3e2c6695b6a1ea5a9cd ;; esac fi - MEV_RELAYS=$(whiptail --title "Configure MEV relays" --inputbox "What MEV relay(s) do you want to use? \ -(right-click to paste)" 10 65 "${MEV_RELAYS}" 3>&1 1>&2 2>&3) + # Replace newlines with "\n" for the whiptail input + __formatted_mev_relays=$(printf '%s' "${MEV_RELAYS}" | sed ':a;N;$!ba;s/\n/\\n/g') + + __formatted_mev_relays=$(whiptail --title "Configure MEV relays" --inputbox "What MEV relay(s) do you want to use? \ +(right-click to paste)" 15 65 "${__formatted_mev_relays}" 3>&1 1>&2 2>&3) + + # Replace "\n" back to newlines to restore multi-line format + MEV_RELAYS=$(printf '%s' "${__formatted_mev_relays}" | sed 's/\\n/\n/g') echo "Your MEV relay(s): ${MEV_RELAYS}" else MEV_BOOST="false" @@ -3360,18 +3492,6 @@ __query_dkg() { rm -f ssv-config/dkg-config.yaml.original } -__set_value_in_env() { -# Assumes that "__var" has been set to the name of the variable to be changed - if [ "${!__var+x}" ]; then - if ! grep -qF "${__var}" "${__env_file}" 2>/dev/null ; then - echo "${__var}=${!__var}" >> "${__env_file}" - else -# Handle & in GRAFFITI gracefully - sed -i'.original' -e "s~^\(${__var}\s*=\s*\).*\$~\1${!__var//&/\\&}~" "${__env_file}" - fi - fi -} - __handle_error() { if [[ ! $- =~ e ]]; then @@ -3426,7 +3546,7 @@ __handle_error() { __check_legacy() { __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # Literal match intended # shellcheck disable=SC2076 @@ -3683,6 +3803,7 @@ config() { COMPOSE_FILE="${COMPOSE_FILE}:deposit-cli.yml" fi # Not multi-arch, this would break on ARM64 + # COMPOSE_FILE="${COMPOSE_FILE}:ethdo.yml" if [ "${__deployment}" = "rocket" ]; then COMPOSE_FILE="${COMPOSE_FILE}:ext-network.yml" @@ -3692,72 +3813,70 @@ config() { echo "Your COMPOSE_FILE is:" "${COMPOSE_FILE}" __var=FEE_RECIPIENT - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __var=GRAFFITI - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __var=CL_NODE - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __var=RAPID_SYNC_URL - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __var=COMPOSE_FILE - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __var=EL_NODE - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __var=JWT_SECRET - __set_value_in_env + __update_value_in_env "${__var}" "${!__var-}" "${__env_file}" __var=NETWORK - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __var=MEV_BOOST - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" __var=MEV_RELAYS - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" if [ "${__deployment}" = "lido_obol" ]; then - var=LIDO_DV_EXIT_EXIT_EPOCH - __set_value_in_env - var=VE_OPERATOR_ID - __set_value_in_env - var=VE_LOCATOR_ADDRESS - __set_value_in_env - var=VE_ORACLE_ADDRESSES_ALLOWLIST - __set_value_in_env - var=VE_STAKING_MODULE_ID - __set_value_in_env + __var=LIDO_DV_EXIT_EXIT_EPOCH + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" + __var=VE_OPERATOR_ID + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" + __var=VE_LOCATOR_ADDRESS + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" + __var=VE_ORACLE_ADDRESSES_ALLOWLIST + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" + __var=VE_STAKING_MODULE_ID + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" # We are using the variable # shellcheck disable=SC2034 ENABLE_DIST_ATTESTATION_AGGR="true" - var=ENABLE_DIST_ATTESTATION_AGGR - __set_value_in_env + __var=ENABLE_DIST_ATTESTATION_AGGR + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" fi if [[ "${NETWORK}" = "gnosis" ]] && [[ "${CONSENSUS_CLIENT}" =~ "nimbus" ]] ; then # We are using the variable # shellcheck disable=SC2034 NIM_DOCKERFILE=Dockerfile.sourcegnosis __var=NIM_DOCKERFILE - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" fi if uname -m | grep -q riscv64; then # We are using the variable # shellcheck disable=SC2034 NIM_DOCKERFILE=Dockerfile.source __var=NIM_DOCKERFILE - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" # We are using the variable # shellcheck disable=SC2034 GETH_DOCKERFILE=Dockerfile.source __var=GETH_DOCKERFILE - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" fi __var="SIREN_PASSWORD" - SIREN_PASSWORD=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + SIREN_PASSWORD=$(__get_value_from_env "${__var}" "${__env_file}") if [ -z "${SIREN_PASSWORD}" ]; then SIREN_PASSWORD=$(head -c 8 /dev/urandom | od -A n -t u8 | tr -d '[:space:]' | sha256sum | head -c 32) - __set_value_in_env + __update_value_in_env "${__var}" "${!__var}" "${__env_file}" fi __enable_v6 - ${__as_owner} rm .env.original - __pull_and_build __nag_os_version @@ -3772,7 +3891,7 @@ version() { grep "^This is" README.md echo __var="COMPOSE_FILE" - __value=$(sed -n -e "s/^${__var}=\(.*\)/\1/p" "${__env_file}" || true) + __value=$(__get_value_from_env "${__var}" "${__env_file}") # Client versions case "${__value}" in *lido-obol.yml* )