diff --git a/tests/ha/4gws_create_delete.sh b/tests/ha/4gws_create_delete.sh index 2764a087..31341e93 100755 --- a/tests/ha/4gws_create_delete.sh +++ b/tests/ha/4gws_create_delete.sh @@ -58,6 +58,10 @@ gw_name() { docker ps --format '{{.ID}}\t{{.Names}}' --filter status=running --filter status=exited | awk '$2 ~ /nvmeof/ && $2 ~ /'$i'/ {print $1}' } +gw_ip() { + docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$(gw_name $1)" +} + # Function to access numbers by index access_number_by_index() { numbers=$1 @@ -109,6 +113,51 @@ choose_n_m() { done } +count_namespaces_in_anagrp() { + json="$1" # subsystems json data + subsystem_idx="$2" # subsystem index (e.g., 0, 1) + ana_group="$3" # ana group id + + echo "$json" | jq ".subsystems[$subsystem_idx].namespaces | map(select(.anagrpid == $ana_group)) | length" +} + +verify_num_namespaces() { + # verify initial distribution of namespaces + for g in $(seq $NUM_GATEWAYS); do + for i in $(seq 10); do + echo "verify_num_namespaces $i $GW_NAME $GW_IP" + GW_NAME=$(gw_name $g) + GW_IP=$(gw_ip $g) + subs=$(docker compose run --rm nvmeof-cli --server-address $GW_IP --server-port 5500 get_subsystems 2>&1 | sed 's/Get subsystems://') + + # ensure namespaces are evenly distributed across ANA groups. + # each subsystem should have at least half of the namespaces if they + # were equally divided among the four ANA groups. + if [ "$(count_namespaces_in_anagrp "$subs" 0 1)" -lt $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \ + "$(count_namespaces_in_anagrp "$subs" 0 2)" -lt $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \ + "$(count_namespaces_in_anagrp "$subs" 0 3)" -lt $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \ + "$(count_namespaces_in_anagrp "$subs" 0 4)" -lt $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \ + "$(count_namespaces_in_anagrp "$subs" 1 1)" -lt $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \ + "$(count_namespaces_in_anagrp "$subs" 1 2)" -lt $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \ + "$(count_namespaces_in_anagrp "$subs" 1 3)" -lt $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \ + "$(count_namespaces_in_anagrp "$subs" 1 4)" -lt $MIN_NUM_NAMESPACES_IN_ANA_GROUP ]; then + + echo "Not ready $i $GW_NAME $GW_IP" + sleep 5 + continue + fi + echo "✅ verify_num_namespaces ready $i $GW_NAME $GW_IP" + break + done + + # Check if the inner loop completed without breaking + if [ $i -eq 10 ]; then + echo "verify_num_namespaces ‼️ Timeout reached for $GW_NAME $GW_IP" + exit 1 + fi + done +} + validate_all_active() { for s in $(seq $NUM_SUBSYSTEMS); do all_ana_states=$(for g in $(seq $NUM_GATEWAYS); do @@ -123,6 +172,9 @@ validate_all_active() { exit 1 fi done + + # ensure namespaces are evenly distributed across ANA groups + verify_num_namespaces } @@ -132,6 +184,7 @@ validate_all_active() { NUM_SUBSYSTEMS=2 NUM_GATEWAYS=4 +MIN_NUM_NAMESPACES_IN_ANA_GROUP=4 FAILING_GATEWAYS=2 NUM_OPTIMIZED_FAILOVER=2 NUM_OPTIMIZED_REBALANCE=1