tests/ha/4gws_create_delete.sh: add verify number of namespaces per ana group in all active step

Signed-off-by: Alexander Indenbaum <aindenba@redhat.com>

Author: Alexander Indenbaum

tests/ha/4gws_create_delete.sh

@@ -58,6 +58,10 @@ gw_name() {
   docker ps --format '{{.ID}}\t{{.Names}}' --filter status=running --filter status=exited | awk '$2 ~ /nvmeof/ && $2 ~ /'$i'/ {print $1}'
 }
 
+gw_ip() {
+  docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' "$(gw_name $1)"
+}
+
 # Function to access numbers by index
 access_number_by_index() {
     numbers=$1
@@ -109,6 +113,51 @@ choose_n_m() {
     done
 }
 
+count_namespaces_in_anagrp() {
+    json="$1"            # subsystems json data
+    subsystem_idx="$2"   # subsystem index (e.g., 0, 1)
+    ana_group="$3"       # ana group id
+
+    echo "$json" | jq ".subsystems[$subsystem_idx].namespaces | map(select(.anagrpid == $ana_group)) | length"
+}
+
+verify_num_namespaces() {
+  # verify initial distribution of namespaces
+  for g in $(seq $NUM_GATEWAYS); do
+    for i in $(seq 10); do
+      echo "verify_num_namespaces $i $GW_NAME $GW_IP"
+      GW_NAME=$(gw_name $g)
+      GW_IP=$(gw_ip $g)
+      subs=$(docker compose  run --rm nvmeof-cli --server-address $GW_IP --server-port 5500 get_subsystems 2>&1 | sed 's/Get subsystems://')
+
+      # ensure namespaces are evenly distributed across ANA groups.
+      # each subsystem should have at least half of the namespaces if they
+      # were equally divided among the four ANA groups.
+      if [ "$(count_namespaces_in_anagrp "$subs" 0 1)" -ge $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \
+           "$(count_namespaces_in_anagrp "$subs" 0 2)" -ge $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \
+           "$(count_namespaces_in_anagrp "$subs" 0 3)" -ge $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \
+           "$(count_namespaces_in_anagrp "$subs" 0 4)" -ge $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \
+           "$(count_namespaces_in_anagrp "$subs" 1 1)" -ge $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \
+           "$(count_namespaces_in_anagrp "$subs" 1 2)" -ge $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \
+           "$(count_namespaces_in_anagrp "$subs" 1 3)" -ge $MIN_NUM_NAMESPACES_IN_ANA_GROUP -o \
+           "$(count_namespaces_in_anagrp "$subs" 1 4)" -ge $MIN_NUM_NAMESPACES_IN_ANA_GROUP ]; then
+
+          echo "Not ready $i $GW_NAME $GW_IP"
+          sleep 5
+          continue
+      fi
+      echo "✅ verify_num_namespaces ready $i $GW_NAME $GW_IP"
+      break
+    done
+
+    # Check if the inner loop completed without breaking
+    if [ $i -eq 10 ]; then
+      echo "verify_num_namespaces ‼️  Timeout reached for $GW_NAME $GW_IP"
+      exit 1
+    fi
+  done
+}
+
 validate_all_active() {
   for s in $(seq $NUM_SUBSYSTEMS); do
     all_ana_states=$(for g in $(seq $NUM_GATEWAYS); do
@@ -123,6 +172,9 @@ validate_all_active() {
       exit 1
     fi
   done
+
+  # ensure namespaces are evenly distributed across ANA groups
+  verify_num_namespaces
 }
 
 
@@ -132,6 +184,7 @@ validate_all_active() {
 
 NUM_SUBSYSTEMS=2
 NUM_GATEWAYS=4
+MIN_NUM_NAMESPACES_IN_ANA_GROUP=4
 FAILING_GATEWAYS=2
 NUM_OPTIMIZED_FAILOVER=2
 NUM_OPTIMIZED_REBALANCE=1