add blocklist test and also add verification in state transition test

Signed-off-by: hezko <tomer.haska@gmail.com>

Author: Tomer Haskalovitch

.github/workflows/build-container.yml

@@ -538,7 +538,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ["sanity", "ns_lb_change", "no_subsystems", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status"]
+        test: ["sanity", "ns_lb_change", "no_subsystems", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 1024  # 4 spdk instances

tests/ha/blocklist.sh

@@ -0,0 +1,38 @@
+source .env
+
+verify_blocklist() {
+  stopped_gw_name=$1
+  NODE_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $stopped_gw_name)
+  BLOCKLIST=$(docker compose exec -T ceph ceph osd blocklist ls)
+
+  echo "verifying there is at least 1 entry in the blocklist related to the stopped gateway"
+  if echo "$BLOCKLIST" | grep -q "$NODE_IP"; then
+    echo "ip $NODE_IP for the stopped gateway was found the blocklist."
+  else
+      echo "ip $NODE_IP for node the stopped gateway was not found in blocklist."
+      exit 1
+  fi
+
+  echo "verifying there are no entries in the blocklist which are not related to the stopped gateway"
+  if echo "$BLOCKLIST" | grep -qv "$NODE_IP"; then
+    echo "found at least 1 entry in blocklist which is not related to gateway in the stopped gateway. failing"
+    exit 1
+  else
+      echo "didn't find unexpected entries which are not relaetd to the stopped gateway."
+  fi
+  echo "blocklist verification successful"
+}
+
+echo "obtaining gw1 container id and its ip"
+GW1_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /1/ {print $1}')
+
+echo "clearing blocklist"
+docker compose exec -T ceph ceph osd blocklist clear
+
+echo "shutting down gw1:$GW1_NAME"
+docker stop $GW1_NAME
+
+echo "waiting for 30s after shutdown"
+sleep 30
+
+verify_blocklist "$GW1_NAME"

tests/ha/state_transitions.sh

@@ -65,6 +65,29 @@ verify_ana_groups() {
     fi
 }
 
+verify_blocklist() {
+  stopped_gw_name=$1
+  NODE_IP=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' $stopped_gw_name)
+  BLOCKLIST=$(docker compose exec -T ceph ceph osd blocklist ls)
+
+  echo "verifying there is at least 1 entry in the blocklist related to the stopped gateway"
+  if echo "$BLOCKLIST" | grep -q "$NODE_IP"; then
+    echo "ip $NODE_IP for the stopped gateway was found the blocklist."
+  else
+      echo "ip $NODE_IP for node the stopped gateway was not found in blocklist."
+      exit 1
+  fi
+
+  echo "verifying there are no entries in the blocklist which are not related to the stopped gateway"
+  if echo "$BLOCKLIST" | grep -qv "$NODE_IP"; then
+    echo "found at least 1 entry in blocklist which is not related to gateway in the stopped gateway. failing"
+    exit 1
+  else
+      echo "didn't find unexpected entries which are not relaetd to the stopped gateway."
+  fi
+  echo "blocklist verification successful"
+}
+
 #
 # MAIN
 #
@@ -94,6 +117,7 @@ GW1_FAILOVER_OPTIMIZED=$(expect_optimized $GW1_NAME 2)
 gw1_ana1=$(access_number_by_index "$GW1_FAILOVER_OPTIMIZED" 0)
 gw1_ana2=$(access_number_by_index "$GW1_FAILOVER_OPTIMIZED" 1)
 verify_ana_groups "$gw1_ana1" "$gw1_ana2"
+verify_blocklist "$GW2_NAME"
 
 #
 # Step 3 failback