new github redeploy test

.env

@@ -65,8 +65,8 @@ SPDK_CENTOS_REPO_VER="9.0-21.el9"
 
 # Ceph Cluster
 CEPH_CLUSTER_VERSION="${CEPH_VERSION}"
-CEPH_BRANCH=main
-CEPH_SHA=latest
+CEPH_BRANCH=wip-leonidc-1803-redeploy-fix
+CEPH_SHA=2f2b7874cfe8af46cd4f61c9133da8b93790c884
 
 CEPH_DEVEL_MGR_PATH=../ceph
 

.github/workflows/build-container.yml

@@ -491,7 +491,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ["sanity", "no_huge", "ns_lb_change", "no_subsystems", "auto_load_balance", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit", "cluster_pool", "flat_bdev_per_cluster", "set_qos", "set_qos_2ms"]
+        test: ["sanity", "no_huge", "ns_lb_change", "redeploy", "no_subsystems", "auto_load_balance", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status", "blocklist", "main_exit", "cluster_pool", "flat_bdev_per_cluster", "set_qos", "set_qos_2ms"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 1024  # 4 spdk instances

tests/ha/redeploy.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+set -xe
+
+GW1_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /1/ {print $1}')
+GW2_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /2/ {print $1}')
+#GW3_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /3/ {print $1}')
+
+
+  verify_number_active_groups()
+  {
+     NUM="$1"
+     #test that ana group is Active
+     json=$(docker compose exec -T ceph ceph nvme-gw show  rbd '')
+     states=$(echo "$json" | jq -r '.["Created Gateways:"][] | ."ana states"')
+     echo "$states"
+     rc=$(echo "$states" | grep  ' ACTIVE' | wc -l)
+     echo $rc
+     if [ "$rc" -ne "$NUM" ]; then
+       echo "Error!: wrong number of Active ANA groups found $rc"
+       exit 1
+     else
+       echo "Correct number of Active ANA groups found $rc"
+     fi
+     echo "$json" | jq -r '."GW-epoch"'
+
+  }
+
+echo "ℹ️ ℹ️ Start test:  Redeploy test  - simulate fast reboot and verify that no failovers during 12 sec"
+
+sleep 10
+epoch0=$(verify_number_active_groups 2)
+
+#simulate fast-reboot
+# get nvme-gw show 
+docker stop $GW1_NAME
+docker start $GW1_NAME
+sleep 16
+epoch1=$(verify_number_active_groups 2)
+#verify only one Ana group is Active - means no failover 
+
+sleep 8
+epoch2=$(verify_number_active_groups 2)
+
+if (( epoch1 - epoch0 > 3 )); then
+	exit 1
+fi
+
+
+# now redeploy all  all Gws
+docker stop $GW1_NAME
+docker start $GW1_NAME
+sleep 1
+docker stop $GW2_NAME
+docker start $GW2_NAME
+
+sleep 16
+epoch3=$(verify_number_active_groups 2)
+
+if (( epoch3 - epoch2 > 5 )); then
+        exit 1
+fi
+
+
+sleep 8
+verify_number_active_groups 2
+
+echo "ℹ️ ℹ️  test passed"