Merge pull request #801 from VallariAg/nvmeof-ceph-status

Show nvmeof gateways in "ceph status"

.github/workflows/build-container.yml

@@ -737,7 +737,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: ["sanity", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify"]
+        test: ["sanity", "state_transitions", "state_transitions_both_gws", "state_transitions_loop", "state_transitions_rand_loop", "late_registration", "late_registration_loop", "4gws", "4gws_loop", "4gws_create_delete", "4gws_create_delete_loop", "namespaces", "namespaces_loop", "mtls", "notify", "ceph_status"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 1024  # 4 spdk instances

control/cephutils.py

@@ -96,6 +96,22 @@ def pool_exists(self, pool) -> bool:
 
         return False
 
+    def service_daemon_register(self, cluster, metadata):
+        try:
+            if cluster: # rados client 
+                daemon_name = metadata['id']
+                cluster.service_daemon_register("nvmeof", daemon_name, metadata)
+                self.logger.info(f"Registered {daemon_name} to service_map!")
+        except Exception:
+            self.logger.exception(f"Can't register daemon to service_map!")
+
+    def service_daemon_update(self, cluster, status_buffer):
+        try:
+            if cluster and status_buffer:
+                cluster.service_daemon_update(status_buffer)
+        except Exception:
+            self.logger.exception(f"Can't update daemon status to service_map!") 
+
     def create_image(self, pool_name, image_name, size) -> bool:
         # Check for pool existence in advance as we don't create it if it's not there
         if not self.pool_exists(pool_name):

control/server.py

@@ -203,6 +203,19 @@ def serve(self):
             log_level = log_level.strip().upper()
             log_req = pb2.set_spdk_nvmf_logs_req(log_level=log_level, print_level=log_level)
             self.gateway_rpc.set_spdk_nvmf_logs(log_req)
+
+        self._register_service_map()
+
+    def _register_service_map(self):
+        # show gateway in "ceph status" output
+        conn = self.omap_state.conn
+        metadata = {
+            "id": self.name.removeprefix("client.nvmeof."),
+            "pool_name": self.config.get("ceph", "pool"),
+            "daemon_type": "gateway", # "nvmeof: 3 <daemon_type> active (3 hosts)"
+            "group": self.config.get_with_default("gateway", "group", ""),
+        } 
+        self.ceph_utils.service_daemon_register(conn, metadata) 
 
     def _monitor_client_version(self) -> str:
         """Return monitor client version string."""

control/state.py

@@ -573,6 +573,9 @@ def cleanup_omap(self, omap_lock = None):
                 self.ioctx = None
             except Exception:
                 pass
+        if self.conn:
+            self.conn.shutdown()
+            self.conn = None
 
 class GatewayStateHandler:
     """Maintains consistency in NVMeoF target state store instances.

tests/ha/ceph_status.sh

@@ -0,0 +1,22 @@
+set -xe
+
+POOL="${RBD_POOL:-rbd}"
+CEPH_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | grep -v nvme | grep ceph | awk '{print $1}')
+GW1_NAME=$(docker ps --format '{{.ID}}\t{{.Names}}' | awk '$2 ~ /nvmeof/ && $2 ~ /1/ {print $1}')
+
+docker compose exec -T ceph ceph service dump
+docker compose exec -T ceph ceph status
+
+echo "ℹ️  Step 1: verify 2 gateways"
+
+docker compose exec -T ceph ceph status | grep "nvmeof: 2 gateways active"
+
+echo "ℹ️  Step 2: stop a gateway"
+
+docker stop $GW1_NAME
+wait
+sleep 5
+
+echo "ℹ️  Step 3: verify 1 gateway"
+
+docker compose exec -T ceph ceph status | grep "nvmeof: 1 gateway active"