Add CLI that enables NVMe ANA report per volumes and utilizes SPDK ANA-groupIds, ANA states for namespaces and listeners

control/cli.py

@@ -187,12 +187,17 @@ def delete_bdev(self, args):
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
         argument("-s", "--serial", help="Serial number", required=False),
         argument("-m", "--max-namespaces", help="Maximum number of namespaces", type=int, default=0, required=False),
+        argument("-a", "--ana-reporting", help="Enable ANA reporting", type=bool, default=False, required=False),
+        argument("-t", "--enable-ha", help="Enable automatic HA"  , type=bool, default=False, required=False),
+
     ])
     def create_subsystem(self, args):
         """Creates a subsystem."""
         req = pb2.create_subsystem_req(subsystem_nqn=args.subnqn,
                                         serial_number=args.serial,
-                                        max_namespaces=args.max_namespaces)
+                                        max_namespaces=args.max_namespaces,
+                                        ana_reporting=args.ana_reporting,
+                                        enable_ha=args.enable_ha)
         ret = self.stub.create_subsystem(req)
         self.logger.info(f"Created subsystem {args.subnqn}: {ret.status}")
 
@@ -209,15 +214,20 @@ def delete_subsystem(self, args):
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),
         argument("-b", "--bdev", help="Bdev name", required=True),
         argument("-i", "--nsid", help="Namespace ID", type=int),
+        argument("-a", "--anagrpid", help="ANA group ID", type=int),
     ])
     def add_namespace(self, args):
         """Adds a namespace to a subsystem."""
+        if args.anagrpid == 0:
+	        args.anagrpid = 1
+
         req = pb2.add_namespace_req(subsystem_nqn=args.subnqn,
                                     bdev_name=args.bdev,
-                                    nsid=args.nsid)
+                                    nsid=args.nsid,
+                                    anagrpid=args.anagrpid)
         ret = self.stub.add_namespace(req)
         self.logger.info(
-            f"Added namespace {ret.nsid} to {args.subnqn}: {ret.status}")
+            f"Added namespace {ret.nsid} to {args.subnqn}, ANA group id {args.anagrpid} : {ret.status}")
 
     @cli.cmd([
         argument("-n", "--subnqn", help="Subsystem NQN", required=True),

control/discovery.py

@@ -316,10 +316,10 @@ def __init__(self, config):
         self.lock = threading.Lock()
 
         self.logger = logging.getLogger(__name__)
-        log_level = self.config.get("discovery", "debug")
-        self.logger.setLevel(level=int(log_level))
+        log_level = self.config.getint_with_default("discovery", "debug", 20)
+        self.logger.setLevel(level=log_level)
 
-        gateway_group = self.config.get("gateway", "group")
+        gateway_group = self.config.get_with_default("gateway", "group", "")
         self.omap_name = f"nvmeof.{gateway_group}.state" \
             if gateway_group else "nvmeof.state"
         self.logger.info(f"log pages info from omap: {self.omap_name}")
@@ -330,9 +330,9 @@ def __init__(self, config):
         conn.connect()
         self.ioctx = conn.open_ioctx(ceph_pool)
 
-        self.discovery_addr = self.config.get("discovery", "addr")
-        self.discovery_port = self.config.get("discovery", "port")
-        if self.discovery_addr == '' or self.discovery_port == '':
+        self.discovery_addr = self.config.get_with_default("discovery", "addr", "0.0.0.0")
+        self.discovery_port = self.config.get_with_default("discovery", "port", "8009")
+        if not self.discovery_addr or not self.discovery_port:
             self.logger.error("discovery addr/port are empty.")
             assert 0
         self.logger.info(f"discovery addr: {self.discovery_addr} port: {self.discovery_port}")

control/grpc.py

@@ -21,6 +21,7 @@
 from .proto import gateway_pb2 as pb2
 from .proto import gateway_pb2_grpc as pb2_grpc
 
+MAX_ANA_GROUPS = 4
 
 class GatewayService(pb2_grpc.GatewayServicer):
     """Implements gateway service interface.
@@ -190,7 +191,7 @@ def create_subsystem(self, request, context=None):
         """Creates a subsystem."""
 
         self.logger.info(
-            f"Received request to create subsystem {request.subsystem_nqn}")
+            f"Received request to create subsystem {request.subsystem_nqn}, ana reporting: {request.ana_reporting}  ")
         min_cntlid = self.config.getint_with_default("gateway", "min_controller_id", 1)
         max_cntlid = self.config.getint_with_default("gateway", "max_controller_id", 65519)
         if not request.serial_number:
@@ -205,6 +206,7 @@ def create_subsystem(self, request, context=None):
                 max_namespaces=request.max_namespaces,
                 min_cntlid=min_cntlid,
                 max_cntlid=max_cntlid,
+                ana_reporting = request.ana_reporting,
             )
             self.logger.info(f"create_subsystem {request.subsystem_nqn}: {ret}")
         except Exception as ex:
@@ -259,7 +261,9 @@ def delete_subsystem(self, request, context=None):
 
     def add_namespace(self, request, context=None):
         """Adds a namespace to a subsystem."""
-
+        if request.anagrpid > MAX_ANA_GROUPS:
+            raise Exception(f"Error group ID {request.anagrpid} is more than configured maximum {MAX_ANA_GROUPS}\n")
+
         self.logger.info(f"Received request to add {request.bdev_name} to"
                          f" {request.subsystem_nqn}")
         try:
@@ -268,6 +272,7 @@ def add_namespace(self, request, context=None):
                 nqn=request.subsystem_nqn,
                 bdev_name=request.bdev_name,
                 nsid=request.nsid,
+                anagrpid=request.anagrpid,
             )
             self.logger.info(f"add_namespace: {nsid}")
         except Exception as ex:
@@ -296,7 +301,7 @@ def add_namespace(self, request, context=None):
     def remove_namespace(self, request, context=None):
         """Removes a namespace from a subsystem."""
 
-        self.logger.info(f"Received request to remove {request.nsid} from"
+        self.logger.info(f"Received request to remove nsid {request.nsid} from"
                          f" {request.subsystem_nqn}")
         try:
             ret = rpc_nvmf.nvmf_subsystem_remove_ns(
@@ -412,7 +417,6 @@ def remove_host(self, request, context=None):
 
     def create_listener(self, request, context=None):
         """Creates a listener for a subsystem at a given IP/Port."""
-
         ret = True
         self.logger.info(f"Received request to create {request.gateway_name}"
                          f" {request.trtype} listener for {request.nqn} at"
@@ -438,6 +442,30 @@ def create_listener(self, request, context=None):
                 context.set_details(f"{ex}")
             return pb2.req_status()
 
+        state = self.gateway_state.omap.get_state()
+        for key,val  in state.items():
+              if (key.startswith(self.gateway_state.omap.SUBSYSTEM_PREFIX + request.nqn)):
+                     self.logger.debug(f"values of key: {key}  val: {val} \n")
+                     req = json_format.Parse(val, pb2.create_subsystem_req())
+                     self.logger.info(f" enable_ha :{req.enable_ha} \n")
+                     break
+
+        if req.enable_ha:
+              for x in range (MAX_ANA_GROUPS):
+                   try:
+                      ret = rpc_nvmf.nvmf_subsystem_listener_set_ana_state(
+                        self.spdk_rpc_client,
+                        nqn=request.nqn,
+                        ana_state="inaccessible",
+                        trtype=request.trtype,
+                        traddr=request.traddr,
+                        trsvcid=request.trsvcid,
+                        adrfam=request.adrfam,
+                        anagrpid=(x+1) )
+                   except Exception as ex:
+                        self.logger.error(f" set_listener_ana_state failed with: \n {ex}")
+                        raise
+
         if context:
             # Update gateway state
             try:

control/proto/gateway.proto

@@ -64,6 +64,8 @@ message create_subsystem_req {
 	string subsystem_nqn = 1;
 	string serial_number = 2;
 	int32 max_namespaces = 3;
+	bool  ana_reporting  = 4;
+	bool  enable_ha      = 5;
 }
 
 message delete_subsystem_req {
@@ -74,6 +76,7 @@ message add_namespace_req {
 	string subsystem_nqn = 1;
 	string bdev_name = 2;
 	optional int32 nsid = 3;
+        optional int32 anagrpid = 4;
 }
 
 message remove_namespace_req {

tests/test_cli.py

@@ -12,6 +12,7 @@
 serial = "SPDK00000000000001"
 host_list = ["nqn.2016-06.io.spdk:host1", "*"]
 nsid = "1"
+anagrpid = "2"
 trtype = "TCP"
 gateway_name = socket.gethostname()
 addr = "127.0.0.1"
@@ -98,3 +99,46 @@ def test_delete_subsystem(self, caplog, gateway):
         assert "Failed to delete" not in caplog.text
         cli(["delete_subsystem", "-n", subsystem2])
         assert "Failed to delete" not in caplog.text
+
+
+class TestCreateWithAna:
+    def test_create_bdev_ana(self, caplog, gateway):
+        cli(["create_bdev", "-i", image, "-p", pool, "-b", bdev])
+        assert "Failed to create" not in caplog.text
+
+
+    def test_create_subsystem_ana(self, caplog, gateway):
+        cli(["create_subsystem", "-n", subsystem, "-a", "true", "-t", "true"])
+        assert "Failed to create" not in caplog.text
+        cli(["get_subsystems"])
+        assert serial not in caplog.text
+
+    def test_add_namespace_ana(self, caplog, gateway):
+        cli(["add_namespace", "-n", subsystem, "-b", bdev, "-a", anagrpid])
+        assert "Failed to add" not in caplog.text
+
+    @pytest.mark.parametrize("listener", listener_list)
+    def test_create_listener_ana(self, caplog, listener, gateway):
+        cli(["create_listener", "-n", subsystem] + listener)
+        assert "Failed to create" not in caplog.text
+
+
+class TestDeleteAna:
+
+    @pytest.mark.parametrize("listener", listener_list)
+    def test_delete_listener_ana(self, caplog, listener, gateway):
+        cli(["delete_listener", "-n", subsystem] + listener)
+        assert "Failed to delete" not in caplog.text
+
+    def test_remove_namespace_ana(self, caplog, gateway):
+        cli(["remove_namespace", "-n", subsystem, "-i", nsid])
+        assert "Failed to remove" not in caplog.text
+
+    def test_delete_bdev_ana(self, caplog, gateway):
+        cli(["delete_bdev", "-b", bdev, "-f"])
+        assert "Failed to delete" not in caplog.text
+
+    def test_delete_subsystem_ana(self, caplog, gateway):
+        cli(["delete_subsystem", "-n", subsystem])
+        assert "Failed to delete" not in caplog.text
+