Handle IPv6 parameters in NVMe CLI

.env

@@ -27,7 +27,8 @@ NVMEOF_DESCRIPTION="Service to provide block storage on top of Ceph for platform
 NVMEOF_URL="https://github.com/ceph/ceph-nvmeof"
 NVMEOF_TAGS="ceph,nvme-of,nvme-of gateway,rbd,block storage"
 NVMEOF_WANTS="ceph,rbd"
-NVMEOF_IP_ADDRESS="192.168.13.3"
+NVMEOF_IP_ADDRESS=192.168.13.3
+NVMEOF_IPV6_ADDRESS=2001:db8::3
 NVMEOF_IO_PORT=4420
 NVMEOF_GW_PORT=5500
 NVMEOF_DISC_PORT=8009
@@ -57,10 +58,10 @@ CEPH_CLUSTER_VERSION="${CEPH_VERSION}"
 CEPH_VSTART_ARGS="--without-dashboard --memstore"
 
 # Demo settings
-RBD_POOL="rbd"
-RBD_IMAGE_NAME="demo_image"
-RBD_IMAGE_SIZE="10M"
-BDEV_NAME="demo_bdev"
+RBD_POOL=rbd
+RBD_IMAGE_NAME=demo_image
+RBD_IMAGE_SIZE=10M
+BDEV_NAME=demo_bdev
 NQN="nqn.2016-06.io.spdk:cnode1"
 SERIAL="SPDK00000000000001"
 

.github/workflows/build-container.yml

@@ -226,6 +226,7 @@ jobs:
           shopt -s expand_aliases
           eval $(make alias)
           nvmeof-cli get_subsystems
+          nvmeof-cli-ipv6 get_subsystems
 
       - name: Run bdevperf
         run: |

control/cli.py

@@ -17,7 +17,7 @@
 
 from .proto import gateway_pb2_grpc as pb2_grpc
 from .proto import gateway_pb2 as pb2
-
+from .config import GatewayConfig
 
 def argument(*name_or_flags, **kwargs):
     """Helper function to format arguments for argparse command decorator."""
@@ -124,7 +124,9 @@ def stub(self):
 
     def connect(self, host, port, client_key, client_cert, server_cert):
         """Connects to server and sets stub."""
-        server = "{}:{}".format(host, port)
+        # We need to enclose IPv6 addresses in brackets before concatenating a colon and port number to it
+        host = GatewayConfig.escape_address_if_ipv6(host)
+        server = f"{host}:{port}"
 
         if client_key and client_cert:
             # Create credentials for mutual TLS and a secure channel

control/config.py

@@ -64,3 +64,10 @@ def dump_config_file(self, logger):
                 self.conffile_logged = True
         except Exception:
             pass
+
+    # We need to enclose IPv6 addresses in brackets before concatenating a colon and port number to it
+    def escape_address_if_ipv6(addr) -> str:
+        ret_addr = addr
+        if ":" in addr and not addr.strip().startswith("["):
+            ret_addr = f"[{addr}]"
+        return ret_addr

control/grpc.py

@@ -23,6 +23,7 @@
 from google.protobuf import json_format
 from .proto import gateway_pb2 as pb2
 from .proto import gateway_pb2_grpc as pb2_grpc
+from .config import GatewayConfig
 
 MAX_ANA_GROUPS = 4
 
@@ -535,9 +536,10 @@ def matching_listener_exists(self, context, nqn, gw_name, trtype, traddr, trsvci
     def create_listener_safe(self, request, context=None):
         """Creates a listener for a subsystem at a given IP/Port."""
         ret = True
+        traddr = GatewayConfig.escape_address_if_ipv6(request.traddr)
         self.logger.info(f"Received request to create {request.gateway_name}"
                          f" {request.trtype} listener for {request.nqn} at"
-                         f" {request.traddr}:{request.trsvcid}.")
+                         f" {traddr}:{request.trsvcid}.")
         try:
             if request.gateway_name == self.gateway_name:
                 listener_already_exist = self.matching_listener_exists(
@@ -627,9 +629,10 @@ def delete_listener_safe(self, request, context=None):
         """Deletes a listener from a subsystem at a given IP/Port."""
 
         ret = True
+        traddr = GatewayConfig.escape_address_if_ipv6(request.traddr)
         self.logger.info(f"Received request to delete {request.gateway_name}"
                          f" {request.trtype} listener for {request.nqn} at"
-                         f" {request.traddr}:{request.trsvcid}.")
+                         f" {traddr}:{request.trsvcid}.")
         try:
             if request.gateway_name == self.gateway_name:
                 ret = rpc_nvmf.nvmf_subsystem_remove_listener(

control/server.py

@@ -28,6 +28,7 @@
 from .state import GatewayState, LocalGatewayState, OmapGatewayState, GatewayStateHandler
 from .grpc import GatewayService
 from .discovery import DiscoveryService
+from .config import GatewayConfig
 
 def sigchld_handler(signum, frame):
     """Handle SIGCHLD, runs when a spdk process terminates."""
@@ -158,6 +159,8 @@ def _add_server_listener(self):
         enable_auth = self.config.getboolean("gateway", "enable_auth")
         gateway_addr = self.config.get("gateway", "addr")
         gateway_port = self.config.get("gateway", "port")
+        # We need to enclose IPv6 addresses in brackets before concatenating a colon and port number to it
+        gateway_addr = GatewayConfig.escape_address_if_ipv6(gateway_addr)
         if enable_auth:
             # Read in key and certificates for authentication
             server_key = self.config.get("mtls", "server_key")

docker-compose.yaml

@@ -78,6 +78,7 @@ services:
     networks:
       default:
         ipv4_address: 192.168.13.2
+        ipv6_address: 2001:db8::2
   nvmeof-base:
     build:
       context: .
@@ -213,6 +214,8 @@ volumes:
   ceph-conf:
 networks:
   default:
+    enable_ipv6: true
     ipam:
       config:
         - subnet: 192.168.13.0/24
+        - subnet: 2001:0DB8::/112

mk/demo.mk

@@ -10,9 +10,12 @@ rbd: CMD = bash -c "rbd -p $(RBD_POOL) info $(RBD_IMAGE_NAME) || rbd -p $(RBD_PO
 demo: export NVMEOF_HOSTNAME != docker ps -q -f name=$(NVMEOF_CONTAINER_NAME)
 demo: rbd ## Expose RBD_IMAGE_NAME as NVMe-oF target
 	$(NVMEOF_CLI) create_bdev --pool $(RBD_POOL) --image $(RBD_IMAGE_NAME) --bdev $(BDEV_NAME)
+	$(NVMEOF_CLI_IPV6) create_bdev --pool $(RBD_POOL) --image $(RBD_IMAGE_NAME) --bdev $(BDEV_NAME)_ipv6
 	$(NVMEOF_CLI) create_subsystem --subnqn $(NQN)
 	$(NVMEOF_CLI) add_namespace --subnqn $(NQN) --bdev $(BDEV_NAME)
+	$(NVMEOF_CLI) add_namespace --subnqn $(NQN) --bdev $(BDEV_NAME)_ipv6
 	$(NVMEOF_CLI) create_listener --subnqn $(NQN) --gateway-name $(NVMEOF_HOSTNAME) --traddr $(NVMEOF_IP_ADDRESS) --trsvcid $(NVMEOF_IO_PORT)
+	$(NVMEOF_CLI_IPV6) create_listener --subnqn $(NQN) --gateway-name $(NVMEOF_HOSTNAME) --traddr '$(NVMEOF_IPV6_ADDRESS)' --trsvcid $(NVMEOF_IO_PORT) --adrfam IPV6
 	$(NVMEOF_CLI) add_host --subnqn $(NQN) --host "*"
 
 .PHONY: demo rbd

mk/misc.mk

@@ -2,8 +2,9 @@
 
 # nvmeof_cli
 NVMEOF_CLI = $(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) run --rm nvmeof-cli --server-address $(NVMEOF_IP_ADDRESS) --server-port $(NVMEOF_GW_PORT)
+NVMEOF_CLI_IPV6 = $(DOCKER_COMPOSE_ENV) $(DOCKER_COMPOSE) run --rm nvmeof-cli --server-address $(NVMEOF_IPV6_ADDRESS) --server-port $(NVMEOF_GW_PORT)
 
 alias: ## Print bash alias command for the nvmeof-cli. Usage: "eval $(make alias)"
-	@echo alias nvmeof-cli=\"$(NVMEOF_CLI)\"
+	@echo alias nvmeof-cli=\"$(NVMEOF_CLI)\" \; alias nvmeof-cli-ipv6=\'$(NVMEOF_CLI_IPV6)\'
 
 .PHONY: alias

tests/test_cli.py

@@ -12,11 +12,15 @@
 serial = "SPDK00000000000001"
 host_list = ["nqn.2016-06.io.spdk:host1", "*"]
 nsid = "1"
+nsid_ipv6 = "2"
 anagrpid = "2"
 trtype = "TCP"
 gateway_name = socket.gethostname()
 addr = "127.0.0.1"
+addr_ipv6 = "::1"
+server_addr_ipv6 = "2001:db8::3"
 listener_list = [["-g", gateway_name, "-a", addr, "-s", "5001"], ["-g", gateway_name, "-a", addr,"-s", "5002"]]
+listener_list_ipv6 = [["-g", gateway_name, "-a", addr_ipv6, "-s", "5003"], ["-g", gateway_name, "-a", addr_ipv6, "-s", "5004"]]
 config = "ceph-nvmeof.conf"
 
 @pytest.fixture(scope="module")
@@ -38,6 +42,10 @@ def test_get_subsystems(self, caplog, gateway):
         cli(["get_subsystems"])
         assert "Failed to get" not in caplog.text
 
+    def test_get_subsystems_ipv6(self, caplog, gateway):
+        cli(["--server-address", server_addr_ipv6, "get_subsystems"])
+        assert "Failed to get" not in caplog.text
+
 
 class TestCreate:
     def test_create_bdev(self, caplog, gateway):
@@ -46,6 +54,12 @@ def test_create_bdev(self, caplog, gateway):
         cli(["create_bdev", "-i", image, "-p", pool, "-b", bdev1])
         assert "Failed to create" not in caplog.text
 
+    def test_create_bdev_ipv6(self, caplog, gateway):
+        cli(["--server-address", server_addr_ipv6, "create_bdev", "-i", image, "-p", pool, "-b", bdev + "_ipv6"])
+        assert "Failed to create" not in caplog.text
+        cli(["--server-address", server_addr_ipv6, "create_bdev", "-i", image, "-p", pool, "-b", bdev1 + "_ipv6"])
+        assert "Failed to create" not in caplog.text
+
     def test_create_subsystem(self, caplog, gateway):
         cli(["create_subsystem", "-n", subsystem])
         assert "Failed to create" not in caplog.text
@@ -62,6 +76,12 @@ def test_add_namespace(self, caplog, gateway):
         cli(["add_namespace", "-n", subsystem, "-b", bdev1])
         assert "Failed to add" not in caplog.text
 
+    def test_add_namespace_ipv6(self, caplog, gateway):
+        cli(["--server-address", server_addr_ipv6, "add_namespace", "-n", subsystem, "-b", bdev + "_ipv6"])
+        assert "Failed to add" not in caplog.text
+        cli(["--server-address", server_addr_ipv6, "add_namespace", "-n", subsystem, "-b", bdev1 + "_ipv6"])
+        assert "Failed to add" not in caplog.text
+
     @pytest.mark.parametrize("host", host_list)
     def test_add_host(self, caplog, host):
         cli(["add_host", "-n", subsystem, "-t", host])
@@ -72,6 +92,11 @@ def test_create_listener(self, caplog, listener, gateway):
         cli(["create_listener", "-n", subsystem] + listener)
         assert "Failed to create" not in caplog.text
 
+    @pytest.mark.parametrize("listener_ipv6", listener_list_ipv6)
+    def test_create_listener_ipv6(self, caplog, listener_ipv6, gateway):
+        cli(["--server-address", server_addr_ipv6, "create_listener", "-n", subsystem, "--adrfam", "IPV6"] + listener_ipv6)
+        assert "Failed to create" not in caplog.text
+
 
 class TestDelete:
     @pytest.mark.parametrize("host", host_list)
@@ -84,15 +109,26 @@ def test_delete_listener(self, caplog, listener, gateway):
         cli(["delete_listener", "-n", subsystem] + listener)
         assert "Failed to delete" not in caplog.text
 
+    @pytest.mark.parametrize("listener_ipv6", listener_list_ipv6)
+    def test_delete_listener_ipv6(self, caplog, listener_ipv6, gateway):
+        cli(["--server-address", server_addr_ipv6, "delete_listener", "-n", subsystem, "--adrfam", "IPV6"] + listener_ipv6)
+        assert "Failed to delete" not in caplog.text
+
     def test_remove_namespace(self, caplog, gateway):
         cli(["remove_namespace", "-n", subsystem, "-i", nsid])
         assert "Failed to remove" not in caplog.text
+        cli(["remove_namespace", "-n", subsystem, "-i", nsid_ipv6])
+        assert "Failed to remove" not in caplog.text
 
     def test_delete_bdev(self, caplog, gateway):
         cli(["delete_bdev", "-b", bdev, "-f"])
         assert "Failed to delete" not in caplog.text
         cli(["delete_bdev", "-b", bdev1, "--force"])
         assert "Failed to delete" not in caplog.text
+        cli(["delete_bdev", "-b", bdev + "_ipv6", "-f"])
+        assert "Failed to delete" not in caplog.text
+        cli(["delete_bdev", "-b", bdev1 + "_ipv6", "--force"])
+        assert "Failed to delete" not in caplog.text
 
     def test_delete_subsystem(self, caplog, gateway):
         cli(["delete_subsystem", "-n", subsystem])
@@ -106,6 +142,10 @@ def test_create_bdev_ana(self, caplog, gateway):
         cli(["create_bdev", "-i", image, "-p", pool, "-b", bdev])
         assert "Failed to create" not in caplog.text
 
+    def test_create_bdev_ana_ipv6(self, caplog, gateway):
+        cli(["--server-address", server_addr_ipv6, "create_bdev", "-i", image, "-p", pool, "-b", bdev + "_ipv6"])
+        assert "Failed to create" not in caplog.text
+
 
     def test_create_subsystem_ana(self, caplog, gateway):
         cli(["create_subsystem", "-n", subsystem, "-a", "-t"])
@@ -137,6 +177,8 @@ def test_remove_namespace_ana(self, caplog, gateway):
     def test_delete_bdev_ana(self, caplog, gateway):
         cli(["delete_bdev", "-b", bdev, "-f"])
         assert "Failed to delete" not in caplog.text
+        cli(["delete_bdev", "-b", bdev + "_ipv6", "-f"])
+        assert "Failed to delete" not in caplog.text
 
     def test_delete_subsystem_ana(self, caplog, gateway):
         cli(["delete_subsystem", "-n", subsystem])