New NVMeOf CLI

Fixes #151

Signed-off-by: Gil Bregman <gbregman@il.ibm.com>

Author: gbregman

.env

@@ -1,5 +1,5 @@
 # Globals
-VERSION="0.0.6"
+VERSION="0.0.7"
 CEPH_VERSION="18.2.1"
 SPDK_VERSION="23.01.1"
 CONTAINER_REGISTRY="quay.io/ceph"

.github/workflows/build-container.yml

@@ -65,7 +65,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-          test: ["cli", "state", "multi_gateway", "server", "grpc", "omap_lock"]
+          test: ["cli", "state", "multi_gateway", "server", "grpc", "omap_lock", "old_omap"]
     runs-on: ubuntu-latest
     env:
       HUGEPAGES: 512 # for multi gateway test, approx 256 per gateway instance
@@ -223,13 +223,29 @@ jobs:
         run: |
           make demo OPTS=-T NVMEOF_CONTAINER_NAME="ceph-nvmeof_nvmeof_1"
 
-      - name: Get subsystems
+      - name: List resources
         run: |
           # https://github.com/actions/toolkit/issues/766
           shopt -s expand_aliases
           eval $(make alias)
-          nvmeof-cli get_subsystems
-          nvmeof-cli-ipv6 get_subsystems
+          cephnvmf subsystem list
+          subs=$(cephnvmf --output stdio --format json subsystem list | grep nqn | sed 's/"nqn": "//' | sed 's/",$//')
+          for sub in $subs
+          do
+              cephnvmf namespace list --subsystem $sub
+              cephnvmf listener list --subsystem $sub
+              cephnvmf host list --subsystem $sub
+          done
+
+      - name: Verify no HA
+        run: |
+          shopt -s expand_aliases
+          eval $(make alias)
+          ha_status=$(cephnvmf --output stdio --format json subsystem list | grep "enable_ha" | grep "true" | tr -d '\n\r')
+          if [ -n "$ha_status" ]; then
+              echo "Found a subsystem with HA enabled, failing"
+              exit 1
+          fi
 
       - name: Run bdevperf
         run: |
@@ -390,21 +406,20 @@ jobs:
 
           gw1=$(container_ip $GW1)
           echo ℹ️ Using GW RPC $GW1 address $gw1 port $NVMEOF_GW_PORT
-          cli_gw $gw1 get_subsystems
-          cli_gw $gw1 create_bdev --pool $RBD_POOL --image $RBD_IMAGE_NAME --bdev $BDEV_NAME
-          cli_gw $gw1 create_subsystem --subnqn $NQN --serial $SERIAL
-          cli_gw $gw1 add_namespace --subnqn $NQN --bdev $BDEV_NAME
+          cli_gw $gw1 subsystem list
+          cli_gw $gw1 subsystem add --subsystem $NQN --serial $SERIAL
+          cli_gw $gw1 namespace add --subsystem $NQN --rbd-pool $RBD_POOL --rbd-image $RBD_IMAGE_NAME
           for gw in $GW1 $GW2; do
             ip=$(container_ip $gw)
             name=$(container_id $gw) # default hostname - container id
             echo ℹ️ Create listener address $ip gateway $name
-            cli_gw $ip create_listener --subnqn $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT
+            cli_gw $ip listener add --subsystem $NQN --gateway-name $name --traddr $ip --trsvcid $NVMEOF_IO_PORT
           done
-          cli_gw $gw1 add_host --subnqn $NQN --host "*"
+          cli_gw $gw1 host add --subsystem $NQN --host "*"
           for gw in $GW1 $GW2; do
             ip=$(container_ip $gw)
             echo ℹ️ Subsystems for name $gw ip $ip
-            cli_gw $ip get_subsystems
+            cli_gw $ip subsystem list
           done
 
       - name: Run bdevperf discovery

Dockerfile

@@ -16,7 +16,10 @@ FROM quay.io/ceph/spdk:${NVMEOF_SPDK_VERSION:-NULL} AS base-gateway
 RUN \
     --mount=type=cache,target=/var/cache/dnf \
     --mount=type=cache,target=/var/lib/dnf \
-    dnf install -y python3-rados
+    dnf install -y python3-rados python3-pip
+RUN \
+    --mount=type=cache,target=/root/.cache/pip \
+    pip install -U pip setuptools
 ENTRYPOINT ["python3", "-m", "control"]
 CMD ["-c", "/src/ceph-nvmeof.conf"]
 
@@ -99,6 +102,17 @@ LABEL io.ceph.component="$NVMEOF_NAME" \
 
 ENV PYTHONPATH=$APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib
 
+RUN \
+    --mount=type=cache,target=/var/cache/dnf \
+    --mount=type=cache,target=/var/lib/dnf \
+    dnf install -y python3-pip
+RUN \
+    --mount=type=cache,target=/root/.cache/pip \
+    TABULATE_INSTALL=lib-only pip install -U --target $APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib tabulate
+RUN \
+    --mount=type=cache,target=/root/.cache/pip \
+    pip install -U --target $APPDIR/__pypackages__/$PYTHON_MAJOR.$PYTHON_MINOR/lib pyyaml
+
 WORKDIR $APPDIR
 
 #------------------------------------------------------------------------------

README.md

@@ -92,36 +92,25 @@ The following command executes all the steps required to set up the NVMe-oF envi
 
 ```bash
 $ make demo
-
-DOCKER_BUILDKIT=1 docker-compose exec ceph-vstart-cluster bash -c "rbd info demo_image || rbd create demo_image --size 10M"
+docker-compose  exec  ceph bash -c "rbd -p rbd info demo_image || rbd -p rbd create demo_image --size 10M"
 rbd: error opening image demo_image: (2) No such file or directory
-
-DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_bdev --pool rbd --image demo_image --bdev demo_bdev
-Creating nvmeof_ceph-nvmeof-cli_run ... done
-INFO:__main__:Created bdev demo_bdev: True
-
-DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_subsystem --subnqn nqn.2016-06.io.spdk:cnode1 --serial SPDK00000000000001
-Creating nvmeof_ceph-nvmeof-cli_run ... done
-INFO:__main__:Created subsystem nqn.2016-06.io.spdk:cnode1: True
-
-DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 add_namespace --subnqn nqn.2016-06.io.spdk:cnode1 --bdev demo_bdev
-Creating nvmeof_ceph-nvmeof-cli_run ... done
-INFO:__main__:Added namespace 1 to nqn.2016-06.io.spdk:cnode1: True
-
-DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 create_listener --subnqn nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420
-Creating nvmeof_ceph-nvmeof-cli_run ... done
-INFO:__main__:Created nqn.2016-06.io.spdk:cnode1 listener: True
-
-DOCKER_BUILDKIT=1 docker-compose run --rm ceph-nvmeof-cli --server-address ceph-nvmeof --server-port 5500 add_host --subnqn nqn.2016-06.io.spdk:cnode1 --host "*"
-Creating nvmeof_ceph-nvmeof-cli_run ... done
-INFO:__main__:Allowed open host access to nqn.2016-06.io.spdk:cnode1: True
+docker-compose  run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 subsystem add --subsystem "nqn.2016-06.io.spdk:cnode1" --ana-reporting --enable-ha
+Adding subsystem nqn.2016-06.io.spdk:cnode1: Successful
+docker-compose  run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 namespace add --subsystem "nqn.2016-06.io.spdk:cnode1" --rbd-pool rbd --rbd-image demo_image
+Adding namespace 1 to nqn.2016-06.io.spdk:cnode1, load balancing group 1: Successful
+docker-compose  run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 listener add --subsystem "nqn.2016-06.io.spdk:cnode1" --gateway-name fbca1a3d3ed8 --traddr 192.168.13.3 --trsvcid 4420
+Adding listener 192.168.13.3:4420 to nqn.2016-06.io.spdk:cnode1: Successful
+docker-compose  run --rm nvmeof-cli --server-address 2001:db8::3 --server-port 5500 listener add --subsystem "nqn.2016-06.io.spdk:cnode1" --gateway-name fbca1a3d3ed8 --traddr 2001:db8::3 --trsvcid 4420 --adrfam IPV6
+Adding listener [2001:db8::3]:4420 to nqn.2016-06.io.spdk:cnode1: Successful
+docker-compose  run --rm nvmeof-cli --server-address 192.168.13.3 --server-port 5500 host add --subsystem "nqn.2016-06.io.spdk:cnode1" --host "*"
+Allowing any host for nqn.2016-06.io.spdk:cnode1: Successful
 ```
 
 #### Manual Steps
 
 The same configuration can also be manually run:
 
-1. First of all, let's create the `nvmeof-cli` shortcut to interact with the NVMe-oF gateway:
+1. First of all, let's create the `cephnvmf` shortcut to interact with the NVMe-oF gateway:
 
     ```bash
     eval $(make alias)
@@ -133,34 +122,28 @@ The same configuration can also be manually run:
     make rbd
     ```
 
-1. Create a bdev (Block Device) from an RBD image:
-
-    ```bash
-    nvmeof-cli create_bdev --pool rbd --image demo_image --bdev demo_bdev
-    ```
-
 1. Create a subsystem:
 
     ```bash
-    nvmeof-cli create_subsystem --subnqn nqn.2016-06.io.spdk:cnode1 --serial SPDK00000000000001
+    cephnvmf subsystem add --subsystem nqn.2016-06.io.spdk:cnode1
     ```
 
 1. Add a namespace:
 
     ```bash
-    nvmeof-cli add_namespace --subnqn nqn.2016-06.io.spdk:cnode1 --bdev demo_bdev
+    cephnvmf namespace add --subsystem nqn.2016-06.io.spdk:cnode1 --rbd-pool rbd --rbd-image demo_image
     ```
 
 1. Create a listener so that NVMe initiators can connect to:
 
     ```bash
-    nvmeof-cli create_listener ---subnqn nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420
+    cephnvmf listener add ---subsystem nqn.2016-06.io.spdk:cnode1 -g gateway_name -a gateway_addr -s 4420
     ```
 
 1. Define which hosts can connect:
 
     ```bash
-    nvmeof-cli add_host --subnqn nqn.2016-06.io.spdk:cnode1 --host "*"
+    cephnvmf host add --subsystem nqn.2016-06.io.spdk:cnode1 --host "*"
     ```