From 99cfd739c3e6d2d7ab209175f8988adc5afed35e Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 22 Jan 2024 10:40:23 +0100 Subject: [PATCH 1/6] feat(storage/nvme): add create backend tcp path cmd Signed-off-by: Artsiom Koltun --- cmd/storage/backend/backend.go | 1 + cmd/storage/backend/nvme_path.go | 78 ++++++++++++++++++++++++++++++++ storage/backend/nvme_path.go | 59 ++++++++++++++++++++++++ 3 files changed, 138 insertions(+) create mode 100644 cmd/storage/backend/nvme_path.go create mode 100644 storage/backend/nvme_path.go diff --git a/cmd/storage/backend/backend.go b/cmd/storage/backend/backend.go index 0b45030..d06ee76 100644 --- a/cmd/storage/backend/backend.go +++ b/cmd/storage/backend/backend.go @@ -37,6 +37,7 @@ func newCreateNvmeCommand() *cobra.Command { } cmd.AddCommand(newCreateNvmeControllerCommand()) + cmd.AddCommand(newCreateNvmePathCommand()) return cmd } diff --git a/cmd/storage/backend/nvme_path.go b/cmd/storage/backend/nvme_path.go new file mode 100644 index 0000000..fe5f530 --- /dev/null +++ b/cmd/storage/backend/nvme_path.go @@ -0,0 +1,78 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2024 Intel Corporation + +// Package backend implements the CLI commands for storage backend +package backend + +import ( + "context" + "net" + + "github.com/opiproject/godpu/cmd/storage/common" + backendclient "github.com/opiproject/godpu/storage/backend" + "github.com/spf13/cobra" +) + +func newCreateNvmePathCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "path", + Aliases: []string{"p"}, + Short: "Creates nvme path to an external nvme device", + Args: cobra.NoArgs, + Run: func(c *cobra.Command, args []string) { + err := c.Help() + cobra.CheckErr(err) + }, + } + + cmd.AddCommand(newCreateNvmePathTCPCommand()) + + return cmd +} + +func newCreateNvmePathTCPCommand() *cobra.Command { + id := "" + nqn := "" + hostnqn := "" + controller := "" + var ip net.IP + var port uint16 + cmd := &cobra.Command{ + Use: "tcp", + Aliases: []string{"t"}, + Short: "Creates nvme path to a remote nvme TCP controller", + Args: cobra.NoArgs, + Run: func(c *cobra.Command, args []string) { + addr, err := c.Flags().GetString(common.AddrCmdLineArg) + cobra.CheckErr(err) + + timeout, err := c.Flags().GetDuration(common.TimeoutCmdLineArg) + cobra.CheckErr(err) + + client, err := backendclient.New(addr) + cobra.CheckErr(err) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + response, err := client.CreateNvmeTCPPath(ctx, id, controller, ip, port, nqn, hostnqn) + cobra.CheckErr(err) + + common.PrintResponse(response.Name) + }, + } + + cmd.Flags().StringVar(&id, "id", "", "id for created resource. Assigned by server if omitted.") + cmd.Flags().StringVar(&controller, "controller", "", "backend controller name for this path") + cmd.Flags().IPVar(&ip, "ip", nil, "ip address of the path to connect to.") + cmd.Flags().Uint16Var(&port, "port", 0, "port of the path to connect to.") + cmd.Flags().StringVar(&nqn, "nqn", "", "nqn of the target subsystem.") + cmd.Flags().StringVar(&hostnqn, "hostnqn", "", "host nqn") + + cobra.CheckErr(cmd.MarkFlagRequired("controller")) + cobra.CheckErr(cmd.MarkFlagRequired("ip")) + cobra.CheckErr(cmd.MarkFlagRequired("port")) + cobra.CheckErr(cmd.MarkFlagRequired("nqn")) + + return cmd +} diff --git a/storage/backend/nvme_path.go b/storage/backend/nvme_path.go new file mode 100644 index 0000000..6187db3 --- /dev/null +++ b/storage/backend/nvme_path.go @@ -0,0 +1,59 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2024 Intel Corporation + +// Package backend implements the go library for OPI backend storage +package backend + +import ( + "context" + "fmt" + "net" + + pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" +) + +// CreateNvmeTCPPath creates a path to nvme TCP controller +func (c *Client) CreateNvmeTCPPath( + ctx context.Context, + id string, + controller string, + ip net.IP, + port uint16, + nqn, hostnqn string, +) (*pb.NvmePath, error) { + var adrfam pb.NvmeAddressFamily + switch { + case ip.To4() != nil: + adrfam = pb.NvmeAddressFamily_NVME_ADDRESS_FAMILY_IPV4 + case ip.To16() != nil: + adrfam = pb.NvmeAddressFamily_NVME_ADDRESS_FAMILY_IPV6 + default: + return nil, fmt.Errorf("invalid ip address format: %v", ip) + } + + conn, connClose, err := c.connector.NewConn() + if err != nil { + return nil, err + } + defer connClose() + + client := c.createNvmeClient(conn) + response, err := client.CreateNvmePath( + ctx, + &pb.CreateNvmePathRequest{ + NvmePathId: id, + Parent: controller, + NvmePath: &pb.NvmePath{ + Trtype: pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP, + Traddr: ip.String(), + Fabrics: &pb.FabricsPath{ + Trsvcid: int64(port), + Subnqn: nqn, + Adrfam: adrfam, + Hostnqn: hostnqn, + }, + }, + }) + + return response, err +} From 4ec7fd4354d0a34a5a586151d164d508da839bf8 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 22 Jan 2024 10:41:04 +0100 Subject: [PATCH 2/6] test(storage/nvme): add client create backend tcp path tests Signed-off-by: Artsiom Koltun --- storage/backend/nvme_path_test.go | 155 ++++++++++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 storage/backend/nvme_path_test.go diff --git a/storage/backend/nvme_path_test.go b/storage/backend/nvme_path_test.go new file mode 100644 index 0000000..d7add6d --- /dev/null +++ b/storage/backend/nvme_path_test.go @@ -0,0 +1,155 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright (C) 2024 Intel Corporation + +// Package backend implements the go library for OPI backend storage +package backend + +import ( + "context" + "errors" + "fmt" + "net" + "testing" + "time" + + "github.com/opiproject/godpu/mocks" + pb "github.com/opiproject/opi-api/storage/v1alpha1/gen/go" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/protobuf/proto" +) + +func TestCreateNvmeTCPPath(t *testing.T) { + testControllerName := "remotenvme0Name" + testPathID := "remotepath0" + testIPv4 := net.ParseIP("127.0.0.1") + testNqn := "nqn.2019-06.io.spdk:0" + testPath := &pb.NvmePath{ + Trtype: pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP, + Traddr: "127.0.0.1", + Fabrics: &pb.FabricsPath{ + Trsvcid: 4420, + Subnqn: testNqn, + Adrfam: pb.NvmeAddressFamily_NVME_ADDRESS_FAMILY_IPV4, + }, + } + + tests := map[string]struct { + giveClientErr error + giveConnectorErr error + giveIP net.IP + wantErr error + wantRequest *pb.CreateNvmePathRequest + wantResponse *pb.NvmePath + wantConnClosed bool + }{ + "successful call": { + giveConnectorErr: nil, + giveClientErr: nil, + giveIP: testIPv4, + wantErr: nil, + wantRequest: &pb.CreateNvmePathRequest{ + Parent: testControllerName, + NvmePathId: testPathID, + NvmePath: proto.Clone(testPath).(*pb.NvmePath), + }, + wantResponse: proto.Clone(testPath).(*pb.NvmePath), + wantConnClosed: true, + }, + "client err": { + giveConnectorErr: nil, + giveClientErr: errors.New("Some client error"), + giveIP: testIPv4, + wantErr: errors.New("Some client error"), + wantRequest: &pb.CreateNvmePathRequest{ + Parent: testControllerName, + NvmePathId: testPathID, + NvmePath: proto.Clone(testPath).(*pb.NvmePath), + }, + wantResponse: nil, + wantConnClosed: true, + }, + "connector err": { + giveConnectorErr: errors.New("Some conn error"), + giveClientErr: nil, + giveIP: testIPv4, + wantErr: errors.New("Some conn error"), + wantRequest: nil, + wantResponse: nil, + wantConnClosed: false, + }, + "ipv6 address": { + giveConnectorErr: nil, + giveClientErr: nil, + giveIP: net.ParseIP("2001:db8::68"), + wantErr: nil, + wantRequest: &pb.CreateNvmePathRequest{ + Parent: testControllerName, + NvmePathId: testPathID, + NvmePath: &pb.NvmePath{ + Trtype: pb.NvmeTransportType_NVME_TRANSPORT_TYPE_TCP, + Traddr: "2001:db8::68", + Fabrics: &pb.FabricsPath{ + Trsvcid: 4420, + Subnqn: testNqn, + Adrfam: pb.NvmeAddressFamily_NVME_ADDRESS_FAMILY_IPV6, + }, + }, + }, + wantResponse: &pb.NvmePath{}, + wantConnClosed: true, + }, + "invalid ip": { + giveConnectorErr: nil, + giveClientErr: nil, + giveIP: net.ParseIP("invalid ip"), + wantErr: fmt.Errorf("invalid ip address format: %v", ""), + wantRequest: nil, + wantResponse: nil, + wantConnClosed: false, + }, + } + + for testName, tt := range tests { + t.Run(testName, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + mockClient := mocks.NewNvmeRemoteControllerServiceClient(t) + if tt.wantRequest != nil { + toReturn := proto.Clone(tt.wantResponse).(*pb.NvmePath) + mockClient.EXPECT().CreateNvmePath(ctx, tt.wantRequest). + Return(toReturn, tt.giveClientErr) + } + + connClosed := false + mockConn := mocks.NewConnector(t) + mockConn.EXPECT().NewConn().Return( + &grpc.ClientConn{}, + func() { connClosed = true }, + tt.giveConnectorErr, + ).Maybe() + + c, _ := NewWithArgs( + mockConn, + func(grpc.ClientConnInterface) pb.NvmeRemoteControllerServiceClient { + return mockClient + }, + ) + + response, err := c.CreateNvmeTCPPath( + ctx, + testPathID, + testControllerName, + tt.giveIP, + 4420, + testNqn, + "", + ) + + require.Equal(t, tt.wantErr, err) + require.True(t, proto.Equal(response, tt.wantResponse)) + require.Equal(t, tt.wantConnClosed, connClosed) + }) + } +} From 00994b1e4fad5b2676893990a14334f7ed329053 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 22 Jan 2024 10:50:25 +0100 Subject: [PATCH 3/6] feat(storage/nvme): add delete backend path cmd Signed-off-by: Artsiom Koltun --- cmd/storage/backend/backend.go | 1 + cmd/storage/backend/nvme_path.go | 34 ++++++++++++++++++++++++++++++++ storage/backend/nvme_path.go | 23 +++++++++++++++++++++ 3 files changed, 58 insertions(+) diff --git a/cmd/storage/backend/backend.go b/cmd/storage/backend/backend.go index d06ee76..17f45c4 100644 --- a/cmd/storage/backend/backend.go +++ b/cmd/storage/backend/backend.go @@ -73,6 +73,7 @@ func newDeleteNvmeCommand() *cobra.Command { } cmd.AddCommand(newDeleteNvmeControllerCommand()) + cmd.AddCommand(newDeleteNvmePathCommand()) return cmd } diff --git a/cmd/storage/backend/nvme_path.go b/cmd/storage/backend/nvme_path.go index fe5f530..9ad344f 100644 --- a/cmd/storage/backend/nvme_path.go +++ b/cmd/storage/backend/nvme_path.go @@ -76,3 +76,37 @@ func newCreateNvmePathTCPCommand() *cobra.Command { return cmd } + +func newDeleteNvmePathCommand() *cobra.Command { + name := "" + allowMissing := false + cmd := &cobra.Command{ + Use: "path", + Aliases: []string{"p"}, + Short: "Deletes nvme path to an external nvme device", + Args: cobra.NoArgs, + Run: func(c *cobra.Command, args []string) { + addr, err := c.Flags().GetString(common.AddrCmdLineArg) + cobra.CheckErr(err) + + timeout, err := c.Flags().GetDuration(common.TimeoutCmdLineArg) + cobra.CheckErr(err) + + client, err := backendclient.New(addr) + cobra.CheckErr(err) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + err = client.DeleteNvmePath(ctx, name, allowMissing) + cobra.CheckErr(err) + }, + } + + cmd.Flags().StringVar(&name, "name", "", "name of deleted nvme path") + cmd.Flags().BoolVar(&allowMissing, "allowMissing", false, "cmd succeeds if attempts to delete a resource that is not present") + + cobra.CheckErr(cmd.MarkFlagRequired("name")) + + return cmd +} diff --git a/storage/backend/nvme_path.go b/storage/backend/nvme_path.go index 6187db3..0c5a75c 100644 --- a/storage/backend/nvme_path.go +++ b/storage/backend/nvme_path.go @@ -57,3 +57,26 @@ func (c *Client) CreateNvmeTCPPath( return response, err } + +// DeleteNvmePath deletes an nvme path to an external nvme controller +func (c *Client) DeleteNvmePath( + ctx context.Context, + name string, + allowMissing bool, +) error { + conn, connClose, err := c.connector.NewConn() + if err != nil { + return err + } + defer connClose() + + client := c.createNvmeClient(conn) + _, err = client.DeleteNvmePath( + ctx, + &pb.DeleteNvmePathRequest{ + Name: name, + AllowMissing: allowMissing, + }) + + return err +} From ee5fcdb2297cae95e06a9f0019834352b8bb7571 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 22 Jan 2024 11:07:27 +0100 Subject: [PATCH 4/6] test(storage/nvme): add client delete backend path tests Signed-off-by: Artsiom Koltun --- storage/backend/nvme_path_test.go | 71 +++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/storage/backend/nvme_path_test.go b/storage/backend/nvme_path_test.go index d7add6d..3160e77 100644 --- a/storage/backend/nvme_path_test.go +++ b/storage/backend/nvme_path_test.go @@ -17,6 +17,7 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/grpc" "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/emptypb" ) func TestCreateNvmeTCPPath(t *testing.T) { @@ -153,3 +154,73 @@ func TestCreateNvmeTCPPath(t *testing.T) { }) } } + +func TestDeleteNvmePath(t *testing.T) { + testPathName := "path0" + testRequest := &pb.DeleteNvmePathRequest{ + Name: testPathName, + AllowMissing: true, + } + tests := map[string]struct { + giveClientErr error + giveConnectorErr error + wantErr error + wantRequest *pb.DeleteNvmePathRequest + wantConnClosed bool + }{ + "successful call": { + giveConnectorErr: nil, + giveClientErr: nil, + wantErr: nil, + wantRequest: proto.Clone(testRequest).(*pb.DeleteNvmePathRequest), + wantConnClosed: true, + }, + "client err": { + giveConnectorErr: nil, + giveClientErr: errors.New("Some client error"), + wantErr: errors.New("Some client error"), + wantRequest: proto.Clone(testRequest).(*pb.DeleteNvmePathRequest), + wantConnClosed: true, + }, + "connector err": { + giveConnectorErr: errors.New("Some conn error"), + giveClientErr: nil, + wantErr: errors.New("Some conn error"), + wantRequest: nil, + wantConnClosed: false, + }, + } + + for testName, tt := range tests { + t.Run(testName, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + mockClient := mocks.NewNvmeRemoteControllerServiceClient(t) + if tt.wantRequest != nil { + mockClient.EXPECT().DeleteNvmePath(ctx, tt.wantRequest). + Return(&emptypb.Empty{}, tt.giveClientErr) + } + + connClosed := false + mockConn := mocks.NewConnector(t) + mockConn.EXPECT().NewConn().Return( + &grpc.ClientConn{}, + func() { connClosed = true }, + tt.giveConnectorErr, + ) + + c, _ := NewWithArgs( + mockConn, + func(grpc.ClientConnInterface) pb.NvmeRemoteControllerServiceClient { + return mockClient + }, + ) + + err := c.DeleteNvmePath(ctx, testPathName, true) + + require.Equal(t, tt.wantErr, err) + require.Equal(t, tt.wantConnClosed, connClosed) + }) + } +} From 03bedcdcbfe1b2742ce974849f6f2663f14ac6f7 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 22 Jan 2024 12:07:17 +0100 Subject: [PATCH 5/6] feat(storage/nvme): add create backend pcie path cmd Signed-off-by: Artsiom Koltun --- cmd/storage/backend/nvme_path.go | 40 ++++++++++++++++++++++++++++++++ storage/backend/nvme_path.go | 28 ++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/cmd/storage/backend/nvme_path.go b/cmd/storage/backend/nvme_path.go index 9ad344f..282ba4c 100644 --- a/cmd/storage/backend/nvme_path.go +++ b/cmd/storage/backend/nvme_path.go @@ -26,6 +26,7 @@ func newCreateNvmePathCommand() *cobra.Command { } cmd.AddCommand(newCreateNvmePathTCPCommand()) + cmd.AddCommand(newCreateNvmePathPcieCommand()) return cmd } @@ -77,6 +78,45 @@ func newCreateNvmePathTCPCommand() *cobra.Command { return cmd } +func newCreateNvmePathPcieCommand() *cobra.Command { + id := "" + controller := "" + bdf := "" + cmd := &cobra.Command{ + Use: "pcie", + Aliases: []string{"p"}, + Short: "Creates nvme path to PCIe controller", + Args: cobra.NoArgs, + Run: func(c *cobra.Command, args []string) { + addr, err := c.Flags().GetString(common.AddrCmdLineArg) + cobra.CheckErr(err) + + timeout, err := c.Flags().GetDuration(common.TimeoutCmdLineArg) + cobra.CheckErr(err) + + client, err := backendclient.New(addr) + cobra.CheckErr(err) + + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + response, err := client.CreateNvmePciePath(ctx, id, controller, bdf) + cobra.CheckErr(err) + + common.PrintResponse(response.Name) + }, + } + + cmd.Flags().StringVar(&id, "id", "", "id for created resource. Assigned by server if omitted.") + cmd.Flags().StringVar(&controller, "controller", "", "backend controller name for this path") + cmd.Flags().StringVar(&bdf, "bdf", "", "bdf PCI address of NVMe/PCIe controller") + + cobra.CheckErr(cmd.MarkFlagRequired("controller")) + cobra.CheckErr(cmd.MarkFlagRequired("bdf")) + + return cmd +} + func newDeleteNvmePathCommand() *cobra.Command { name := "" allowMissing := false diff --git a/storage/backend/nvme_path.go b/storage/backend/nvme_path.go index 0c5a75c..863a992 100644 --- a/storage/backend/nvme_path.go +++ b/storage/backend/nvme_path.go @@ -58,6 +58,34 @@ func (c *Client) CreateNvmeTCPPath( return response, err } +// CreateNvmePciePath creates a path to nvme PCIe controller +func (c *Client) CreateNvmePciePath( + ctx context.Context, + id string, + controller string, + bdf string, +) (*pb.NvmePath, error) { + conn, connClose, err := c.connector.NewConn() + if err != nil { + return nil, err + } + defer connClose() + + client := c.createNvmeClient(conn) + response, err := client.CreateNvmePath( + ctx, + &pb.CreateNvmePathRequest{ + NvmePathId: id, + Parent: controller, + NvmePath: &pb.NvmePath{ + Trtype: pb.NvmeTransportType_NVME_TRANSPORT_TYPE_PCIE, + Traddr: bdf, + }, + }) + + return response, err +} + // DeleteNvmePath deletes an nvme path to an external nvme controller func (c *Client) DeleteNvmePath( ctx context.Context, From 368fc65f147d068dedd597b53cdb6940106aea18 Mon Sep 17 00:00:00 2001 From: Artsiom Koltun Date: Mon, 22 Jan 2024 12:15:44 +0100 Subject: [PATCH 6/6] test(storage/nvme): add client create backend pcie path tests Signed-off-by: Artsiom Koltun --- storage/backend/nvme_path_test.go | 92 +++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/storage/backend/nvme_path_test.go b/storage/backend/nvme_path_test.go index 3160e77..36b39da 100644 --- a/storage/backend/nvme_path_test.go +++ b/storage/backend/nvme_path_test.go @@ -155,6 +155,98 @@ func TestCreateNvmeTCPPath(t *testing.T) { } } +func TestCreateNvmePciePath(t *testing.T) { + testControllerName := "remotenvme0Name" + testPathID := "remotepath0" + testBDF := "0000:64:00.0" + testPath := &pb.NvmePath{ + Trtype: pb.NvmeTransportType_NVME_TRANSPORT_TYPE_PCIE, + Traddr: testBDF, + } + + tests := map[string]struct { + giveClientErr error + giveConnectorErr error + wantErr error + wantRequest *pb.CreateNvmePathRequest + wantResponse *pb.NvmePath + wantConnClosed bool + }{ + "successful call": { + giveConnectorErr: nil, + giveClientErr: nil, + wantErr: nil, + wantRequest: &pb.CreateNvmePathRequest{ + Parent: testControllerName, + NvmePathId: testPathID, + NvmePath: proto.Clone(testPath).(*pb.NvmePath), + }, + wantResponse: proto.Clone(testPath).(*pb.NvmePath), + wantConnClosed: true, + }, + "client err": { + giveConnectorErr: nil, + giveClientErr: errors.New("Some client error"), + wantErr: errors.New("Some client error"), + wantRequest: &pb.CreateNvmePathRequest{ + Parent: testControllerName, + NvmePathId: testPathID, + NvmePath: proto.Clone(testPath).(*pb.NvmePath), + }, + wantResponse: nil, + wantConnClosed: true, + }, + "connector err": { + giveConnectorErr: errors.New("Some conn error"), + giveClientErr: nil, + wantErr: errors.New("Some conn error"), + wantRequest: nil, + wantResponse: nil, + wantConnClosed: false, + }, + } + + for testName, tt := range tests { + t.Run(testName, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + mockClient := mocks.NewNvmeRemoteControllerServiceClient(t) + if tt.wantRequest != nil { + toReturn := proto.Clone(tt.wantResponse).(*pb.NvmePath) + mockClient.EXPECT().CreateNvmePath(ctx, tt.wantRequest). + Return(toReturn, tt.giveClientErr) + } + + connClosed := false + mockConn := mocks.NewConnector(t) + mockConn.EXPECT().NewConn().Return( + &grpc.ClientConn{}, + func() { connClosed = true }, + tt.giveConnectorErr, + ).Maybe() + + c, _ := NewWithArgs( + mockConn, + func(grpc.ClientConnInterface) pb.NvmeRemoteControllerServiceClient { + return mockClient + }, + ) + + response, err := c.CreateNvmePciePath( + ctx, + testPathID, + testControllerName, + testBDF, + ) + + require.Equal(t, tt.wantErr, err) + require.True(t, proto.Equal(response, tt.wantResponse)) + require.Equal(t, tt.wantConnClosed, connClosed) + }) + } +} + func TestDeleteNvmePath(t *testing.T) { testPathName := "path0" testRequest := &pb.DeleteNvmePathRequest{