From b4d2391a0a86c7f6e8e54d11369ff380cc22671a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Mon, 18 Nov 2024 10:55:45 +0000 Subject: [PATCH] Format code with black --- control/__main__.py | 10 +- control/cephutils.py | 118 +- control/cli.py | 1419 ++++++++++++++------ control/config.py | 7 +- control/discovery.py | 479 ++++--- control/grpc.py | 2323 +++++++++++++++++++++++---------- control/prometheus.py | 262 ++-- control/server.py | 319 +++-- control/state.py | 540 +++++--- control/utils.py | 216 ++- tests/conftest.py | 17 +- tests/test_cli.py | 1851 +++++++++++++++++++++++--- tests/test_cli_change_keys.py | 119 +- tests/test_cli_change_lb.py | 546 +++++++- tests/test_dhchap.py | 492 ++++++- tests/test_grpc.py | 156 ++- tests/test_log_files.py | 9 +- tests/test_multi_gateway.py | 104 +- tests/test_nsid.py | 127 +- tests/test_omap_lock.py | 339 +++-- tests/test_psk.py | 208 ++- tests/test_server.py | 35 +- tests/test_state.py | 58 +- 23 files changed, 7459 insertions(+), 2295 deletions(-) diff --git a/control/__main__.py b/control/__main__.py index 5ec1c091..0ed856d4 100644 --- a/control/__main__.py +++ b/control/__main__.py @@ -12,10 +12,12 @@ from .config import GatewayConfig from .utils import GatewayLogger -if __name__ == '__main__': - parser = argparse.ArgumentParser(prog="python3 -m control", - description="Manage NVMe gateways", - formatter_class=argparse.ArgumentDefaultsHelpFormatter) +if __name__ == "__main__": + parser = argparse.ArgumentParser( + prog="python3 -m control", + description="Manage NVMe gateways", + formatter_class=argparse.ArgumentDefaultsHelpFormatter, + ) parser.add_argument( "-c", "--config", diff --git a/control/cephutils.py b/control/cephutils.py index 542fc223..a20a071f 100644 --- a/control/cephutils.py +++ b/control/cephutils.py @@ -15,28 +15,31 @@ import json from .utils import GatewayLogger + class CephUtils: - """Miscellaneous functions which connect to Ceph - """ + """Miscellaneous functions which connect to Ceph""" def __init__(self, config): self.logger = GatewayLogger(config).logger - self.ceph_conf = config.get_with_default("ceph", "config_file", "/etc/ceph/ceph.conf") + self.ceph_conf = config.get_with_default( + "ceph", "config_file", "/etc/ceph/ceph.conf" + ) self.rados_id = config.get_with_default("ceph", "id", "") self.anagroup_list = [] self.last_sent = time.time() def execute_ceph_monitor_command(self, cmd): - self.logger.debug(f"Execute monitor command: {cmd}") - with rados.Rados(conffile=self.ceph_conf, rados_id=self.rados_id) as cluster: - rply = cluster.mon_command(cmd, b'') + self.logger.debug(f"Execute monitor command: {cmd}") + with rados.Rados(conffile=self.ceph_conf, rados_id=self.rados_id) as cluster: + rply = cluster.mon_command(cmd, b"") self.logger.debug(f"Monitor reply: {rply}") return rply + def get_gw_id_owner_ana_group(self, pool, group, anagrp): - str = '{' + f'"prefix":"nvme-gw show", "pool":"{pool}", "group":"{group}"' + '}' + str = "{" + f'"prefix":"nvme-gw show", "pool":"{pool}", "group":"{group}"' + "}" self.logger.debug(f"nvme-show string: {str}") rply = self.execute_ceph_monitor_command(str) - self.logger.debug(f"reply \"{rply}\"") + self.logger.debug(f'reply "{rply}"') conv_str = rply[1].decode() data = json.loads(conv_str) @@ -45,31 +48,35 @@ def get_gw_id_owner_ana_group(self, pool, group, anagrp): comp_str = f"{anagrp}: ACTIVE" for gateway in data["Created Gateways:"]: if comp_str in gateway["ana states"]: - gw_id = gateway["gw-id"] - self.logger.debug(f"found gw owner of anagrp {anagrp}: gw {gw_id}") - break + gw_id = gateway["gw-id"] + self.logger.debug(f"found gw owner of anagrp {anagrp}: gw {gw_id}") + break return gw_id def get_number_created_gateways(self, pool, group): now = time.time() - if (now - self.last_sent) < 10 and self.anagroup_list : - self.logger.info(f"Caching response of the monitor: {self.anagroup_list}") - return self.anagroup_list - else : + if (now - self.last_sent) < 10 and self.anagroup_list: + self.logger.info(f"Caching response of the monitor: {self.anagroup_list}") + return self.anagroup_list + else: try: self.anagroup_list = [] self.last_sent = now - str = '{' + f'"prefix":"nvme-gw show", "pool":"{pool}", "group":"{group}"' + '}' + str = ( + "{" + + f'"prefix":"nvme-gw show", "pool":"{pool}", "group":"{group}"' + + "}" + ) self.logger.debug(f"nvme-show string: {str}") rply = self.execute_ceph_monitor_command(str) - self.logger.debug(f"reply \"{rply}\"") + self.logger.debug(f'reply "{rply}"') conv_str = rply[1].decode() pos = conv_str.find("[") if pos != -1: new_str = conv_str[pos + len("[") :] - pos = new_str.find("]") - new_str = new_str[: pos].strip() - int_str_list = new_str.split(' ') + pos = new_str.find("]") + new_str = new_str[:pos].strip() + int_str_list = new_str.split(" ") self.logger.debug(f"new_str : {new_str}") for x in int_str_list: self.anagroup_list.append(int(x)) @@ -86,9 +93,17 @@ def get_number_created_gateways(self, pool, group): def fetch_and_display_ceph_version(self): try: rply = self.execute_ceph_monitor_command('{"prefix":"mon versions"}') - ceph_ver = rply[1].decode().removeprefix("{").strip().split(":")[0].removeprefix('"').removesuffix('"') + ceph_ver = ( + rply[1] + .decode() + .removeprefix("{") + .strip() + .split(":")[0] + .removeprefix('"') + .removesuffix('"') + ) ceph_ver = ceph_ver.removeprefix("ceph version ") - self.logger.info(f"Connected to Ceph with version \"{ceph_ver}\"") + self.logger.info(f'Connected to Ceph with version "{ceph_ver}"') except Exception: self.logger.exception(f"Failure fetching Ceph version:") pass @@ -96,7 +111,9 @@ def fetch_and_display_ceph_version(self): def fetch_ceph_fsid(self) -> str: fsid = None try: - with rados.Rados(conffile=self.ceph_conf, rados_id=self.rados_id) as cluster: + with rados.Rados( + conffile=self.ceph_conf, rados_id=self.rados_id + ) as cluster: fsid = cluster.get_fsid() except Exception: self.logger.exception(f"Failure fetching Ceph fsid:") @@ -105,7 +122,9 @@ def fetch_ceph_fsid(self) -> str: def pool_exists(self, pool) -> bool: try: - with rados.Rados(conffile=self.ceph_conf, rados_id=self.rados_id) as cluster: + with rados.Rados( + conffile=self.ceph_conf, rados_id=self.rados_id + ) as cluster: if cluster.pool_exists(pool): return True except Exception: @@ -116,8 +135,8 @@ def pool_exists(self, pool) -> bool: def service_daemon_register(self, cluster, metadata): try: - if cluster: # rados client - daemon_name = metadata['id'] + if cluster: # rados client + daemon_name = metadata["id"] cluster.service_daemon_register("nvmeof", daemon_name, metadata) self.logger.info(f"Registered {daemon_name} to service_map!") except Exception: @@ -128,26 +147,34 @@ def service_daemon_update(self, cluster, status_buffer): if cluster and status_buffer: cluster.service_daemon_update(status_buffer) except Exception: - self.logger.exception(f"Can't update daemon status to service_map!") + self.logger.exception(f"Can't update daemon status to service_map!") def create_image(self, pool_name, image_name, size) -> bool: # Check for pool existence in advance as we don't create it if it's not there if not self.pool_exists(pool_name): - raise rbd.ImageNotFound(f"Pool {pool_name} doesn't exist", errno = errno.ENODEV) + raise rbd.ImageNotFound( + f"Pool {pool_name} doesn't exist", errno=errno.ENODEV + ) image_exists = False try: image_size = self.get_image_size(pool_name, image_name) image_exists = True except rbd.ImageNotFound: - self.logger.debug(f"Image {pool_name}/{image_name} doesn't exist, will create it using size {size}") + self.logger.debug( + f"Image {pool_name}/{image_name} doesn't exist, will create it using size {size}" + ) pass if image_exists: if image_size != size: - raise rbd.ImageExists(f"Image {pool_name}/{image_name} already exists with a size of {image_size} bytes which differs from the requested size of {size} bytes", - errno = errno.EEXIST) - return False # Image exists with an idetical size, there is nothing to do here + raise rbd.ImageExists( + f"Image {pool_name}/{image_name} already exists with a size of {image_size} bytes which differs from the requested size of {size} bytes", + errno=errno.EEXIST, + ) + return ( + False # Image exists with an idetical size, there is nothing to do here + ) with rados.Rados(conffile=self.ceph_conf, rados_id=self.rados_id) as cluster: with cluster.open_ioctx(pool_name) as ioctx: @@ -155,11 +182,17 @@ def create_image(self, pool_name, image_name, size) -> bool: try: rbd_inst.create(ioctx, image_name, size) except rbd.ImageExists as ex: - self.logger.exception(f"Image {pool_name}/{image_name} was created just now") - raise rbd.ImageExists(f"Image {pool_name}/{image_name} was just created by someone else, please retry", - errno = errno.EAGAIN) + self.logger.exception( + f"Image {pool_name}/{image_name} was created just now" + ) + raise rbd.ImageExists( + f"Image {pool_name}/{image_name} was just created by someone else, please retry", + errno=errno.EAGAIN, + ) except Exception as ex: - self.logger.exception(f"Can't create image {pool_name}/{image_name}") + self.logger.exception( + f"Can't create image {pool_name}/{image_name}" + ) raise ex return True @@ -167,7 +200,9 @@ def create_image(self, pool_name, image_name, size) -> bool: def get_image_size(self, pool_name, image_name) -> int: image_size = 0 if not self.pool_exists(pool_name): - raise rbd.ImageNotFound(f"Pool {pool_name} doesn't exist", errno = errno.ENODEV) + raise rbd.ImageNotFound( + f"Pool {pool_name} doesn't exist", errno=errno.ENODEV + ) with rados.Rados(conffile=self.ceph_conf, rados_id=self.rados_id) as cluster: with cluster.open_ioctx(pool_name) as ioctx: @@ -176,9 +211,14 @@ def get_image_size(self, pool_name, image_name) -> int: with rbd.Image(ioctx, image_name) as img: image_size = img.size() except rbd.ImageNotFound: - raise rbd.ImageNotFound(f"Image {pool_name}/{image_name} doesn't exist", errno = errno.ENODEV) + raise rbd.ImageNotFound( + f"Image {pool_name}/{image_name} doesn't exist", + errno=errno.ENODEV, + ) except Exception as ex: - self.logger.exception(f"Error while trying to get the size of image {pool_name}/{image_name}") + self.logger.exception( + f"Error while trying to get the size of image {pool_name}/{image_name}" + ) raise ex return image_size diff --git a/control/cli.py b/control/cli.py index 2eded857..d74e6e08 100644 --- a/control/cli.py +++ b/control/cli.py @@ -25,16 +25,19 @@ from .utils import GatewayUtils from .utils import GatewayEnumUtils -BASE_GATEWAY_VERSION="1.1.0" +BASE_GATEWAY_VERSION = "1.1.0" + def errprint(msg): - print(msg, file = sys.stderr) + print(msg, file=sys.stderr) + def argument(*name_or_flags, **kwargs): """Helper function to format arguments for argparse command decorator.""" return (list(name_or_flags), kwargs) -def get_enum_keys_list(e_type, include_first = True): + +def get_enum_keys_list(e_type, include_first=True): k_list = [] for k in e_type.keys(): k_list.append(k.lower()) @@ -44,6 +47,7 @@ def get_enum_keys_list(e_type, include_first = True): return k_list + def break_string(s, delim, count): start = 0 for i in range(count): @@ -51,14 +55,15 @@ def break_string(s, delim, count): if ind < 0: return s start = ind + 1 - return s[0:ind + 1] + "\n" + s[ind + 1:] + return s[0 : ind + 1] + "\n" + s[ind + 1 :] + class ErrorCatchingArgumentParser(argparse.ArgumentParser): def __init__(self, *args, **kwargs): self.logger = logging.getLogger(__name__) super(ErrorCatchingArgumentParser, self).__init__(*args, **kwargs) - def exit(self, status = 0, message = None): + def exit(self, status=0, message=None): if status != 0: if message: self.logger.error(message) @@ -73,6 +78,7 @@ def error(self, message): self.logger.error(f"error: {message}") exit(2) + class Parser: """Class to simplify creation of client CLI. @@ -83,60 +89,66 @@ class Parser: def __init__(self): self.parser = ErrorCatchingArgumentParser( - prog="python3 -m control.cli", - description="CLI to manage NVMe gateways") + prog="python3 -m control.cli", description="CLI to manage NVMe gateways" + ) self.parser.add_argument( "--format", help="CLI output format", type=str, default="text", choices=["text", "json", "yaml", "plain", "python"], - required=False) + required=False, + ) self.parser.add_argument( "--output", help="CLI output method", type=str, default="log", choices=["log", "stdio"], - required=False) + required=False, + ) self.parser.add_argument( "--log-level", help="CLI log level", type=str, default="info", choices=get_enum_keys_list(pb2.GwLogLevel, False), - required=False) + required=False, + ) self.parser.add_argument( "--server-address", - default=(os.getenv('CEPH_NVMEOF_SERVER_ADDRESS') or "localhost"), + default=(os.getenv("CEPH_NVMEOF_SERVER_ADDRESS") or "localhost"), type=str, help="Server address (default: CEPH_NVMEOF_SERVER_ADDRESS env variable or 'localhost')", ) self.parser.add_argument( "--server-port", - default=int(os.getenv('CEPH_NVMEOF_SERVER_PORT') or "5500"), + default=int(os.getenv("CEPH_NVMEOF_SERVER_PORT") or "5500"), type=int, help="Server port (default: CEPH_NVMEOF_SERVER_PORT env variable or '5500')", ) self.parser.add_argument( "--client-key", type=argparse.FileType("rb"), - help="Path to the client key file") + help="Path to the client key file", + ) self.parser.add_argument( "--client-cert", type=argparse.FileType("rb"), - help="Path to the client certificate file") + help="Path to the client certificate file", + ) self.parser.add_argument( "--server-cert", type=argparse.FileType("rb"), - help="Path to the server certificate file" + help="Path to the server certificate file", ) self.parser.add_argument( - "--verbose", - help="Run CLI in verbose mode", - action='store_true') + "--verbose", help="Run CLI in verbose mode", action="store_true" + ) - self.subparsers = self.parser.add_subparsers(title="Commands", dest="subcommand") + self.subparsers = self.parser.add_subparsers( + title="Commands", dest="subcommand" + ) def cmd(self, actions=[], aliases=[], hlp=None): """Decorator to create an argparse command. @@ -150,8 +162,9 @@ def decorator(func): if hlp: helpstr = hlp - parser = self.subparsers.add_parser(func.__name__, - description=helpstr, aliases=aliases, help=helpstr) + parser = self.subparsers.add_parser( + func.__name__, description=helpstr, aliases=aliases, help=helpstr + ) subp = parser.add_subparsers(title="Action", dest="action") for act in actions: act_name = act["name"] @@ -167,7 +180,9 @@ def wrapper(*args, **kwargs): return func(*args, **kwargs) except grpc.RpcError as e: self.parser.error( - f"{func.__name__} failed: code={e.code()} message={e.details()}") + f"{func.__name__} failed: code={e.code()} message={e.details()}" + ) + parser.set_defaults(func=wrapper) return wrapper @@ -193,7 +208,7 @@ class GatewayClient: def __init__(self): self._stub = None - logging.basicConfig(format='%(message)s') + logging.basicConfig(format="%(message)s") self.logger = logging.getLogger(__name__) self.logger.setLevel(logging.INFO) @@ -218,7 +233,9 @@ def connect(self, args, host, port, client_key, client_cert, server_cert): if client_key and client_cert: # Create credentials for mutual TLS and a secure channel if out_func: - out_func("Enable server auth since both --client-key and --client-cert are provided") + out_func( + "Enable server auth since both --client-key and --client-cert are provided" + ) with client_cert as f: client_cert = f.read() with client_key as f: @@ -285,10 +302,12 @@ def version(self, args): out_func(f"CLI version: {ver}") elif args.format == "json" or args.format == "yaml": cli_ver = pb2.cli_version(status=rc, error_message=errmsg, version=ver) - out_ver = json_format.MessageToJson(cli_ver, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + out_ver = json_format.MessageToJson( + cli_ver, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{out_ver}") elif args.format == "yaml": @@ -326,11 +345,15 @@ def gw_get_info(self): if gw_ver == None: gw_info.status = errno.EINVAL gw_info.bool_status = False - gw_info.error_message = f"Can't parse gateway version \"{gw_info.version}\"." + gw_info.error_message = ( + f'Can\'t parse gateway version "{gw_info.version}".' + ) elif gw_ver < base_ver: gw_info.status = errno.EINVAL gw_info.bool_status = False - gw_info.error_message = f"Can't work with gateway version older than {BASE_GATEWAY_VERSION}" + gw_info.error_message = ( + f"Can't work with gateway version older than {BASE_GATEWAY_VERSION}" + ) return gw_info def gw_info(self, args): @@ -340,7 +363,10 @@ def gw_info(self, args): try: gw_info = self.gw_get_info() except Exception as ex: - gw_info = pb2.gateway_info(status = errno.EINVAL, error_message = f"Failure getting gateway's information:\n{ex}") + gw_info = pb2.gateway_info( + status=errno.EINVAL, + error_message=f"Failure getting gateway's information:\n{ex}", + ) if args.format == "text" or args.format == "plain": if gw_info.status == 0: @@ -354,7 +380,9 @@ def gw_info(self, args): out_func(f"Gateway's group: {gw_info.group}") if gw_info.hostname: out_func(f"Gateway's host name: {gw_info.hostname}") - out_func(f"Gateway's load balancing group: {gw_info.load_balancing_group}") + out_func( + f"Gateway's load balancing group: {gw_info.load_balancing_group}" + ) out_func(f"Gateway's address: {gw_info.addr}") out_func(f"Gateway's port: {gw_info.port}") if gw_info.max_subsystems: @@ -362,7 +390,9 @@ def gw_info(self, args): if gw_info.max_namespaces: out_func(f"Gateway's max namespaces: {gw_info.max_namespaces}") if gw_info.max_hosts_per_subsystem: - out_func(f"Gateway's max hosts per subsystem: {gw_info.max_hosts_per_subsystem}") + out_func( + f"Gateway's max hosts per subsystem: {gw_info.max_hosts_per_subsystem}" + ) if gw_info.spdk_version: out_func(f"SPDK version: {gw_info.spdk_version}") if not gw_info.bool_status: @@ -373,10 +403,11 @@ def gw_info(self, args): err_func(f"Getting gateway's information returned status mismatch") elif args.format == "json" or args.format == "yaml": gw_info_str = json_format.MessageToJson( - gw_info, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + gw_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{gw_info_str}") elif args.format == "yaml": @@ -396,7 +427,10 @@ def gw_version(self, args): try: gw_info = self.gw_get_info() except Exception as ex: - gw_info = pb2.gateway_info(status = errno.EINVAL, error_message = f"Failure getting gateway's version:\n{ex}") + gw_info = pb2.gateway_info( + status=errno.EINVAL, + error_message=f"Failure getting gateway's version:\n{ex}", + ) if args.format == "text" or args.format == "plain": if gw_info.status == 0: @@ -404,18 +438,28 @@ def gw_version(self, args): else: err_func(f"{gw_info.error_message}") elif args.format == "json" or args.format == "yaml": - gw_ver = pb2.gw_version(status=gw_info.status, error_message=gw_info.error_message, version=gw_info.version) - out_ver = json_format.MessageToJson(gw_ver, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + gw_ver = pb2.gw_version( + status=gw_info.status, + error_message=gw_info.error_message, + version=gw_info.version, + ) + out_ver = json_format.MessageToJson( + gw_ver, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{out_ver}") elif args.format == "yaml": obj = json.loads(out_ver) out_func(yaml.dump(obj)) elif args.format == "python": - return pb2.gw_version(status=gw_info.status, error_message=gw_info.error_message, version=gw_info.version) + return pb2.gw_version( + status=gw_info.status, + error_message=gw_info.error_message, + version=gw_info.version, + ) else: assert False @@ -429,19 +473,26 @@ def gw_get_log_level(self, args): try: ret = self.stub.get_gateway_log_level(req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure getting gateway log level:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure getting gateway log level:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: - level = GatewayEnumUtils.get_key_from_value(pb2.GwLogLevel, ret.log_level) - out_func(f"Gateway log level is \"{level}\"") + level = GatewayEnumUtils.get_key_from_value( + pb2.GwLogLevel, ret.log_level + ) + out_func(f'Gateway log level is "{level}"') else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": - out_log_level = json_format.MessageToJson(ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + out_log_level = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{out_log_level}") elif args.format == "yaml": @@ -471,19 +522,23 @@ def gw_set_log_level(self, args): try: ret = self.stub.set_gateway_log_level(req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure setting gateway log level:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure setting gateway log level:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: - out_func(f"Set gateway log level to \"{log_level}\": Successful") + out_func(f'Set gateway log level to "{log_level}": Successful') else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -495,15 +550,34 @@ def gw_set_log_level(self, args): assert False gw_set_log_level_args = [ - argument("--level", "-l", help="Gateway log level", required=True, - type=str, choices=get_enum_keys_list(pb2.GwLogLevel, False)), + argument( + "--level", + "-l", + help="Gateway log level", + required=True, + type=str, + choices=get_enum_keys_list(pb2.GwLogLevel, False), + ), ] gw_actions = [] - gw_actions.append({"name" : "version", "args" : [], "help" : "Display gateway's version"}) - gw_actions.append({"name" : "info", "args" : [], "help" : "Display gateway's information"}) - gw_actions.append({"name" : "get_log_level", "args" : [], "help" : "Get gateway's log level"}) - gw_actions.append({"name" : "set_log_level", "args" : gw_set_log_level_args, "help" : "Set gateway's log level"}) + gw_actions.append( + {"name": "version", "args": [], "help": "Display gateway's version"} + ) + gw_actions.append( + {"name": "info", "args": [], "help": "Display gateway's information"} + ) + gw_actions.append( + {"name": "get_log_level", "args": [], "help": "Get gateway's log level"} + ) + gw_actions.append( + { + "name": "set_log_level", + "args": gw_set_log_level_args, + "help": "Set gateway's log level", + } + ) gw_choices = get_actions(gw_actions) + @cli.cmd(gw_actions) def gw(self, args): """Gateway commands""" @@ -517,7 +591,9 @@ def gw(self, args): elif args.action == "set_log_level": return self.gw_set_log_level(args) if not args.action: - self.cli.parser.error(f"missing action for gw command (choose from {GatewayClient.gw_choices})") + self.cli.parser.error( + f"missing action for gw command (choose from {GatewayClient.gw_choices})" + ) def spdk_log_level_disable(self, args): """Disable SPDK nvmf log flags""" @@ -528,7 +604,10 @@ def spdk_log_level_disable(self, args): try: ret = self.stub.disable_spdk_nvmf_logs(req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure disabling SPDK nvmf log flags:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure disabling SPDK nvmf log flags:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: @@ -537,10 +616,11 @@ def spdk_log_level_disable(self, args): err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -562,24 +642,31 @@ def spdk_log_level_get(self, args): try: ret = self.stub.get_spdk_nvmf_log_flags_and_level(req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure getting SPDK log levels and nvmf log flags:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure getting SPDK log levels and nvmf log flags:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: for flag in ret.nvmf_log_flags: enabled_str = "enabled" if flag.enabled else "disabled" - out_func(f"SPDK nvmf log flag \"{flag.name}\" is {enabled_str}") + out_func(f'SPDK nvmf log flag "{flag.name}" is {enabled_str}') level = GatewayEnumUtils.get_key_from_value(pb2.LogLevel, ret.log_level) out_func(f"SPDK log level is {level}") - level = GatewayEnumUtils.get_key_from_value(pb2.LogLevel, ret.log_print_level) + level = GatewayEnumUtils.get_key_from_value( + pb2.LogLevel, ret.log_print_level + ) out_func(f"SPDK log print level is {level}") else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": - out_log_level = json_format.MessageToJson(ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + out_log_level = json_format.MessageToJson( + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{out_log_level}") elif args.format == "yaml": @@ -608,14 +695,19 @@ def spdk_log_level_set(self, args): print_level = args.print.upper() try: - req = pb2.set_spdk_nvmf_logs_req(log_level=log_level, print_level=print_level) + req = pb2.set_spdk_nvmf_logs_req( + log_level=log_level, print_level=print_level + ) except ValueError as err: self.cli.parser.error(f"invalid log level {log_level}, error {err}") try: ret = self.stub.set_spdk_nvmf_logs(req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure setting SPDK log levels and nvmf log flags:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure setting SPDK log levels and nvmf log flags:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: @@ -624,10 +716,11 @@ def spdk_log_level_set(self, args): err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -642,17 +735,48 @@ def spdk_log_level_set(self, args): spdk_log_get_args = [] spdk_log_set_args = [ - argument("--level", "-l", help="SPDK nvmf log level", required=False, - type=str, choices=get_enum_keys_list(pb2.LogLevel)), - argument("--print", "-p", help="SPDK nvmf log print level", required=False, - type=str, choices=get_enum_keys_list(pb2.LogLevel)), + argument( + "--level", + "-l", + help="SPDK nvmf log level", + required=False, + type=str, + choices=get_enum_keys_list(pb2.LogLevel), + ), + argument( + "--print", + "-p", + help="SPDK nvmf log print level", + required=False, + type=str, + choices=get_enum_keys_list(pb2.LogLevel), + ), ] spdk_log_disable_args = [] spdk_log_actions = [] - spdk_log_actions.append({"name" : "get", "args" : spdk_log_get_args, "help" : "Get SPDK log levels and nvmf log flags"}) - spdk_log_actions.append({"name" : "set", "args" : spdk_log_set_args, "help" : "Set SPDK log levels and nvmf log flags"}) - spdk_log_actions.append({"name" : "disable", "args" : spdk_log_disable_args, "help" : "Disable SPDK nvmf log flags"}) + spdk_log_actions.append( + { + "name": "get", + "args": spdk_log_get_args, + "help": "Get SPDK log levels and nvmf log flags", + } + ) + spdk_log_actions.append( + { + "name": "set", + "args": spdk_log_set_args, + "help": "Set SPDK log levels and nvmf log flags", + } + ) + spdk_log_actions.append( + { + "name": "disable", + "args": spdk_log_disable_args, + "help": "Disable SPDK nvmf log flags", + } + ) spdk_log_choices = get_actions(spdk_log_actions) + @cli.cmd(spdk_log_actions) def spdk_log_level(self, args): """SPDK nvmf log level commands""" @@ -663,7 +787,9 @@ def spdk_log_level(self, args): elif args.action == "disable": return self.spdk_log_level_disable(args) if not args.action: - self.cli.parser.error(f"missing action for spdk_log_level command (choose from {GatewayClient.spdk_log_choices})") + self.cli.parser.error( + f"missing action for spdk_log_level command (choose from {GatewayClient.spdk_log_choices})" + ) def subsystem_add(self, args): """Create a subsystem""" @@ -674,23 +800,30 @@ def subsystem_add(self, args): if args.subsystem == GatewayUtils.DISCOVERY_NQN: self.cli.parser.error("Can't add a discovery subsystem") - req = pb2.create_subsystem_req(subsystem_nqn=args.subsystem, - serial_number=args.serial_number, - max_namespaces=args.max_namespaces, - enable_ha=True, - no_group_append=args.no_group_append, - dhchap_key=args.dhchap_key) + req = pb2.create_subsystem_req( + subsystem_nqn=args.subsystem, + serial_number=args.serial_number, + max_namespaces=args.max_namespaces, + enable_ha=True, + no_group_append=args.no_group_append, + dhchap_key=args.dhchap_key, + ) try: ret = self.stub.create_subsystem(req) except Exception as ex: - ret = pb2.subsys_status(status = errno.EINVAL, error_message = f"Failure adding subsystem {args.subsystem}:\n{ex}", - nqn = args.subsystem) + ret = pb2.subsys_status( + status=errno.EINVAL, + error_message=f"Failure adding subsystem {args.subsystem}:\n{ex}", + nqn=args.subsystem, + ) new_nqn = "" try: new_nqn = ret.nqn - except Exception as ex: # In case of an old gateway the returned value wouldn't have the nqn field - pass + except ( + Exception + ) as ex: # In case of an old gateway the returned value wouldn't have the nqn field + pass if not new_nqn: new_nqn = args.subsystem @@ -701,10 +834,11 @@ def subsystem_add(self, args): err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -728,7 +862,10 @@ def subsystem_del(self, args): try: ret = self.stub.delete_subsystem(req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure deleting subsystem {args.subsystem}:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure deleting subsystem {args.subsystem}:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: @@ -737,10 +874,11 @@ def subsystem_del(self, args): err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -760,34 +898,63 @@ def subsystem_list(self, args): subsystems = None try: - subsystems = self.stub.list_subsystems(pb2.list_subsystems_req(subsystem_nqn=args.subsystem, serial_number=args.serial_number)) + subsystems = self.stub.list_subsystems( + pb2.list_subsystems_req( + subsystem_nqn=args.subsystem, serial_number=args.serial_number + ) + ) except Exception as ex: - subsystems = pb2.subsystems_info_cli(status = errno.EINVAL, error_message = f"Failure listing subsystems:\n{ex}") + subsystems = pb2.subsystems_info_cli( + status=errno.EINVAL, error_message=f"Failure listing subsystems:\n{ex}" + ) if args.format == "text" or args.format == "plain": if subsystems.status == 0: subsys_list = [] for s in subsystems.subsystems: if args.subsystem and args.subsystem != s.nqn: - err_func("Failure listing subsystem {args.subsystem}: Got subsystem {s.nqn} instead") + err_func( + "Failure listing subsystem {args.subsystem}: Got subsystem {s.nqn} instead" + ) return errno.ENODEV if args.serial_number and args.serial_number != s.serial_number: - err_func("Failure listing subsystem with serial number {args.serial_number}: Got serial number {s.serial_number} instead") + err_func( + "Failure listing subsystem with serial number {args.serial_number}: Got serial number {s.serial_number} instead" + ) return errno.ENODEV ctrls_id = f"{s.min_cntlid}-{s.max_cntlid}" has_dhchap = "Yes" if s.has_dhchap_key else "No" allow_any = "Yes" if s.allow_any_host else "No" - one_subsys = [s.subtype, s.nqn, s.serial_number, ctrls_id, s.namespace_count, s.max_namespaces, allow_any, has_dhchap] + one_subsys = [ + s.subtype, + s.nqn, + s.serial_number, + ctrls_id, + s.namespace_count, + s.max_namespaces, + allow_any, + has_dhchap, + ] subsys_list.append(one_subsys) if len(subsys_list) > 0: if args.format == "text": table_format = "fancy_grid" else: table_format = "plain" - subsys_out = tabulate(subsys_list, - headers = ["Subtype", "NQN", "Serial\nNumber", "Controller IDs", - "Namespace\nCount", "Max\nNamespaces", "Allow\nAny Host", "DHCHAP\nKey"], - tablefmt=table_format) + subsys_out = tabulate( + subsys_list, + headers=[ + "Subtype", + "NQN", + "Serial\nNumber", + "Controller IDs", + "Namespace\nCount", + "Max\nNamespaces", + "Allow\nAny Host", + "DHCHAP\nKey", + ], + tablefmt=table_format, + ) prefix = "Subsystems" if args.subsystem: prefix = f"Subsystem {args.subsystem}" @@ -798,17 +965,20 @@ def subsystem_list(self, args): if args.subsystem: out_func(f"No subsystem {args.subsystem}") elif args.serial_number: - out_func(f"No subsystem with serial number {args.serial_number}") + out_func( + f"No subsystem with serial number {args.serial_number}" + ) else: out_func(f"No subsystems") else: err_func(f"{subsystems.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - subsystems, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + subsystems, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -827,12 +997,14 @@ def subsystem_change_key(self, args): rc = 0 out_func, err_func = self.get_output_functions(args) - req = pb2.change_subsystem_key_req(subsystem_nqn=args.subsystem, dhchap_key=args.dhchap_key) + req = pb2.change_subsystem_key_req( + subsystem_nqn=args.subsystem, dhchap_key=args.dhchap_key + ) try: ret = self.stub.change_subsystem_key(req) except Exception as ex: errmsg = f"Failure changing key for subsystem {args.subsystem}" - ret = pb2.req_status(status = errno.EINVAL, error_message = f"{errmsg}:\n{ex}") + ret = pb2.req_status(status=errno.EINVAL, error_message=f"{errmsg}:\n{ex}") if args.format == "text" or args.format == "plain": if ret.status == 0: @@ -841,10 +1013,11 @@ def subsystem_change_key(self, args): err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -860,13 +1033,31 @@ def subsystem_change_key(self, args): subsys_add_args = [ argument("--subsystem", "-n", help="Subsystem NQN", required=True), argument("--serial-number", "-s", help="Serial number", required=False), - argument("--max-namespaces", "-m", help="Maximum number of namespaces", type=int, required=False), - argument("--no-group-append", help="Do not append gateway group name to the NQN", action='store_true', required=False), - argument("--dhchap-key", "-k", help="Subsystem DH-HMAC-CHAP key", required=False), + argument( + "--max-namespaces", + "-m", + help="Maximum number of namespaces", + type=int, + required=False, + ), + argument( + "--no-group-append", + help="Do not append gateway group name to the NQN", + action="store_true", + required=False, + ), + argument( + "--dhchap-key", "-k", help="Subsystem DH-HMAC-CHAP key", required=False + ), ] subsys_del_args = [ argument("--subsystem", "-n", help="Subsystem NQN", required=True), - argument("--force", help="Delete subsytem's namespaces if any, then delete subsystem. If not set a subsystem deletion would fail in case it contains namespaces", action='store_true', required=False), + argument( + "--force", + help="Delete subsytem's namespaces if any, then delete subsystem. If not set a subsystem deletion would fail in case it contains namespaces", + action="store_true", + required=False, + ), ] subsys_list_args = [ argument("--subsystem", "-n", help="Subsystem NQN", required=False), @@ -874,14 +1065,29 @@ def subsystem_change_key(self, args): ] subsys_change_key_args = [ argument("--subsystem", "-n", help="Subsystem NQN", required=True), - argument("--dhchap-key", "-k", help="Subsystem DH-HMAC-CHAP key", required=False), + argument( + "--dhchap-key", "-k", help="Subsystem DH-HMAC-CHAP key", required=False + ), ] subsystem_actions = [] - subsystem_actions.append({"name" : "add", "args" : subsys_add_args, "help" : "Create a subsystem"}) - subsystem_actions.append({"name" : "del", "args" : subsys_del_args, "help" : "Delete a subsystem"}) - subsystem_actions.append({"name" : "list", "args" : subsys_list_args, "help" : "List subsystems"}) - subsystem_actions.append({"name" : "change_key", "args" : subsys_change_key_args, "help" : "Change subsystem key"}) + subsystem_actions.append( + {"name": "add", "args": subsys_add_args, "help": "Create a subsystem"} + ) + subsystem_actions.append( + {"name": "del", "args": subsys_del_args, "help": "Delete a subsystem"} + ) + subsystem_actions.append( + {"name": "list", "args": subsys_list_args, "help": "List subsystems"} + ) + subsystem_actions.append( + { + "name": "change_key", + "args": subsys_change_key_args, + "help": "Change subsystem key", + } + ) subsystem_choices = get_actions(subsystem_actions) + @cli.cmd(subsystem_actions) def subsystem(self, args): """Subsystem commands""" @@ -894,7 +1100,9 @@ def subsystem(self, args): elif args.action == "change_key": return self.subsystem_change_key(args) if not args.action: - self.cli.parser.error(f"missing action for subsystem command (choose from {GatewayClient.subsystem_choices})") + self.cli.parser.error( + f"missing action for subsystem command (choose from {GatewayClient.subsystem_choices})" + ) def listener_add(self, args): """Create a listener""" @@ -905,7 +1113,7 @@ def listener_add(self, args): args.trsvcid = 4420 elif args.trsvcid <= 0: self.cli.parser.error("trsvcid value must be positive") - elif args.trsvcid > 0xffff: + elif args.trsvcid > 0xFFFF: self.cli.parser.error("trsvcid value must be smaller than 65536") if not args.adrfam: args.adrfam = "IPV4" @@ -927,20 +1135,25 @@ def listener_add(self, args): try: ret = self.stub.create_listener(req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, - error_message = f"Failure adding {args.subsystem} listener at {traddr}:{args.trsvcid}:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure adding {args.subsystem} listener at {traddr}:{args.trsvcid}:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: - out_func(f"Adding {args.subsystem} listener at {traddr}:{args.trsvcid}: Successful") + out_func( + f"Adding {args.subsystem} listener at {traddr}:{args.trsvcid}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -959,7 +1172,7 @@ def listener_del(self, args): out_func, err_func = self.get_output_functions(args) if args.trsvcid <= 0: self.cli.parser.error("trsvcid value must be positive") - elif args.trsvcid > 0xffff: + elif args.trsvcid > 0xFFFF: self.cli.parser.error("trsvcid value must be smaller than 65536") if not args.adrfam: args.adrfam = "IPV4" @@ -984,21 +1197,30 @@ def listener_del(self, args): try: ret = self.stub.delete_listener(req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, - error_message = f"Failure deleting listener {traddr}:{args.trsvcid} from {args.subsystem}:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure deleting listener {traddr}:{args.trsvcid} from {args.subsystem}:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: - host_msg = "for all hosts" if args.host_name == "*" else f"for host {args.host_name}" - out_func(f"Deleting listener {traddr}:{args.trsvcid} from {args.subsystem} {host_msg}: Successful") + host_msg = ( + "for all hosts" + if args.host_name == "*" + else f"for host {args.host_name}" + ) + out_func( + f"Deleting listener {traddr}:{args.trsvcid} from {args.subsystem} {host_msg}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1017,26 +1239,50 @@ def listener_list(self, args): out_func, err_func = self.get_output_functions(args) listeners_info = None try: - listeners_info = self.stub.list_listeners(pb2.list_listeners_req(subsystem=args.subsystem)) + listeners_info = self.stub.list_listeners( + pb2.list_listeners_req(subsystem=args.subsystem) + ) except Exception as ex: - listeners_info = pb2.listeners_info(status = errno.EINVAL, error_message = f"Failure listing listeners:\n{ex}", listeners=[]) + listeners_info = pb2.listeners_info( + status=errno.EINVAL, + error_message=f"Failure listing listeners:\n{ex}", + listeners=[], + ) if args.format == "text" or args.format == "plain": if listeners_info.status == 0: listeners_list = [] for l in listeners_info.listeners: - adrfam = GatewayEnumUtils.get_key_from_value(pb2.AddressFamily, l.adrfam) + adrfam = GatewayEnumUtils.get_key_from_value( + pb2.AddressFamily, l.adrfam + ) adrfam = self.format_adrfam(adrfam) secure = "Yes" if l.secure else "No" - listeners_list.append([l.host_name, l.trtype, adrfam, f"{l.traddr}:{l.trsvcid}", secure]) + listeners_list.append( + [ + l.host_name, + l.trtype, + adrfam, + f"{l.traddr}:{l.trsvcid}", + secure, + ] + ) if len(listeners_list) > 0: if args.format == "text": table_format = "fancy_grid" else: table_format = "plain" - listeners_out = tabulate(listeners_list, - headers = ["Host", "Transport", "Address Family", "Address", "Secure"], - tablefmt=table_format) + listeners_out = tabulate( + listeners_list, + headers=[ + "Host", + "Transport", + "Address Family", + "Address", + "Secure", + ], + tablefmt=table_format, + ) out_func(f"Listeners for {args.subsystem}:\n{listeners_out}") else: out_func(f"No listeners for {args.subsystem}") @@ -1044,10 +1290,11 @@ def listener_list(self, args): err_func(f"{listeners_info.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - listeners_info, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + listeners_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1067,23 +1314,48 @@ def listener_list(self, args): argument("--host-name", "-t", help="Host name", required=True), argument("--traddr", "-a", help="NVMe host IP", required=True), argument("--trsvcid", "-s", help="Port number", type=int, required=False), - argument("--adrfam", "-f", help="Address family", default="", choices=get_enum_keys_list(pb2.AddressFamily)), - argument("--secure", help="Use secure channel", action='store_true', required=False), + argument( + "--adrfam", + "-f", + help="Address family", + default="", + choices=get_enum_keys_list(pb2.AddressFamily), + ), + argument( + "--secure", help="Use secure channel", action="store_true", required=False + ), ] listener_del_args = listener_common_args + [ argument("--host-name", "-t", help="Host name", required=True), argument("--traddr", "-a", help="NVMe host IP", required=True), argument("--trsvcid", "-s", help="Port number", type=int, required=True), - argument("--adrfam", "-f", help="Address family", default="", choices=get_enum_keys_list(pb2.AddressFamily)), - argument("--force", help="Delete listener even if there are active connections for the address, or the host name doesn't match", action='store_true', required=False), - ] - listener_list_args = listener_common_args + [ + argument( + "--adrfam", + "-f", + help="Address family", + default="", + choices=get_enum_keys_list(pb2.AddressFamily), + ), + argument( + "--force", + help="Delete listener even if there are active connections for the address, or the host name doesn't match", + action="store_true", + required=False, + ), ] + listener_list_args = listener_common_args + [] listener_actions = [] - listener_actions.append({"name" : "add", "args" : listener_add_args, "help" : "Create a listener"}) - listener_actions.append({"name" : "del", "args" : listener_del_args, "help" : "Delete a listener"}) - listener_actions.append({"name" : "list", "args" : listener_list_args, "help" : "List listeners"}) + listener_actions.append( + {"name": "add", "args": listener_add_args, "help": "Create a listener"} + ) + listener_actions.append( + {"name": "del", "args": listener_del_args, "help": "Delete a listener"} + ) + listener_actions.append( + {"name": "list", "args": listener_list_args, "help": "List listeners"} + ) listener_choices = get_actions(listener_actions) + @cli.cmd(listener_actions) def listener(self, args): """Listener commands""" @@ -1094,7 +1366,9 @@ def listener(self, args): elif args.action == "list": return self.listener_list(args) if not args.action: - self.cli.parser.error(f"missing action for listener command (choose from {GatewayClient.listener_choices})") + self.cli.parser.error( + f"missing action for listener command (choose from {GatewayClient.listener_choices})" + ) def host_add(self, args): """Add a host to a subsystem.""" @@ -1105,21 +1379,31 @@ def host_add(self, args): if args.psk: if len(args.host_nqn) > 1: - self.cli.parser.error(f"Can't have more than one host NQN when PSK keys are used") + self.cli.parser.error( + f"Can't have more than one host NQN when PSK keys are used" + ) if args.dhchap_key: if len(args.host_nqn) > 1: - self.cli.parser.error(f"Can't have more than one host NQN when DH-HMAC-CHAP keys are used") + self.cli.parser.error( + f"Can't have more than one host NQN when DH-HMAC-CHAP keys are used" + ) for one_host_nqn in args.host_nqn: if one_host_nqn == "*" and args.psk: self.cli.parser.error(f"PSK key is only allowed for specific hosts") if one_host_nqn == "*" and args.dhchap_key: - self.cli.parser.error(f"DH-HMAC-CHAP key is only allowed for specific hosts") - - req = pb2.add_host_req(subsystem_nqn=args.subsystem, host_nqn=one_host_nqn, - psk=args.psk, dhchap_key=args.dhchap_key) + self.cli.parser.error( + f"DH-HMAC-CHAP key is only allowed for specific hosts" + ) + + req = pb2.add_host_req( + subsystem_nqn=args.subsystem, + host_nqn=one_host_nqn, + psk=args.psk, + dhchap_key=args.dhchap_key, + ) try: ret = self.stub.add_host(req) except Exception as ex: @@ -1127,7 +1411,9 @@ def host_add(self, args): errmsg = f"Failure allowing open host access to {args.subsystem}" else: errmsg = f"Failure adding host {one_host_nqn} to {args.subsystem}" - ret = pb2.req_status(status = errno.EINVAL, error_message = f"{errmsg}:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, error_message=f"{errmsg}:\n{ex}" + ) if not rc: rc = ret.status @@ -1135,17 +1421,22 @@ def host_add(self, args): if args.format == "text" or args.format == "plain": if ret.status == 0: if one_host_nqn == "*": - out_func(f"Allowing open host access to {args.subsystem}: Successful") + out_func( + f"Allowing open host access to {args.subsystem}: Successful" + ) else: - out_func(f"Adding host {one_host_nqn} to {args.subsystem}: Successful") + out_func( + f"Adding host {one_host_nqn} to {args.subsystem}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1168,7 +1459,9 @@ def host_del(self, args): ret_list = [] out_func, err_func = self.get_output_functions(args) for one_host_nqn in args.host_nqn: - req = pb2.remove_host_req(subsystem_nqn=args.subsystem, host_nqn=one_host_nqn) + req = pb2.remove_host_req( + subsystem_nqn=args.subsystem, host_nqn=one_host_nqn + ) try: ret = self.stub.remove_host(req) @@ -1177,7 +1470,9 @@ def host_del(self, args): errmsg = f"Failure disabling open host access to {args.subsystem}" else: errmsg = f"Failure removing host {one_host_nqn} access to {args.subsystem}" - ret = pb2.req_status(status = errno.EINVAL, error_message = f"{errmsg}:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, error_message=f"{errmsg}:\n{ex}" + ) if not rc: rc = ret.status @@ -1185,17 +1480,22 @@ def host_del(self, args): if args.format == "text" or args.format == "plain": if ret.status == 0: if one_host_nqn == "*": - out_func(f"Disabling open host access to {args.subsystem}: Successful") + out_func( + f"Disabling open host access to {args.subsystem}: Successful" + ) else: - out_func(f"Removing host {one_host_nqn} access from {args.subsystem}: Successful") + out_func( + f"Removing host {one_host_nqn} access from {args.subsystem}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1218,27 +1518,35 @@ def host_change_key(self, args): out_func, err_func = self.get_output_functions(args) if args.host_nqn == "*": - self.cli.parser.error(f"Can't change keys for host NQN '*', please use a real NQN") + self.cli.parser.error( + f"Can't change keys for host NQN '*', please use a real NQN" + ) - req = pb2.change_host_key_req(subsystem_nqn=args.subsystem, host_nqn=args.host_nqn, - dhchap_key=args.dhchap_key) + req = pb2.change_host_key_req( + subsystem_nqn=args.subsystem, + host_nqn=args.host_nqn, + dhchap_key=args.dhchap_key, + ) try: ret = self.stub.change_host_key(req) except Exception as ex: errmsg = f"Failure changing key for host {args.host_nqn} on subsystem {args.subsystem}" - ret = pb2.req_status(status = errno.EINVAL, error_message = f"{errmsg}:\n{ex}") + ret = pb2.req_status(status=errno.EINVAL, error_message=f"{errmsg}:\n{ex}") if args.format == "text" or args.format == "plain": if ret.status == 0: - out_func(f"Changing key for host {args.host_nqn} on subsystem {args.subsystem}: Successful") + out_func( + f"Changing key for host {args.host_nqn} on subsystem {args.subsystem}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1258,9 +1566,15 @@ def host_list(self, args): hosts_info = None try: - hosts_info = self.stub.list_hosts(pb2.list_hosts_req(subsystem=args.subsystem)) + hosts_info = self.stub.list_hosts( + pb2.list_hosts_req(subsystem=args.subsystem) + ) except Exception as ex: - hosts_info = pb2.hosts_info(status = errno.EINVAL, error_message = f"Failure listing hosts:\n{ex}", hosts=[]) + hosts_info = pb2.hosts_info( + status=errno.EINVAL, + error_message=f"Failure listing hosts:\n{ex}", + hosts=[], + ) if args.format == "text" or args.format == "plain": if hosts_info.status == 0: @@ -1276,9 +1590,12 @@ def host_list(self, args): table_format = "fancy_grid" else: table_format = "plain" - hosts_out = tabulate(hosts_list, - headers = ["Host NQN", "Uses PSK", "Uses DHCHAP"], - tablefmt=table_format, stralign="center") + hosts_out = tabulate( + hosts_list, + headers=["Host NQN", "Uses PSK", "Uses DHCHAP"], + tablefmt=table_format, + stralign="center", + ) out_func(f"Hosts allowed to access {args.subsystem}:\n{hosts_out}") else: out_func(f"No hosts are allowed to access {args.subsystem}") @@ -1286,10 +1603,11 @@ def host_list(self, args): err_func(f"{hosts_info.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - hosts_info, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + hosts_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1313,18 +1631,34 @@ def host_list(self, args): host_del_args = host_common_args + [ argument("--host-nqn", "-t", help="Host NQN list", nargs="+", required=True), ] - host_list_args = host_common_args + [ - ] + host_list_args = host_common_args + [] host_change_key_args = host_common_args + [ argument("--host-nqn", "-t", help="Host NQN", required=True), argument("--dhchap-key", "-k", help="Host DH-HMAC-CHAP key", required=False), ] host_actions = [] - host_actions.append({"name" : "add", "args" : host_add_args, "help" : "Add host access to a subsystem"}) - host_actions.append({"name" : "del", "args" : host_del_args, "help" : "Remove host access from a subsystem"}) - host_actions.append({"name" : "list", "args" : host_list_args, "help" : "List subsystem's host access"}) - host_actions.append({"name" : "change_key", "args" : host_change_key_args, "help" : "Change host's inband authentication keys"}) + host_actions.append( + {"name": "add", "args": host_add_args, "help": "Add host access to a subsystem"} + ) + host_actions.append( + { + "name": "del", + "args": host_del_args, + "help": "Remove host access from a subsystem", + } + ) + host_actions.append( + {"name": "list", "args": host_list_args, "help": "List subsystem's host access"} + ) + host_actions.append( + { + "name": "change_key", + "args": host_change_key_args, + "help": "Change host's inband authentication keys", + } + ) host_choices = get_actions(host_actions) + @cli.cmd(host_actions) def host(self, args): """Host commands""" @@ -1337,7 +1671,9 @@ def host(self, args): elif args.action == "change_key": return self.host_change_key(args) if not args.action: - self.cli.parser.error(f"missing action for host command (choose from {GatewayClient.host_choices})") + self.cli.parser.error( + f"missing action for host command (choose from {GatewayClient.host_choices})" + ) def connection_list(self, args): """List connections for a subsystem.""" @@ -1345,10 +1681,15 @@ def connection_list(self, args): out_func, err_func = self.get_output_functions(args) connections_info = None try: - connections_info = self.stub.list_connections(pb2.list_connections_req(subsystem=args.subsystem)) + connections_info = self.stub.list_connections( + pb2.list_connections_req(subsystem=args.subsystem) + ) except Exception as ex: - connections_info = pb2.connections_info(status = errno.EINVAL, - error_message = f"Failure listing hosts:\n{ex}", connections=[]) + connections_info = pb2.connections_info( + status=errno.EINVAL, + error_message=f"Failure listing hosts:\n{ex}", + connections=[], + ) if args.format == "text" or args.format == "plain": if connections_info.status == 0: @@ -1359,22 +1700,41 @@ def connection_list(self, args): conn_dhchap = "Yes" if conn.use_dhchap else "No" if conn.connected: conn_secure = "Yes" if conn.secure else "No" - connections_list.append([conn.nqn, - f"{conn.traddr}:{conn.trsvcid}" if conn.connected else "", - "Yes" if conn.connected else "No", - conn.qpairs_count if conn.connected else "", - conn.controller_id if conn.connected else "", - conn_secure, - conn_psk, - conn_dhchap]) + connections_list.append( + [ + conn.nqn, + ( + f"{conn.traddr}:{conn.trsvcid}" + if conn.connected + else "" + ), + "Yes" if conn.connected else "No", + conn.qpairs_count if conn.connected else "", + conn.controller_id if conn.connected else "", + conn_secure, + conn_psk, + conn_dhchap, + ] + ) if len(connections_list) > 0: if args.format == "text": table_format = "fancy_grid" else: table_format = "plain" - connections_out = tabulate(connections_list, - headers = ["Host NQN", "Address", "Connected", "QPairs Count", "Controller ID", "Secure", "Uses\nPSK", "Uses\nDHCHAP"], - tablefmt=table_format) + connections_out = tabulate( + connections_list, + headers=[ + "Host NQN", + "Address", + "Connected", + "QPairs Count", + "Controller ID", + "Secure", + "Uses\nPSK", + "Uses\nDHCHAP", + ], + tablefmt=table_format, + ) out_func(f"Connections for {args.subsystem}:\n{connections_out}") else: out_func(f"No connections for {args.subsystem}") @@ -1382,10 +1742,11 @@ def connection_list(self, args): err_func(f"{connections_info.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - connections_info, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + connections_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1402,15 +1763,24 @@ def connection_list(self, args): argument("--subsystem", "-n", help="Subsystem NQN", required=True), ] connection_actions = [] - connection_actions.append({"name" : "list", "args" : connection_list_args, "help" : "List active connections"}) + connection_actions.append( + { + "name": "list", + "args": connection_list_args, + "help": "List active connections", + } + ) connection_choices = get_actions(connection_actions) + @cli.cmd(connection_actions) def connection(self, args): """Connection commands""" if args.action == "list": return self.connection_list(args) if not args.action: - self.cli.parser.error(f"missing action for connection command (choose from {GatewayClient.connection_choices})") + self.cli.parser.error( + f"missing action for connection command (choose from {GatewayClient.connection_choices})" + ) def ns_add(self, args): """Adds a namespace to a subsystem.""" @@ -1423,12 +1793,14 @@ def ns_add(self, args): self.cli.parser.error("block-size value must be positive") if args.load_balancing_group < 0: - self.cli.parser.error("load-balancing-group value must be positive") + self.cli.parser.error("load-balancing-group value must be positive") if args.nsid != None and args.nsid <= 0: self.cli.parser.error("nsid value must be positive") if args.rbd_create_image: if args.size == None: - self.cli.parser.error("--size argument is mandatory for add command when RBD image creation is enabled") + self.cli.parser.error( + "--size argument is mandatory for add command when RBD image creation is enabled" + ) img_size = self.get_size_in_bytes(args.size) if img_size <= 0: self.cli.parser.error("size value must be positive") @@ -1437,19 +1809,23 @@ def ns_add(self, args): self.cli.parser.error("size value must be aligned to MiBs") else: if args.size != None: - self.cli.parser.error("--size argument is not allowed for add command when RBD image creation is disabled") - - req = pb2.namespace_add_req(rbd_pool_name=args.rbd_pool, - rbd_image_name=args.rbd_image, - subsystem_nqn=args.subsystem, - nsid=args.nsid, - block_size=args.block_size, - uuid=args.uuid, - anagrpid=args.load_balancing_group, - create_image=args.rbd_create_image, - size=img_size, - force=args.force, - no_auto_visible=args.no_auto_visible) + self.cli.parser.error( + "--size argument is not allowed for add command when RBD image creation is disabled" + ) + + req = pb2.namespace_add_req( + rbd_pool_name=args.rbd_pool, + rbd_image_name=args.rbd_image, + subsystem_nqn=args.subsystem, + nsid=args.nsid, + block_size=args.block_size, + uuid=args.uuid, + anagrpid=args.load_balancing_group, + create_image=args.rbd_create_image, + size=img_size, + force=args.force, + no_auto_visible=args.no_auto_visible, + ) try: ret = self.stub.namespace_add(req) except Exception as ex: @@ -1457,7 +1833,7 @@ def ns_add(self, args): if args.nsid: nsid_msg = f"using NSID {args.nsid} " errmsg = f"Failure adding namespace {nsid_msg}to {args.subsystem}" - ret = pb2.req_status(status = errno.EINVAL, error_message = f"{errmsg}:\n{ex}") + ret = pb2.req_status(status=errno.EINVAL, error_message=f"{errmsg}:\n{ex}") if args.format == "text" or args.format == "plain": if ret.status == 0: @@ -1466,10 +1842,11 @@ def ns_add(self, args): err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1490,21 +1867,28 @@ def ns_del(self, args): self.cli.parser.error("nsid value must be positive") try: - ret = self.stub.namespace_delete(pb2.namespace_delete_req(subsystem_nqn=args.subsystem, nsid=args.nsid)) + ret = self.stub.namespace_delete( + pb2.namespace_delete_req(subsystem_nqn=args.subsystem, nsid=args.nsid) + ) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure deleting namespace:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, error_message=f"Failure deleting namespace:\n{ex}" + ) if args.format == "text" or args.format == "plain": if ret.status == 0: - out_func(f"Deleting namespace {args.nsid} from {args.subsystem}: Successful") + out_func( + f"Deleting namespace {args.nsid} from {args.subsystem}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1533,22 +1917,31 @@ def ns_resize(self, args): ns_size //= mib try: - ret = self.stub.namespace_resize(pb2.namespace_resize_req(subsystem_nqn=args.subsystem, nsid=args.nsid, new_size=ns_size)) + ret = self.stub.namespace_resize( + pb2.namespace_resize_req( + subsystem_nqn=args.subsystem, nsid=args.nsid, new_size=ns_size + ) + ) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure resizing namespace:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, error_message=f"Failure resizing namespace:\n{ex}" + ) if args.format == "text" or args.format == "plain": if ret.status == 0: sz_str = self.format_size(ns_size * mib) - out_func(f"Resizing namespace {args.nsid} in {args.subsystem} to {sz_str}: Successful") + out_func( + f"Resizing namespace {args.nsid} in {args.subsystem} to {sz_str}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1577,7 +1970,7 @@ def get_size_in_bytes(self, sz): sz = sz.strip() try: int_size = int(sz) - sz += "MB" # If no unit is specified assume MB + sz += "MB" # If no unit is specified assume MB except Exception: pass @@ -1613,10 +2006,15 @@ def ns_list(self, args): self.cli.parser.error("nsid value must be positive") try: - namespaces_info = self.stub.list_namespaces(pb2.list_namespaces_req(subsystem=args.subsystem, - nsid=args.nsid, uuid=args.uuid)) + namespaces_info = self.stub.list_namespaces( + pb2.list_namespaces_req( + subsystem=args.subsystem, nsid=args.nsid, uuid=args.uuid + ) + ) except Exception as ex: - namespaces_info = pb2.namespaces_info(status = errno.EINVAL, error_message = f"Failure listing namespaces:\n{ex}") + namespaces_info = pb2.namespaces_info( + status=errno.EINVAL, error_message=f"Failure listing namespaces:\n{ex}" + ) if args.format == "text" or args.format == "plain": if namespaces_info.status == 0: @@ -1627,10 +2025,14 @@ def ns_list(self, args): namespaces_list = [] for ns in namespaces_info.namespaces: if args.nsid and args.nsid != ns.nsid: - err_func("Failure listing namespace {args.nsid}: Got namespace {ns.nsid} instead") + err_func( + "Failure listing namespace {args.nsid}: Got namespace {ns.nsid} instead" + ) return errno.ENODEV if args.uuid and args.uuid != ns.uuid: - err_func("Failure listing namespace with UUID {args.uuid}: Got namespace {ns.uuid} instead") + err_func( + "Failure listing namespace with UUID {args.uuid}: Got namespace {ns.uuid} instead" + ) return errno.ENODEV if ns.load_balancing_group == 0: lb_group = "" @@ -1645,30 +2047,46 @@ def ns_list(self, args): else: visibility = "All Hosts" - namespaces_list.append([ns.nsid, - break_string(ns.bdev_name, "-", 2), - f"{ns.rbd_pool_name}/{ns.rbd_image_name}", - self.format_size(ns.rbd_image_size), - self.format_size(ns.block_size), - break_string(ns.uuid, "-", 3), - lb_group, - visibility, - self.get_qos_limit_str_value(ns.rw_ios_per_second), - self.get_qos_limit_str_value(ns.rw_mbytes_per_second), - self.get_qos_limit_str_value(ns.r_mbytes_per_second), - self.get_qos_limit_str_value(ns.w_mbytes_per_second)]) + namespaces_list.append( + [ + ns.nsid, + break_string(ns.bdev_name, "-", 2), + f"{ns.rbd_pool_name}/{ns.rbd_image_name}", + self.format_size(ns.rbd_image_size), + self.format_size(ns.block_size), + break_string(ns.uuid, "-", 3), + lb_group, + visibility, + self.get_qos_limit_str_value(ns.rw_ios_per_second), + self.get_qos_limit_str_value(ns.rw_mbytes_per_second), + self.get_qos_limit_str_value(ns.r_mbytes_per_second), + self.get_qos_limit_str_value(ns.w_mbytes_per_second), + ] + ) if len(namespaces_list) > 0: if args.format == "text": table_format = "fancy_grid" else: table_format = "plain" - namespaces_out = tabulate(namespaces_list, - headers = ["NSID", "Bdev\nName", "RBD\nImage", - "Image\nSize", "Block\nSize", "UUID", "Load\nBalancing\nGroup", "Visibility", - "R/W IOs\nper\nsecond", "R/W MBs\nper\nsecond", - "Read MBs\nper\nsecond", "Write MBs\nper\nsecond"], - tablefmt=table_format) + namespaces_out = tabulate( + namespaces_list, + headers=[ + "NSID", + "Bdev\nName", + "RBD\nImage", + "Image\nSize", + "Block\nSize", + "UUID", + "Load\nBalancing\nGroup", + "Visibility", + "R/W IOs\nper\nsecond", + "R/W MBs\nper\nsecond", + "Read MBs\nper\nsecond", + "Write MBs\nper\nsecond", + ], + tablefmt=table_format, + ) if args.nsid: prefix = f"Namespace {args.nsid} in" elif args.uuid: @@ -1678,19 +2096,24 @@ def ns_list(self, args): out_func(f"{prefix} subsystem {args.subsystem}:\n{namespaces_out}") else: if args.nsid: - out_func(f"No namespace {args.nsid} in subsystem {args.subsystem}") + out_func( + f"No namespace {args.nsid} in subsystem {args.subsystem}" + ) elif args.uuid: - out_func(f"No namespace with UUID {args.uuid} in subsystem {args.subsystem}") + out_func( + f"No namespace with UUID {args.uuid} in subsystem {args.subsystem}" + ) else: out_func(f"No namespaces in subsystem {args.subsystem}") else: err_func(f"{namespaces_info.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - namespaces_info, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + namespaces_info, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1711,10 +2134,15 @@ def ns_get_io_stats(self, args): self.cli.parser.error("nsid value must be positive") try: - get_stats_req = pb2.namespace_get_io_stats_req(subsystem_nqn=args.subsystem, nsid=args.nsid) + get_stats_req = pb2.namespace_get_io_stats_req( + subsystem_nqn=args.subsystem, nsid=args.nsid + ) ns_io_stats = self.stub.namespace_get_io_stats(get_stats_req) except Exception as ex: - ns_io_stats = pb2.namespace_io_stats_info(status = errno.EINVAL, error_message = f"Failure getting namespace's IO stats:\n{ex}") + ns_io_stats = pb2.namespace_io_stats_info( + status=errno.EINVAL, + error_message=f"Failure getting namespace's IO stats:\n{ex}", + ) if ns_io_stats.status == 0: if ns_io_stats.subsystem_nqn != args.subsystem: @@ -1726,32 +2154,34 @@ def ns_get_io_stats(self, args): # only show IO errors in verbose mode if not args.verbose: - io_stats = pb2.namespace_io_stats_info(status = ns_io_stats.status, - error_message = ns_io_stats.error_message, - subsystem_nqn = ns_io_stats.subsystem_nqn, - nsid = ns_io_stats.nsid, - uuid = ns_io_stats.uuid, - bdev_name = ns_io_stats.bdev_name, - tick_rate = ns_io_stats.tick_rate, - ticks = ns_io_stats.ticks, - bytes_read = ns_io_stats.bytes_read, - num_read_ops = ns_io_stats.num_read_ops, - bytes_written = ns_io_stats.bytes_written, - num_write_ops = ns_io_stats.num_write_ops, - bytes_unmapped = ns_io_stats.bytes_unmapped, - num_unmap_ops = ns_io_stats.num_unmap_ops, - read_latency_ticks = ns_io_stats.read_latency_ticks, - max_read_latency_ticks = ns_io_stats.max_read_latency_ticks, - min_read_latency_ticks = ns_io_stats.min_read_latency_ticks, - write_latency_ticks = ns_io_stats.write_latency_ticks, - max_write_latency_ticks = ns_io_stats.max_write_latency_ticks, - min_write_latency_ticks = ns_io_stats.min_write_latency_ticks, - unmap_latency_ticks = ns_io_stats.unmap_latency_ticks, - max_unmap_latency_ticks = ns_io_stats.max_unmap_latency_ticks, - min_unmap_latency_ticks = ns_io_stats.min_unmap_latency_ticks, - copy_latency_ticks = ns_io_stats.copy_latency_ticks, - max_copy_latency_ticks = ns_io_stats.max_copy_latency_ticks, - min_copy_latency_ticks = ns_io_stats.min_copy_latency_ticks) + io_stats = pb2.namespace_io_stats_info( + status=ns_io_stats.status, + error_message=ns_io_stats.error_message, + subsystem_nqn=ns_io_stats.subsystem_nqn, + nsid=ns_io_stats.nsid, + uuid=ns_io_stats.uuid, + bdev_name=ns_io_stats.bdev_name, + tick_rate=ns_io_stats.tick_rate, + ticks=ns_io_stats.ticks, + bytes_read=ns_io_stats.bytes_read, + num_read_ops=ns_io_stats.num_read_ops, + bytes_written=ns_io_stats.bytes_written, + num_write_ops=ns_io_stats.num_write_ops, + bytes_unmapped=ns_io_stats.bytes_unmapped, + num_unmap_ops=ns_io_stats.num_unmap_ops, + read_latency_ticks=ns_io_stats.read_latency_ticks, + max_read_latency_ticks=ns_io_stats.max_read_latency_ticks, + min_read_latency_ticks=ns_io_stats.min_read_latency_ticks, + write_latency_ticks=ns_io_stats.write_latency_ticks, + max_write_latency_ticks=ns_io_stats.max_write_latency_ticks, + min_write_latency_ticks=ns_io_stats.min_write_latency_ticks, + unmap_latency_ticks=ns_io_stats.unmap_latency_ticks, + max_unmap_latency_ticks=ns_io_stats.max_unmap_latency_ticks, + min_unmap_latency_ticks=ns_io_stats.min_unmap_latency_ticks, + copy_latency_ticks=ns_io_stats.copy_latency_ticks, + max_copy_latency_ticks=ns_io_stats.max_copy_latency_ticks, + min_copy_latency_ticks=ns_io_stats.min_copy_latency_ticks, + ) ns_io_stats = io_stats if args.format == "text" or args.format == "plain": @@ -1765,18 +2195,42 @@ def ns_get_io_stats(self, args): stats_list.append(["Num Write Ops", ns_io_stats.num_write_ops]) stats_list.append(["Bytes Unmapped", ns_io_stats.bytes_unmapped]) stats_list.append(["Num Unmap Ops", ns_io_stats.num_unmap_ops]) - stats_list.append(["Read Latency Ticks", ns_io_stats.read_latency_ticks]) - stats_list.append(["Max Read Latency Ticks", ns_io_stats.max_read_latency_ticks]) - stats_list.append(["Min Read Latency Ticks", ns_io_stats.min_read_latency_ticks]) - stats_list.append(["Write Latency Ticks", ns_io_stats.write_latency_ticks]) - stats_list.append(["Max Write Latency Ticks", ns_io_stats.max_write_latency_ticks]) - stats_list.append(["Min Write Latency Ticks", ns_io_stats.min_write_latency_ticks]) - stats_list.append(["Unmap Latency Ticks", ns_io_stats.unmap_latency_ticks]) - stats_list.append(["Max Unmap Latency Ticks", ns_io_stats.max_unmap_latency_ticks]) - stats_list.append(["Min Unmap Latency Ticks", ns_io_stats.min_unmap_latency_ticks]) - stats_list.append(["Copy Latency Ticks", ns_io_stats.copy_latency_ticks]) - stats_list.append(["Max Copy Latency Ticks", ns_io_stats.max_copy_latency_ticks]) - stats_list.append(["Min Copy Latency Ticks", ns_io_stats.min_copy_latency_ticks]) + stats_list.append( + ["Read Latency Ticks", ns_io_stats.read_latency_ticks] + ) + stats_list.append( + ["Max Read Latency Ticks", ns_io_stats.max_read_latency_ticks] + ) + stats_list.append( + ["Min Read Latency Ticks", ns_io_stats.min_read_latency_ticks] + ) + stats_list.append( + ["Write Latency Ticks", ns_io_stats.write_latency_ticks] + ) + stats_list.append( + ["Max Write Latency Ticks", ns_io_stats.max_write_latency_ticks] + ) + stats_list.append( + ["Min Write Latency Ticks", ns_io_stats.min_write_latency_ticks] + ) + stats_list.append( + ["Unmap Latency Ticks", ns_io_stats.unmap_latency_ticks] + ) + stats_list.append( + ["Max Unmap Latency Ticks", ns_io_stats.max_unmap_latency_ticks] + ) + stats_list.append( + ["Min Unmap Latency Ticks", ns_io_stats.min_unmap_latency_ticks] + ) + stats_list.append( + ["Copy Latency Ticks", ns_io_stats.copy_latency_ticks] + ) + stats_list.append( + ["Max Copy Latency Ticks", ns_io_stats.max_copy_latency_ticks] + ) + stats_list.append( + ["Min Copy Latency Ticks", ns_io_stats.min_copy_latency_ticks] + ) for e in ns_io_stats.io_error: if e.value: stats_list.append([f"IO Error - {e.name}", e.value]) @@ -1785,16 +2239,21 @@ def ns_get_io_stats(self, args): table_format = "fancy_grid" else: table_format = "plain" - stats_out = tabulate(stats_list, headers = ["Stat", "Value"], tablefmt=table_format) - out_func(f"IO statistics for namespace {args.nsid} in {args.subsystem}, bdev {ns_io_stats.bdev_name}:\n{stats_out}") + stats_out = tabulate( + stats_list, headers=["Stat", "Value"], tablefmt=table_format + ) + out_func( + f"IO statistics for namespace {args.nsid} in {args.subsystem}, bdev {ns_io_stats.bdev_name}:\n{stats_out}" + ) else: err_func(f"{ns_io_stats.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ns_io_stats, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ns_io_stats, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1817,23 +2276,32 @@ def ns_change_load_balancing_group(self, args): self.cli.parser.error("load-balancing-group value must be positive") try: - change_lb_group_req = pb2.namespace_change_load_balancing_group_req(subsystem_nqn=args.subsystem, - nsid=args.nsid, anagrpid=args.load_balancing_group) + change_lb_group_req = pb2.namespace_change_load_balancing_group_req( + subsystem_nqn=args.subsystem, + nsid=args.nsid, + anagrpid=args.load_balancing_group, + ) ret = self.stub.namespace_change_load_balancing_group(change_lb_group_req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure changing namespace load balancing group:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure changing namespace load balancing group:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: - out_func(f"Changing load balancing group of namespace {args.nsid} in {args.subsystem} to {args.load_balancing_group}: Successful") + out_func( + f"Changing load balancing group of namespace {args.nsid} in {args.subsystem} to {args.load_balancing_group}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1858,13 +2326,20 @@ def ns_set_qos(self, args): out_func, err_func = self.get_output_functions(args) if args.nsid <= 0: self.cli.parser.error("nsid value must be positive") - if args.rw_ios_per_second == None and args.rw_megabytes_per_second == None and args.r_megabytes_per_second == None and args.w_megabytes_per_second == None: + if ( + args.rw_ios_per_second == None + and args.rw_megabytes_per_second == None + and args.r_megabytes_per_second == None + and args.w_megabytes_per_second == None + ): self.cli.parser.error("At least one QOS limit should be set") if args.format == "text" or args.format == "plain": if args.rw_ios_per_second and (args.rw_ios_per_second % 1000) != 0: rounded_rate = int((args.rw_ios_per_second + 1000) / 1000) * 1000 - err_func(f"IOs per second {args.rw_ios_per_second} will be rounded up to {rounded_rate}") + err_func( + f"IOs per second {args.rw_ios_per_second} will be rounded up to {rounded_rate}" + ) qos_args = {} qos_args["subsystem_nqn"] = args.subsystem @@ -1882,19 +2357,25 @@ def ns_set_qos(self, args): set_qos_req = pb2.namespace_set_qos_req(**qos_args) ret = self.stub.namespace_set_qos_limits(set_qos_req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure setting namespaces QOS limits:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure setting namespaces QOS limits:\n{ex}", + ) if args.format == "text" or args.format == "plain": if ret.status == 0: - out_func(f"Setting QOS limits of namespace {args.nsid} in {args.subsystem}: Successful") + out_func( + f"Setting QOS limits of namespace {args.nsid} in {args.subsystem}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1919,25 +2400,33 @@ def ns_add_host(self, args): for one_host_nqn in args.host_nqn: try: - add_host_req = pb2.namespace_add_host_req(subsystem_nqn=args.subsystem, nsid=args.nsid, host_nqn=one_host_nqn) + add_host_req = pb2.namespace_add_host_req( + subsystem_nqn=args.subsystem, nsid=args.nsid, host_nqn=one_host_nqn + ) ret = self.stub.namespace_add_host(add_host_req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure adding host to namespace:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure adding host to namespace:\n{ex}", + ) if not rc: rc = ret.status if args.format == "text" or args.format == "plain": if ret.status == 0: - out_func(f"Adding host {one_host_nqn} to namespace {args.nsid} on {args.subsystem}: Successful") + out_func( + f"Adding host {one_host_nqn} to namespace {args.nsid} on {args.subsystem}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -1965,25 +2454,33 @@ def ns_del_host(self, args): for one_host_nqn in args.host_nqn: try: - del_host_req = pb2.namespace_delete_host_req(subsystem_nqn=args.subsystem, nsid=args.nsid, host_nqn=one_host_nqn) + del_host_req = pb2.namespace_delete_host_req( + subsystem_nqn=args.subsystem, nsid=args.nsid, host_nqn=one_host_nqn + ) ret = self.stub.namespace_delete_host(del_host_req) except Exception as ex: - ret = pb2.req_status(status = errno.EINVAL, error_message = f"Failure deleting host from namespace:\n{ex}") + ret = pb2.req_status( + status=errno.EINVAL, + error_message=f"Failure deleting host from namespace:\n{ex}", + ) if not rc: rc = ret.status if args.format == "text" or args.format == "plain": if ret.status == 0: - out_func(f"Deleting host {args.host_nqn} from namespace {args.nsid} on {args.subsystem}: Successful") + out_func( + f"Deleting host {args.host_nqn} from namespace {args.nsid} on {args.subsystem}: Successful" + ) else: err_func(f"{ret.error_message}") elif args.format == "json" or args.format == "yaml": ret_str = json_format.MessageToJson( - ret, - indent=4, - including_default_value_fields=True, - preserving_proto_field_name=True) + ret, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) if args.format == "json": out_func(f"{ret_str}") elif args.format == "yaml": @@ -2007,19 +2504,48 @@ def ns_del_host(self, args): argument("--uuid", "-u", help="UUID"), argument("--rbd-pool", "-p", help="RBD pool name", required=True), argument("--rbd-image", "-i", help="RBD image name", required=True), - argument("--rbd-create-image", "-c", help="Create RBD image if needed", action='store_true', required=False), + argument( + "--rbd-create-image", + "-c", + help="Create RBD image if needed", + action="store_true", + required=False, + ), argument("--block-size", "-s", help="Block size", type=int), - argument("--load-balancing-group", "-l", help="Load balancing group", type=int, default=0), - argument("--size", help="Size in bytes or specified unit (K, KB, M, MB, G, GB, T, TB, P, PB)"), - argument("--force", help="Create a namespace even when its image is already used by another namespace", action='store_true', required=False), - argument("--no-auto-visible", help="Make the namespace visible only to specific hosts", action='store_true', required=False), + argument( + "--load-balancing-group", + "-l", + help="Load balancing group", + type=int, + default=0, + ), + argument( + "--size", + help="Size in bytes or specified unit (K, KB, M, MB, G, GB, T, TB, P, PB)", + ), + argument( + "--force", + help="Create a namespace even when its image is already used by another namespace", + action="store_true", + required=False, + ), + argument( + "--no-auto-visible", + help="Make the namespace visible only to specific hosts", + action="store_true", + required=False, + ), ] ns_del_args_list = ns_common_args + [ argument("--nsid", help="Namespace ID", type=int, required=True), ] ns_resize_args_list = ns_common_args + [ argument("--nsid", help="Namespace ID", type=int, required=True), - argument("--size", help="Size in bytes or specified unit (K, KB, M, MB, G, GB, T, TB, P, PB)", required=True), + argument( + "--size", + help="Size in bytes or specified unit (K, KB, M, MB, G, GB, T, TB, P, PB)", + required=True, + ), ] ns_list_args_list = ns_common_args + [ argument("--nsid", help="Namespace ID", type=int), @@ -2030,14 +2556,36 @@ def ns_del_host(self, args): ] ns_change_load_balancing_group_args_list = ns_common_args + [ argument("--nsid", help="Namespace ID", type=int, required=True), - argument("--load-balancing-group", "-l", help="Load balancing group", type=int, required=True), + argument( + "--load-balancing-group", + "-l", + help="Load balancing group", + type=int, + required=True, + ), ] ns_set_qos_args_list = ns_common_args + [ argument("--nsid", help="Namespace ID", type=int, required=True), - argument("--rw-ios-per-second", help="R/W IOs per second limit, 0 means unlimited", type=int), - argument("--rw-megabytes-per-second", help="R/W megabytes per second limit, 0 means unlimited", type=int), - argument("--r-megabytes-per-second", help="Read megabytes per second limit, 0 means unlimited", type=int), - argument("--w-megabytes-per-second", help="Write megabytes per second limit, 0 means unlimited", type=int), + argument( + "--rw-ios-per-second", + help="R/W IOs per second limit, 0 means unlimited", + type=int, + ), + argument( + "--rw-megabytes-per-second", + help="R/W megabytes per second limit, 0 means unlimited", + type=int, + ), + argument( + "--r-megabytes-per-second", + help="Read megabytes per second limit, 0 means unlimited", + type=int, + ), + argument( + "--w-megabytes-per-second", + help="Write megabytes per second limit, 0 means unlimited", + type=int, + ), ] ns_add_host_args_list = ns_common_args + [ argument("--nsid", help="Namespace ID", type=int, required=True), @@ -2048,16 +2596,55 @@ def ns_del_host(self, args): argument("--host-nqn", "-t", help="Host NQN list", nargs="+", required=True), ] ns_actions = [] - ns_actions.append({"name" : "add", "args" : ns_add_args_list, "help" : "Create a namespace"}) - ns_actions.append({"name" : "del", "args" : ns_del_args_list, "help" : "Delete a namespace"}) - ns_actions.append({"name" : "resize", "args" : ns_resize_args_list, "help" : "Resize a namespace"}) - ns_actions.append({"name" : "list", "args" : ns_list_args_list, "help" : "List namespaces"}) - ns_actions.append({"name" : "get_io_stats", "args" : ns_get_io_stats_args_list, "help" : "Get I/O stats for a namespace"}) - ns_actions.append({"name" : "change_load_balancing_group", "args" : ns_change_load_balancing_group_args_list, "help" : "Change load balancing group for a namespace"}) - ns_actions.append({"name" : "set_qos", "args" : ns_set_qos_args_list, "help" : "Set QOS limits for a namespace"}) - ns_actions.append({"name" : "add_host", "args" : ns_add_host_args_list, "help" : "Add a host to a namespace"}) - ns_actions.append({"name" : "del_host", "args" : ns_del_host_args_list, "help" : "Delete a host from a namespace"}) + ns_actions.append( + {"name": "add", "args": ns_add_args_list, "help": "Create a namespace"} + ) + ns_actions.append( + {"name": "del", "args": ns_del_args_list, "help": "Delete a namespace"} + ) + ns_actions.append( + {"name": "resize", "args": ns_resize_args_list, "help": "Resize a namespace"} + ) + ns_actions.append( + {"name": "list", "args": ns_list_args_list, "help": "List namespaces"} + ) + ns_actions.append( + { + "name": "get_io_stats", + "args": ns_get_io_stats_args_list, + "help": "Get I/O stats for a namespace", + } + ) + ns_actions.append( + { + "name": "change_load_balancing_group", + "args": ns_change_load_balancing_group_args_list, + "help": "Change load balancing group for a namespace", + } + ) + ns_actions.append( + { + "name": "set_qos", + "args": ns_set_qos_args_list, + "help": "Set QOS limits for a namespace", + } + ) + ns_actions.append( + { + "name": "add_host", + "args": ns_add_host_args_list, + "help": "Add a host to a namespace", + } + ) + ns_actions.append( + { + "name": "del_host", + "args": ns_del_host_args_list, + "help": "Delete a host from a namespace", + } + ) ns_choices = get_actions(ns_actions) + @cli.cmd(ns_actions, ["ns"]) def namespace(self, args): """Namespace commands""" @@ -2080,7 +2667,9 @@ def namespace(self, args): elif args.action == "del_host": return self.ns_del_host(args) if not args.action: - self.cli.parser.error(f"missing action for namespace command (choose from {GatewayClient.ns_choices})") + self.cli.parser.error( + f"missing action for namespace command (choose from {GatewayClient.ns_choices})" + ) @cli.cmd() def get_subsystems(self, args): @@ -2091,29 +2680,37 @@ def get_subsystems(self, args): if args.format == "python": return subsystems subsystems_out = json_format.MessageToJson( - subsystems, - indent=4, including_default_value_fields=True, - preserving_proto_field_name=True) + subsystems, + indent=4, + including_default_value_fields=True, + preserving_proto_field_name=True, + ) out_func(f"Get subsystems:\n{subsystems_out}") + def main_common(client, args): - client.logger.setLevel(GatewayEnumUtils.get_value_from_key(pb2.GwLogLevel, args.log_level.lower())) + client.logger.setLevel( + GatewayEnumUtils.get_value_from_key(pb2.GwLogLevel, args.log_level.lower()) + ) server_address = args.server_address server_port = args.server_port client_key = args.client_key client_cert = args.client_cert server_cert = args.server_cert - client.connect(args, server_address, server_port, client_key, client_cert, server_cert) + client.connect( + args, server_address, server_port, client_key, client_cert, server_cert + ) call_function = getattr(client, args.func.__name__) rc = call_function(args) return rc + def main_test(args): if not args: return None try: i = args.index("--format") - del args[i:i + 2] + del args[i : i + 2] except Exception: pass args = ["--format", "python"] + args @@ -2124,6 +2721,7 @@ def main_test(args): return main_common(client, parsed_args) + def main(args=None) -> int: client = GatewayClient() parsed_args = client.cli.parser.parse_args(args) @@ -2133,5 +2731,6 @@ def main(args=None) -> int: return main_common(client, parsed_args) + if __name__ == "__main__": sys.exit(main()) diff --git a/control/config.py b/control/config.py index 947ad87c..8867d17f 100644 --- a/control/config.py +++ b/control/config.py @@ -10,6 +10,7 @@ import configparser import os + class GatewayConfig: """Loads and returns config file settings. @@ -59,12 +60,14 @@ def dump_config_file(self, logger): logger.info(f"Using configuration file {self.filepath}") with open(self.filepath) as f: logger.info( - f"====================================== Configuration file content ======================================") + f"====================================== Configuration file content ======================================" + ) for line in f: line = line.rstrip() logger.info(f"{line}") logger.info( - f"========================================================================================================") + f"========================================================================================================" + ) self.conffile_logged = True except Exception: pass diff --git a/control/discovery.py b/control/discovery.py index eae1a504..353f1817 100644 --- a/control/discovery.py +++ b/control/discovery.py @@ -11,7 +11,12 @@ import grpc import json from .config import GatewayConfig -from .state import GatewayState, LocalGatewayState, OmapGatewayState, GatewayStateHandler +from .state import ( + GatewayState, + LocalGatewayState, + OmapGatewayState, + GatewayStateHandler, +) from .utils import GatewayLogger from .proto import gateway_pb2 as pb2 @@ -27,9 +32,20 @@ import selectors import os from dataclasses import dataclass, field -from ctypes import Structure, LittleEndianStructure, c_bool, c_ubyte, c_uint8, c_uint16, c_uint32, c_uint64, c_float +from ctypes import ( + Structure, + LittleEndianStructure, + c_bool, + c_ubyte, + c_uint8, + c_uint16, + c_uint32, + c_uint64, + c_float, +) from google.protobuf import json_format + # NVMe tcp pdu type class NVME_TCP_PDU(enum.IntFlag): ICREQ = 0x0 @@ -42,6 +58,7 @@ class NVME_TCP_PDU(enum.IntFlag): C2H_DATA = 0x7 TCP_R2T = 0x9 + # NVMe tcp opcode class NVME_TCP_OPC(enum.IntFlag): DELETE_SQ = 0x0 @@ -52,15 +69,16 @@ class NVME_TCP_OPC(enum.IntFlag): IDENTIFY = 0x6 ABORT = 0x8 SET_FEATURES = 0x9 - GET_FEATURES = 0xa - ASYNC_EVE_REQ = 0xc - NS_MGMT = 0xd + GET_FEATURES = 0xA + ASYNC_EVE_REQ = 0xC + NS_MGMT = 0xD FW_COMMIT = 0x10 FW_IMG_DOWNLOAD = 0x11 NS_ATTACH = 0x15 KEEP_ALIVE = 0x18 FABRIC_TYPE = 0x7F + # NVMe tcp fabric command type class NVME_TCP_FCTYPE(enum.IntFlag): PROP_SET = 0x0 @@ -70,12 +88,13 @@ class NVME_TCP_FCTYPE(enum.IntFlag): AUTH_RECV = 0x6 DISCONNECT = 0x8 + # NVMe controller register space offsets class NVME_CTL(enum.IntFlag): CAPABILITIES = 0x0 VERSION = 0x08 CONFIGURATION = 0x14 - STATUS = 0x1c + STATUS = 0x1C # NVM subsystem types @@ -85,12 +104,14 @@ class NVMF_SUBTYPE(enum.IntFlag): # NVMe type for NVM subsystem NVME = 0x2 + # NVMe over Fabrics transport types class TRANSPORT_TYPES(enum.IntFlag): RDMA = 0x1 FC = 0x2 TCP = 0x3 - INTRA_HOST = 0xfe + INTRA_HOST = 0xFE + # Address family types class ADRFAM_TYPES(enum.IntFlag): @@ -98,7 +119,8 @@ class ADRFAM_TYPES(enum.IntFlag): ipv6 = 0x2 ib = 0x3 fc = 0x4 - intra_host = 0xfe + intra_host = 0xFE + # Transport requirement, secure channel requirements # Connections shall be made over a fabric secure channel @@ -107,6 +129,7 @@ class NVMF_TREQ_SECURE_CHANNEL(enum.IntFlag): REQUIRED = 0x1 NOT_REQUIRED = 0x2 + # maximum number of connections MAX_CONNECTION = 10240 @@ -116,6 +139,7 @@ class NVMF_TREQ_SECURE_CHANNEL(enum.IntFlag): # Max SQ head pointer SQ_HEAD_MAX = 128 + @dataclass class Connection: """Data used multiple times in each connection.""" @@ -124,15 +148,15 @@ class Connection: allow_listeners: list = field(default_factory=list) log_page: bytearray = field(default_factory=bytearray) recv_buffer: bytearray = field(default_factory=bytearray) - nvmeof_connect_data_hostid: tuple = tuple((c_ubyte *16)()) + nvmeof_connect_data_hostid: tuple = tuple((c_ubyte * 16)()) nvmeof_connect_data_cntlid: int = 0 - nvmeof_connect_data_subnqn: tuple = tuple((c_ubyte *256)()) - nvmeof_connect_data_hostnqn: tuple = tuple((c_ubyte *256)()) + nvmeof_connect_data_subnqn: tuple = tuple((c_ubyte * 256)()) + nvmeof_connect_data_hostnqn: tuple = tuple((c_ubyte * 256)()) sq_head_ptr: int = 0 unsent_log_page_len: int = 0 # NVM ExpressTM Revision 1.4, page 47 # see Figure 78: Offset 14h: CC – Controller Configuration - property_configuration: tuple = tuple((c_ubyte *8)()) + property_configuration: tuple = tuple((c_ubyte * 8)()) shutdown_now: bool = False controller_id: uuid = None gen_cnt: int = 0 @@ -141,6 +165,7 @@ class Connection: keep_alive_time: float = 0.0 keep_alive_timeout: int = 0 + class AutoSerializableStructure(LittleEndianStructure): def __add__(self, other): if isinstance(other, LittleEndianStructure): @@ -150,6 +175,7 @@ def __add__(self, other): else: raise ValueError("error message format.") + class Pdu(AutoSerializableStructure): _fields_ = [ ("type", c_uint8), @@ -159,6 +185,7 @@ class Pdu(AutoSerializableStructure): ("packet_length", c_uint32), ] + class ICResp(AutoSerializableStructure): _fields_ = [ # pdu version format @@ -168,9 +195,10 @@ class ICResp(AutoSerializableStructure): # digest types enabled ("digest_types", c_uint8), # Maximum data capsules per r2t supported - ("maximum_data_capsules", c_uint32) + ("maximum_data_capsules", c_uint32), ] + class CqeConnect(AutoSerializableStructure): _fields_ = [ ("controller_id", c_uint16), @@ -179,9 +207,10 @@ class CqeConnect(AutoSerializableStructure): ("sq_head_ptr", c_uint16), ("sq_id", c_uint16), ("cmd_id", c_uint16), - ("status", c_uint16) + ("status", c_uint16), ] + class CqePropertyGetSet(AutoSerializableStructure): _fields_ = [ # property data for property get, reserved for property set @@ -189,18 +218,20 @@ class CqePropertyGetSet(AutoSerializableStructure): ("sq_head_ptr", c_uint16), ("sq_id", c_uint16), ("cmd_id", c_uint16), - ("status", c_uint16) + ("status", c_uint16), ] + class NVMeTcpDataPdu(AutoSerializableStructure): _fields_ = [ ("cmd_id", c_uint16), ("transfer_tag", c_uint16), ("data_offset", c_uint32), ("data_length", c_uint32), - ("reserved", c_uint32) + ("reserved", c_uint32), ] + class NVMeIdentify(AutoSerializableStructure): _fields_ = [ # skip some fields, include VID, SSVID, SN, MN @@ -248,31 +279,34 @@ class NVMeIdentify(AutoSerializableStructure): ("reserved3", c_ubyte * 768), ("nvmeof_attributes", c_ubyte * 256), ("power_state_attributes", c_ubyte * 1024), - ("vendor_specific", c_ubyte * 1024) + ("vendor_specific", c_ubyte * 1024), ] + # for set feature, keep alive and async -class CqeNVMe(AutoSerializableStructure): +class CqeNVMe(AutoSerializableStructure): _fields_ = [ ("dword0", c_uint32), ("dword1", c_uint32), ("sq_head_ptr", c_uint16), ("sq_id", c_uint16), ("cmd_id", c_uint16), - ("status", c_uint16) + ("status", c_uint16), ] + class NVMeGetLogPage(AutoSerializableStructure): _fields_ = [ # generation counter ("genctr", c_uint64), # number of records ("numrec", c_uint64), - #record format + # record format ("recfmt", c_uint16), - ("reserved", c_ubyte * 1006) + ("reserved", c_ubyte * 1006), ] + class DiscoveryLogEntry(AutoSerializableStructure): _fields_ = [ ("trtype", c_uint8), @@ -289,9 +323,10 @@ class DiscoveryLogEntry(AutoSerializableStructure): ("subnqn", c_ubyte * 256), ("traddr", c_ubyte * 256), # Transport specific address subtype - ("tsas", c_ubyte * 256) + ("tsas", c_ubyte * 256), ] + class DiscoveryService: """Implements discovery controller. @@ -310,22 +345,29 @@ def __init__(self, config): self.version = 1 self.config = config self.lock = threading.Lock() - self.omap_state = OmapGatewayState(self.config, f"discovery-{socket.gethostname()}") + self.omap_state = OmapGatewayState( + self.config, f"discovery-{socket.gethostname()}" + ) self.gw_logger_object = GatewayLogger(config) self.logger = self.gw_logger_object.logger gateway_group = self.config.get_with_default("gateway", "group", "") - self.omap_name = f"nvmeof.{gateway_group}.state" \ - if gateway_group else "nvmeof.state" + self.omap_name = ( + f"nvmeof.{gateway_group}.state" if gateway_group else "nvmeof.state" + ) self.logger.info(f"log pages info from omap: {self.omap_name}") - self.discovery_addr = self.config.get_with_default("discovery", "addr", "0.0.0.0") + self.discovery_addr = self.config.get_with_default( + "discovery", "addr", "0.0.0.0" + ) self.discovery_port = self.config.get_with_default("discovery", "port", "8009") if not self.discovery_addr or not self.discovery_port: self.logger.error("discovery addr/port are empty.") assert 0 - self.logger.info(f"discovery addr: {self.discovery_addr} port: {self.discovery_port}") + self.logger.info( + f"discovery addr: {self.discovery_addr} port: {self.discovery_port}" + ) self.sock = None self.conn_vals = {} @@ -379,8 +421,11 @@ def _read_all(self) -> Dict[str, str]: def _get_vals(self, omap_dict, prefix): """Read values from the OMAP dict.""" - return [json.loads(val.decode('utf-8')) for (key, val) in omap_dict.items() - if key.startswith(prefix)] + return [ + json.loads(val.decode("utf-8")) + for (key, val) in omap_dict.items() + if key.startswith(prefix) + ] def reply_initialize(self, conn): """Reply initialize request.""" @@ -407,35 +452,31 @@ def reply_fc_cmd_connect(self, conn, data, cmd_id): self.logger.debug("handle connect request.") self_conn = self.conn_vals[conn.fileno()] - hf_nvmeof_cmd_connect_rsvd1 = struct.unpack_from('<19B', data, 13) - SIGL1 = struct.unpack_from('> 8) & 0x1F get_logpage_lsi = nvme_get_logpage_dword11 >> 16 @@ -740,7 +793,9 @@ def reply_get_log_page(self, conn, data, cmd_id): return -1 if nvme_data_len != nvme_sgl_len: - self.logger.error(f"request data len error, {nvme_data_len=} != {nvme_sgl_len=}.") + self.logger.error( + f"request data len error, {nvme_data_len=} != {nvme_sgl_len=}." + ) return -1 # Filter listeners based on host access permissions @@ -748,12 +803,14 @@ def reply_get_log_page(self, conn, data, cmd_id): if len(allow_listeners) == 0: for host in hosts: a = host["host_nqn"] - if host["host_nqn"] == '*' or host["host_nqn"] == hostnqn: + if host["host_nqn"] == "*" or host["host_nqn"] == hostnqn: for listener in listeners: # TODO: It is better to change nqn in the "listener" # to subsystem_nqn to avoid confusion if host["subsystem_nqn"] == listener["nqn"]: - allow_listeners += [listener,] + allow_listeners += [ + listener, + ] self_conn.allow_listeners = allow_listeners # Prepare all log page data segments @@ -780,36 +837,51 @@ def reply_get_log_page(self, conn, data, cmd_id): log_entry.subtype = NVMF_SUBTYPE.NVME log_entry.treq = NVMF_TREQ_SECURE_CHANNEL.NOT_REQUIRED log_entry.port_id = log_entry_counter - log_entry.controller_id = 0xffff + log_entry.controller_id = 0xFFFF log_entry.asqsz = 128 # transport service indentifier str_trsvcid = str(allow_listeners[log_entry_counter]["trsvcid"]) - log_entry.trsvcid = (c_ubyte * 32)(*[c_ubyte(x) for x \ - in str_trsvcid.encode()]) - log_entry.trsvcid[len(str_trsvcid):] = \ - [c_ubyte(0x20)] * (32 - len(str_trsvcid)) + log_entry.trsvcid = (c_ubyte * 32)( + *[c_ubyte(x) for x in str_trsvcid.encode()] + ) + log_entry.trsvcid[len(str_trsvcid) :] = [c_ubyte(0x20)] * ( + 32 - len(str_trsvcid) + ) # NVM subsystem qualified name - log_entry.subnqn = (c_ubyte * 256)(*[c_ubyte(x) for x \ - in allow_listeners[log_entry_counter]["nqn"].encode()]) - log_entry.subnqn[len(allow_listeners[log_entry_counter]["nqn"]):] = \ - [c_ubyte(0x00)] * (256 - len(allow_listeners[log_entry_counter]["nqn"])) + log_entry.subnqn = (c_ubyte * 256)( + *[ + c_ubyte(x) + for x in allow_listeners[log_entry_counter]["nqn"].encode() + ] + ) + log_entry.subnqn[len(allow_listeners[log_entry_counter]["nqn"]) :] = [ + c_ubyte(0x00) + ] * (256 - len(allow_listeners[log_entry_counter]["nqn"])) # Transport address - log_entry.traddr = (c_ubyte * 256)(*[c_ubyte(x) for x \ - in allow_listeners[log_entry_counter]["traddr"].encode()]) - log_entry.traddr[len(allow_listeners[log_entry_counter]["traddr"]):] = \ - [c_ubyte(0x20)] * (256 - len(allow_listeners[log_entry_counter]["traddr"])) + log_entry.traddr = (c_ubyte * 256)( + *[ + c_ubyte(x) + for x in allow_listeners[log_entry_counter]["traddr"].encode() + ] + ) + log_entry.traddr[ + len(allow_listeners[log_entry_counter]["traddr"]) : + ] = [c_ubyte(0x20)] * ( + 256 - len(allow_listeners[log_entry_counter]["traddr"]) + ) - self_conn.log_page[1024*(log_entry_counter+1): \ - 1024*(log_entry_counter+2)] = log_entry + self_conn.log_page[ + 1024 * (log_entry_counter + 1) : 1024 * (log_entry_counter + 2) + ] = log_entry log_entry_counter += 1 else: self.logger.debug("in the process of sending log pages...") - reply = b'' + reply = b"" pdu_and_nvme_pdu_len = 8 + 16 pdu_reply = Pdu() pdu_reply.type = NVME_TCP_PDU.C2H_DATA - pdu_reply.specical_flag = 0x0c + pdu_reply.specical_flag = 0x0C pdu_reply.header_length = pdu_and_nvme_pdu_len pdu_reply.data_offset = pdu_and_nvme_pdu_len pdu_reply.packet_length = pdu_and_nvme_pdu_len + nvme_data_len @@ -824,17 +896,28 @@ def reply_get_log_page(self, conn, data, cmd_id): nvme_get_log_page_reply.genctr = self_conn.gen_cnt nvme_get_log_page_reply.numrec = len(allow_listeners) - reply = pdu_reply + nvme_tcp_data_pdu + bytes(nvme_get_log_page_reply)[:nvme_data_len] + reply = ( + pdu_reply + + nvme_tcp_data_pdu + + bytes(nvme_get_log_page_reply)[:nvme_data_len] + ) elif nvme_data_len % 1024 == 0: # reply log pages - reply = pdu_reply + nvme_tcp_data_pdu + \ - self_conn.log_page[nvme_logpage_offset:nvme_logpage_offset+nvme_data_len] + reply = ( + pdu_reply + + nvme_tcp_data_pdu + + self_conn.log_page[ + nvme_logpage_offset : nvme_logpage_offset + nvme_data_len + ] + ) self_conn.unsent_log_page_len -= nvme_data_len if self_conn.unsent_log_page_len == 0: - self_conn.log_page = b'' + self_conn.log_page = b"" self_conn.allow_listeners = [] else: - self.logger.error(f"request log page: invalid length error {nvme_data_len=}") + self.logger.error( + f"request log page: invalid length error {nvme_data_len=}" + ) return -1 try: conn.sendall(reply) @@ -849,22 +932,22 @@ def reply_keep_alive(self, conn, data, cmd_id): self.logger.debug("handle keep alive request.") self_conn = self.conn_vals[conn.fileno()] - nvme_sgl = struct.unpack_from('<16B', data, 32) + nvme_sgl = struct.unpack_from("<16B", data, 32) nvme_sgl_desc_type = nvme_sgl[15] & 0xF0 nvme_sgl_desc_sub_type = nvme_sgl[15] & 0x0F - nvme_keep_alive_dword10 = struct.unpack_from('= \ - self.conn_vals[key].keep_alive_timeout / 1000: + if ( + self.conn_vals[key].keep_alive_timeout != 0 + and time.time() - self.conn_vals[key].keep_alive_time + >= self.conn_vals[key].keep_alive_timeout / 1000 + ): # Adding locks to prevent another thread from processing sudden requests. # Is there a better way? with self.lock: - self.logger.debug(f"discover request from {self.conn_vals[key].connection} timeout.") + self.logger.debug( + f"discover request from {self.conn_vals[key].connection} timeout." + ) self.selector.unregister(self.conn_vals[key].connection) self.conn_vals[key].connection.close() del self.conn_vals[key] @@ -976,15 +1066,17 @@ def handle_timeout(self): def reply_fabric_request(self, conn, data, cmd_id): """Reply fabric request.""" - fabric_type = struct.unpack_from(' None: self.set_group_id = set_group_id - def group_id(self, request: monitor_pb2.group_id_req, context = None) -> Empty: + def group_id(self, request: monitor_pb2.group_id_req, context=None) -> Empty: self.set_group_id(request.id) return Empty() + class SubsystemHostAuth: - MAX_PSK_KEY_NAME_LENGTH = 200 # taken from SPDK SPDK_TLS_PSK_MAX_LEN + MAX_PSK_KEY_NAME_LENGTH = 200 # taken from SPDK SPDK_TLS_PSK_MAX_LEN def __init__(self): self.subsys_allow_any_hosts = defaultdict(dict) @@ -87,7 +90,9 @@ def remove_psk_host(self, subsys, host): if subsys in self.host_psk_key: self.host_psk_key[subsys].pop(host, None) if len(self.host_psk_key[subsys]) == 0: - self.host_psk_key.pop(subsys, None) # last host was removed from subsystem + self.host_psk_key.pop( + subsys, None + ) # last host was removed from subsystem def is_psk_host(self, subsys, host) -> bool: key = self.get_host_psk_key(subsys, host) @@ -107,7 +112,9 @@ def remove_dhchap_host(self, subsys, host): if subsys in self.host_dhchap_key: self.host_dhchap_key[subsys].pop(host, None) if len(self.host_dhchap_key[subsys]) == 0: - self.host_dhchap_key.pop(subsys, None) # last host was removed from subsystem + self.host_dhchap_key.pop( + subsys, None + ) # last host was removed from subsystem def is_dhchap_host(self, subsys, host) -> bool: key = self.get_host_dhchap_key(subsys, host) @@ -163,6 +170,7 @@ def get_subsystem_dhchap_key(self, subsys) -> str: key = self.subsys_dhchap_key[subsys] return key + class NamespaceInfo: def __init__(self, nsid, bdev, uuid, anagrpid, no_auto_visible): self.nsid = nsid @@ -199,6 +207,7 @@ def host_count(self): def set_ana_group_id(self, anagrpid): self.anagrpid = anagrpid + class NamespacesLocalList: EMPTY_NAMESPACE = NamespaceInfo(None, None, None, 0, False) @@ -211,16 +220,20 @@ def remove_namespace(self, nqn, nsid=None): if nsid in self.namespace_list[nqn]: self.namespace_list[nqn].pop(nsid, None) if len(self.namespace_list[nqn]) == 0: - self.namespace_list.pop(nqn, None) # last namespace of subsystem was removed + self.namespace_list.pop( + nqn, None + ) # last namespace of subsystem was removed else: self.namespace_list.pop(nqn, None) def add_namespace(self, nqn, nsid, bdev, uuid, anagrpid, no_auto_visible): if not bdev: bdev = GatewayService.find_unique_bdev_name(uuid) - self.namespace_list[nqn][nsid] = NamespaceInfo(nsid, bdev, uuid, anagrpid, no_auto_visible) + self.namespace_list[nqn][nsid] = NamespaceInfo( + nsid, bdev, uuid, anagrpid, no_auto_visible + ) - def find_namespace(self, nqn, nsid, uuid = None) -> NamespaceInfo: + def find_namespace(self, nqn, nsid, uuid=None) -> NamespaceInfo: if nqn not in self.namespace_list: return NamespacesLocalList.EMPTY_NAMESPACE @@ -237,7 +250,7 @@ def find_namespace(self, nqn, nsid, uuid = None) -> NamespaceInfo: return NamespacesLocalList.EMPTY_NAMESPACE - def get_namespace_count(self, nqn, no_auto_visible = None, min_hosts = 0) -> int: + def get_namespace_count(self, nqn, no_auto_visible=None, min_hosts=0) -> int: if nqn and nqn not in self.namespace_list: return 0 @@ -245,7 +258,7 @@ def get_namespace_count(self, nqn, no_auto_visible = None, min_hosts = 0) -> int subsystems = [nqn] else: subsystems = self.namespace_list.keys() - + ns_count = 0 for one_subsys in subsystems: for nsid in self.namespace_list[one_subsys]: @@ -253,7 +266,10 @@ def get_namespace_count(self, nqn, no_auto_visible = None, min_hosts = 0) -> int if ns.empty(): continue if no_auto_visible is not None: - if ns.no_auto_visible == no_auto_visible and ns.host_count() >= min_hosts: + if ( + ns.no_auto_visible == no_auto_visible + and ns.host_count() >= min_hosts + ): ns_count += 1 else: if ns.host_count() >= min_hosts: @@ -261,13 +277,16 @@ def get_namespace_count(self, nqn, no_auto_visible = None, min_hosts = 0) -> int return ns_count - def get_namespace_infos_for_anagrpid(self, nqn: str, anagrpid: int) -> Iterator[NamespaceInfo]: + def get_namespace_infos_for_anagrpid( + self, nqn: str, anagrpid: int + ) -> Iterator[NamespaceInfo]: """Yield NamespaceInfo instances for a given nqn and anagrpid.""" if nqn in self.namespace_list: for ns_info in self.namespace_list[nqn].values(): if ns_info.anagrpid == anagrpid: yield ns_info + class GatewayService(pb2_grpc.GatewayServicer): """Implements gateway service interface. @@ -294,7 +313,17 @@ class GatewayService(pb2_grpc.GatewayServicer): MAX_NAMESPACES_DEFAULT = 256 MAX_HOSTS_PER_SUBSYS_DEFAULT = 32 - def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rpc_lock, omap_lock: OmapLock, group_id: int, spdk_rpc_client, spdk_rpc_subsystems_client, ceph_utils: CephUtils) -> None: + def __init__( + self, + config: GatewayConfig, + gateway_state: GatewayStateHandler, + rpc_lock, + omap_lock: OmapLock, + group_id: int, + spdk_rpc_client, + spdk_rpc_subsystems_client, + ceph_utils: CephUtils, + ) -> None: """Constructor""" self.gw_logger_object = GatewayLogger(config) self.logger = self.gw_logger_object.logger @@ -309,14 +338,20 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp requested_hugepages_val = requested_hugepages_val.strip() try: requested_hugepages_val = int(requested_hugepages_val) - self.logger.info(f"Requested huge pages count is {requested_hugepages_val}") + self.logger.info( + f"Requested huge pages count is {requested_hugepages_val}" + ) except ValueError: - self.logger.warning(f"Requested huge pages count value {requested_hugepages_val} is not numeric") + self.logger.warning( + f"Requested huge pages count value {requested_hugepages_val} is not numeric" + ) requested_hugepages_val = None hugepages_file = os.getenv("HUGEPAGES_DIR", "") if not hugepages_file: hugepages_file = "/sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" - self.logger.warning("No huge pages file defined, will use /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages") + self.logger.warning( + "No huge pages file defined, will use /sys/kernel/mm/hugepages/hugepages-2048kB/nr_hugepages" + ) else: hugepages_file = hugepages_file.strip() if os.access(hugepages_file, os.F_OK): @@ -330,14 +365,26 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp hugepages_val = int(hugepages_val) self.logger.info(f"Actual huge pages count is {hugepages_val}") except ValueError: - self.logger.warning(f"Actual huge pages count value {hugepages_val} is not numeric") + self.logger.warning( + f"Actual huge pages count value {hugepages_val} is not numeric" + ) hugepages_val = "" - if requested_hugepages_val and hugepages_val != "" and requested_hugepages_val > hugepages_val: - self.logger.warning(f"The actual huge page count {hugepages_val} is smaller than the requested value of {requested_hugepages_val}") + if ( + requested_hugepages_val + and hugepages_val != "" + and requested_hugepages_val > hugepages_val + ): + self.logger.warning( + f"The actual huge page count {hugepages_val} is smaller than the requested value of {requested_hugepages_val}" + ) else: - self.logger.warning(f"Can't read actual huge pages count value from {hugepages_file}") + self.logger.warning( + f"Can't read actual huge pages count value from {hugepages_file}" + ) except Exception as ex: - self.logger.exception(f"Can't read actual huge pages count value from {hugepages_file}") + self.logger.exception( + f"Can't read actual huge pages count value from {hugepages_file}" + ) else: self.logger.warning(f"Can't find huge pages file {hugepages_file}") self.config = config @@ -353,31 +400,51 @@ def __init__(self, config: GatewayConfig, gateway_state: GatewayStateHandler, rp self.gateway_name = self.config.get("gateway", "name") if not self.gateway_name: self.gateway_name = socket.gethostname() - override_hostname = self.config.get_with_default("gateway", "override_hostname", "") + override_hostname = self.config.get_with_default( + "gateway", "override_hostname", "" + ) if override_hostname: self.host_name = override_hostname - self.logger.info(f"Gateway's host name was overridden to {override_hostname}") + self.logger.info( + f"Gateway's host name was overridden to {override_hostname}" + ) else: self.host_name = socket.gethostname() - self.verify_nqns = self.config.getboolean_with_default("gateway", "verify_nqns", True) + self.verify_nqns = self.config.getboolean_with_default( + "gateway", "verify_nqns", True + ) self.gateway_group = self.config.get_with_default("gateway", "group", "") - self.max_hosts_per_namespace = self.config.getint_with_default("gateway", "max_hosts_per_namespace", 1) - self.max_namespaces_with_netmask = self.config.getint_with_default("gateway", "max_namespaces_with_netmask", 1000) - self.max_subsystems = self.config.getint_with_default("gateway", "max_subsystems", GatewayService.MAX_SUBSYSTEMS_DEFAULT) - self.max_namespaces = self.config.getint_with_default("gateway", "max_namespaces", GatewayService.MAX_NAMESPACES_DEFAULT) - self.max_hosts_per_subsystem = self.config.getint_with_default("gateway", "max_hosts_per_subsystem", GatewayService.MAX_HOSTS_PER_SUBSYS_DEFAULT) + self.max_hosts_per_namespace = self.config.getint_with_default( + "gateway", "max_hosts_per_namespace", 1 + ) + self.max_namespaces_with_netmask = self.config.getint_with_default( + "gateway", "max_namespaces_with_netmask", 1000 + ) + self.max_subsystems = self.config.getint_with_default( + "gateway", "max_subsystems", GatewayService.MAX_SUBSYSTEMS_DEFAULT + ) + self.max_namespaces = self.config.getint_with_default( + "gateway", "max_namespaces", GatewayService.MAX_NAMESPACES_DEFAULT + ) + self.max_hosts_per_subsystem = self.config.getint_with_default( + "gateway", + "max_hosts_per_subsystem", + GatewayService.MAX_HOSTS_PER_SUBSYS_DEFAULT, + ) self.gateway_pool = self.config.get_with_default("ceph", "pool", "") self.ana_map = defaultdict(dict) self.cluster_nonce = {} self.bdev_cluster = {} - self.bdev_params = {} + self.bdev_params = {} self.subsystem_nsid_bdev_and_uuid = NamespacesLocalList() self.subsystem_listeners = defaultdict(set) self._init_cluster_context() self.subsys_max_ns = {} self.host_info = SubsystemHostAuth() - def get_directories_for_key_file(self, key_type : str, subsysnqn : str, create_dir : bool = False) -> []: + def get_directories_for_key_file( + self, key_type: str, subsysnqn: str, create_dir: bool = False + ) -> []: tmp_dirs = [] dir_prefix = f"{key_type}_{subsysnqn}_" @@ -403,13 +470,17 @@ def get_directories_for_key_file(self, key_type : str, subsysnqn : str, create_d return None return [tmp_dir_name] - def create_host_key_file(self, key_type : str, subsysnqn : str, hostnqn : str, key_value : str) -> str: + def create_host_key_file( + self, key_type: str, subsysnqn: str, hostnqn: str, key_value: str + ) -> str: assert subsysnqn, "Subsystem NQN can't be empty" assert hostnqn, "Host NQN can't be empty" assert key_type, "Key type can't be empty" assert key_value, "Key value can't be empty" - tmp_dir_names = self.get_directories_for_key_file(key_type, subsysnqn, create_dir = True) + tmp_dir_names = self.get_directories_for_key_file( + key_type, subsysnqn, create_dir=True + ) if not tmp_dir_names: return None @@ -417,7 +488,9 @@ def create_host_key_file(self, key_type : str, subsysnqn : str, hostnqn : str, k filepath = None keyfile_prefix = f"{hostnqn}_" try: - (file_fd, filepath) = tempfile.mkstemp(prefix=keyfile_prefix, dir=tmp_dir_names[0], text=True) + (file_fd, filepath) = tempfile.mkstemp( + prefix=keyfile_prefix, dir=tmp_dir_names[0], text=True + ) except Exception: self.logger.exception("Error creating key file") return None @@ -436,17 +509,23 @@ def create_host_key_file(self, key_type : str, subsysnqn : str, hostnqn : str, k return None return filepath - def create_host_psk_file(self, subsysnqn : str, hostnqn : str, key_value : str) -> str: + def create_host_psk_file(self, subsysnqn: str, hostnqn: str, key_value: str) -> str: return self.create_host_key_file(self.PSK_PREFIX, subsysnqn, hostnqn, key_value) - def create_host_dhchap_file(self, subsysnqn : str, hostnqn : str, key_value : str) -> str: - return self.create_host_key_file(self.DHCHAP_PREFIX, subsysnqn, hostnqn, key_value) + def create_host_dhchap_file( + self, subsysnqn: str, hostnqn: str, key_value: str + ) -> str: + return self.create_host_key_file( + self.DHCHAP_PREFIX, subsysnqn, hostnqn, key_value + ) - def remove_host_key_file(self, key_type : str, subsysnqn : str, hostnqn : str) -> None: + def remove_host_key_file(self, key_type: str, subsysnqn: str, hostnqn: str) -> None: assert key_type, "Key type can't be empty" assert subsysnqn, "Subsystem NQN can't be empty" - tmp_dir_names = self.get_directories_for_key_file(key_type, subsysnqn, create_dir = False) + tmp_dir_names = self.get_directories_for_key_file( + key_type, subsysnqn, create_dir=False + ) if not tmp_dir_names: return @@ -454,7 +533,7 @@ def remove_host_key_file(self, key_type : str, subsysnqn : str, hostnqn : str) - if not hostnqn: for one_tmp_dir in tmp_dir_names: try: - shutil.rmtree(one_tmp_dir, ignore_errors = True) + shutil.rmtree(one_tmp_dir, ignore_errors=True) except Exception: pass return @@ -469,50 +548,66 @@ def remove_host_key_file(self, key_type : str, subsysnqn : str, hostnqn : str) - self.logger.exception(f"Error deleting file {f.name}") pass - def remove_host_psk_file(self, subsysnqn : str, hostnqn : str) -> None: + def remove_host_psk_file(self, subsysnqn: str, hostnqn: str) -> None: self.remove_host_key_file(self.PSK_PREFIX, subsysnqn, hostnqn) - def remove_host_dhchap_file(self, subsysnqn : str, hostnqn : str) -> None: + def remove_host_dhchap_file(self, subsysnqn: str, hostnqn: str) -> None: self.remove_host_key_file(self.DHCHAP_PREFIX, subsysnqn, hostnqn) - def remove_all_host_key_files(self, subsysnqn : str, hostnqn : str) -> None: + def remove_all_host_key_files(self, subsysnqn: str, hostnqn: str) -> None: self.remove_host_psk_file(subsysnqn, hostnqn) self.remove_host_dhchap_file(subsysnqn, hostnqn) - def remove_all_subsystem_key_files(self, subsysnqn : str) -> None: + def remove_all_subsystem_key_files(self, subsysnqn: str) -> None: self.remove_all_host_key_files(subsysnqn, None) @staticmethod - def construct_key_name_for_keyring(subsysnqn : str, hostnqn : str, prefix : str = None) -> str: - key_name = hashlib.sha256(subsysnqn.encode()).hexdigest() + "_" + hashlib.sha256(hostnqn.encode()).hexdigest() + def construct_key_name_for_keyring( + subsysnqn: str, hostnqn: str, prefix: str = None + ) -> str: + key_name = ( + hashlib.sha256(subsysnqn.encode()).hexdigest() + + "_" + + hashlib.sha256(hostnqn.encode()).hexdigest() + ) if prefix: key_name = prefix + "_" + key_name return key_name - def remove_key_from_keyring(self, key_type : str, subsysnqn : str, hostnqn : str) -> None: - assert self.rpc_lock.locked(), "RPC is unlocked when calling remove_key_from_keyring()" - key_name = GatewayService.construct_key_name_for_keyring(subsysnqn, hostnqn, key_type) + def remove_key_from_keyring( + self, key_type: str, subsysnqn: str, hostnqn: str + ) -> None: + assert ( + self.rpc_lock.locked() + ), "RPC is unlocked when calling remove_key_from_keyring()" + key_name = GatewayService.construct_key_name_for_keyring( + subsysnqn, hostnqn, key_type + ) try: rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, key_name) except Exception: pass - def remove_psk_key_from_keyring(self, subsysnqn : str, hostnqn : str) -> None: + def remove_psk_key_from_keyring(self, subsysnqn: str, hostnqn: str) -> None: self.remove_key_from_keyring(self.PSK_PREFIX, subsysnqn, hostnqn) - def remove_dhchap_key_from_keyring(self, subsysnqn : str, hostnqn : str) -> None: + def remove_dhchap_key_from_keyring(self, subsysnqn: str, hostnqn: str) -> None: self.remove_key_from_keyring(self.DHCHAP_PREFIX, subsysnqn, hostnqn) - def remove_dhchap_controller_key_from_keyring(self, subsysnqn : str, hostnqn : str) -> None: + def remove_dhchap_controller_key_from_keyring( + self, subsysnqn: str, hostnqn: str + ) -> None: self.remove_key_from_keyring(self.DHCHAP_CONTROLLER_PREFIX, subsysnqn, hostnqn) - def remove_all_host_keys_from_keyring(self, subsysnqn : str, hostnqn : str) -> None: + def remove_all_host_keys_from_keyring(self, subsysnqn: str, hostnqn: str) -> None: self.remove_psk_key_from_keyring(subsysnqn, hostnqn) self.remove_dhchap_key_from_keyring(subsysnqn, hostnqn) self.remove_dhchap_controller_key_from_keyring(subsysnqn, hostnqn) - def remove_all_subsystem_keys_from_keyring(self, subsysnqn : str) -> None: - assert self.rpc_lock.locked(), "RPC is unlocked when calling remove_all_subsystem_keys_from_keyring()" + def remove_all_subsystem_keys_from_keyring(self, subsysnqn: str) -> None: + assert ( + self.rpc_lock.locked() + ), "RPC is unlocked when calling remove_all_subsystem_keys_from_keyring()" try: key_list = rpc_keyring.keyring_get_keys(self.spdk_rpc_client) except Exception: @@ -529,8 +624,11 @@ def remove_all_subsystem_keys_from_keyring(self, subsysnqn : str) -> None: continue if not key_name or not key_path: continue - if (key_path.startswith(f"{self.KEYS_DIR}/{self.PSK_PREFIX}_{subsysnqn}_") or - key_path.startswith(f"{self.KEYS_DIR}/{self.DHCHAP_PREFIX}_{subsysnqn}_")): + if key_path.startswith( + f"{self.KEYS_DIR}/{self.PSK_PREFIX}_{subsysnqn}_" + ) or key_path.startswith( + f"{self.KEYS_DIR}/{self.DHCHAP_PREFIX}_{subsysnqn}_" + ): try: rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, key_name) except Exception: @@ -564,7 +662,7 @@ def parse_json_exeption(self, ex): if resp["code"] < 0: resp["code"] = -resp["code"] else: - resp={} + resp = {} if "timeout" in ex.message.lower(): resp["code"] = errno.ETIMEDOUT else: @@ -576,11 +674,17 @@ def parse_json_exeption(self, ex): def _init_cluster_context(self) -> None: """Init cluster context management variables""" self.clusters = defaultdict(dict) - self.bdevs_per_cluster = self.config.getint_with_default("spdk", "bdevs_per_cluster", 32) + self.bdevs_per_cluster = self.config.getint_with_default( + "spdk", "bdevs_per_cluster", 32 + ) if self.bdevs_per_cluster < 1: - raise Exception(f"invalid configuration: spdk.bdevs_per_cluster_contexts {self.bdevs_per_cluster} < 1") + raise Exception( + f"invalid configuration: spdk.bdevs_per_cluster_contexts {self.bdevs_per_cluster} < 1" + ) self.logger.info(f"NVMeoF bdevs per cluster: {self.bdevs_per_cluster}") - self.librbd_core_mask = self.config.get_with_default("spdk", "librbd_core_mask", None) + self.librbd_core_mask = self.config.get_with_default( + "spdk", "librbd_core_mask", None + ) self.rados_id = self.config.get_with_default("ceph", "id", "") if self.rados_id == "": self.rados_id = None @@ -598,7 +702,9 @@ def _get_cluster(self, anagrp: int) -> str: self.clusters[anagrp][cluster_name] = 1 else: self.clusters[anagrp][cluster_name] += 1 - self.logger.info(f"get_cluster {cluster_name=} number bdevs: {self.clusters[anagrp][cluster_name]}") + self.logger.info( + f"get_cluster {cluster_name=} number bdevs: {self.clusters[anagrp][cluster_name]}" + ) return cluster_name def _put_cluster(self, name: str) -> None: @@ -609,17 +715,20 @@ def _put_cluster(self, name: str) -> None: # free the cluster context if no longer used by any bdev if self.clusters[anagrp][name] == 0: ret = rpc_bdev.bdev_rbd_unregister_cluster( - self.spdk_rpc_client, - name = name + self.spdk_rpc_client, name=name ) self.logger.info(f"Free cluster {name=} {ret=}") assert ret self.clusters[anagrp].pop(name) - else : - self.logger.info(f"put_cluster {name=} number bdevs: {self.clusters[anagrp][name]}") + else: + self.logger.info( + f"put_cluster {name=} number bdevs: {self.clusters[anagrp][name]}" + ) return - assert False, f"Cluster {name} is not found" # we should find the cluster in our state + assert ( + False + ), f"Cluster {name} is not found" # we should find the cluster in our state def _alloc_cluster_name(self, anagrp: int) -> str: """Allocates a new cluster name for ana group""" @@ -635,9 +744,9 @@ def _alloc_cluster(self, anagrp: int) -> str: name = self._alloc_cluster_name(anagrp) nonce = rpc_bdev.bdev_rbd_register_cluster( self.spdk_rpc_client, - name = name, - user_id = self.rados_id, - core_mask = self.librbd_core_mask, + name=name, + user_id=self.rados_id, + core_mask=self.librbd_core_mask, ) with self.shared_state_lock: self.logger.info(f"Allocated cluster {name=} {nonce=} {anagrp=}") @@ -648,18 +757,34 @@ def _grpc_function_with_lock(self, func, request, context): with self.rpc_lock: rc = func(request, context) if not self.omap_lock.omap_file_disable_unlock: - assert not self.omap_lock.locked(), f"OMAP is still locked when we're out of function {func}" + assert ( + not self.omap_lock.locked() + ), f"OMAP is still locked when we're out of function {func}" return rc def execute_grpc_function(self, func, request, context): """This functions handles RPC lock by wrapping 'func' with - self._grpc_function_with_lock, and assumes (?!) the function 'func' - called might take OMAP lock internally, however does NOT ensure - taking OMAP lock in any way. + self._grpc_function_with_lock, and assumes (?!) the function 'func' + called might take OMAP lock internally, however does NOT ensure + taking OMAP lock in any way. """ - return self.omap_lock.execute_omap_locking_function(self._grpc_function_with_lock, func, request, context) + return self.omap_lock.execute_omap_locking_function( + self._grpc_function_with_lock, func, request, context + ) - def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, block_size, create_image, rbd_image_size, context, peer_msg = ""): + def create_bdev( + self, + anagrp: int, + name, + uuid, + rbd_pool_name, + rbd_image_name, + block_size, + create_image, + rbd_image_size, + context, + peer_msg="", + ): """Creates a bdev from an RBD image.""" if create_image: @@ -667,32 +792,48 @@ def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, bl else: cr_img_msg = "will not create image if doesn't exist" - self.logger.info(f"Received request to create bdev {name} from" - f" {rbd_pool_name}/{rbd_image_name} (size {rbd_image_size} bytes)" - f" with block size {block_size}, {cr_img_msg}, context={context}{peer_msg}") + self.logger.info( + f"Received request to create bdev {name} from" + f" {rbd_pool_name}/{rbd_image_name} (size {rbd_image_size} bytes)" + f" with block size {block_size}, {cr_img_msg}, context={context}{peer_msg}" + ) if block_size == 0: - return BdevStatus(status=errno.EINVAL, - error_message=f"Failure creating bdev {name}: block size can't be zero") + return BdevStatus( + status=errno.EINVAL, + error_message=f"Failure creating bdev {name}: block size can't be zero", + ) if create_image: if rbd_image_size <= 0: - return BdevStatus(status=errno.EINVAL, - error_message=f"Failure creating bdev {name}: image size must be positive") + return BdevStatus( + status=errno.EINVAL, + error_message=f"Failure creating bdev {name}: image size must be positive", + ) if rbd_image_size % (1024 * 1024): - return BdevStatus(status=errno.EINVAL, - error_message=f"Failure creating bdev {name}: image size must be aligned to MiBs") + return BdevStatus( + status=errno.EINVAL, + error_message=f"Failure creating bdev {name}: image size must be aligned to MiBs", + ) rc = self.ceph_utils.pool_exists(rbd_pool_name) if not rc: - return BdevStatus(status=errno.ENODEV, - error_message=f"Failure creating bdev {name}: RBD pool {rbd_pool_name} doesn't exist") + return BdevStatus( + status=errno.ENODEV, + error_message=f"Failure creating bdev {name}: RBD pool {rbd_pool_name} doesn't exist", + ) try: - rc = self.ceph_utils.create_image(rbd_pool_name, rbd_image_name, rbd_image_size) + rc = self.ceph_utils.create_image( + rbd_pool_name, rbd_image_name, rbd_image_size + ) if rc: - self.logger.info(f"Image {rbd_pool_name}/{rbd_image_name} created, size is {rbd_image_size} bytes") + self.logger.info( + f"Image {rbd_pool_name}/{rbd_image_name} created, size is {rbd_image_size} bytes" + ) else: - self.logger.info(f"Image {rbd_pool_name}/{rbd_image_name} already exists with size {rbd_image_size} bytes") + self.logger.info( + f"Image {rbd_pool_name}/{rbd_image_name} already exists with size {rbd_image_size} bytes" + ) except Exception as ex: errcode = 0 msg = "" @@ -704,12 +845,17 @@ def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, bl errcode = errno.ENODEV if not msg: msg = str(ex) - errmsg = f"Can't create RBD image {rbd_pool_name}/{rbd_image_name}: {msg}" + errmsg = ( + f"Can't create RBD image {rbd_pool_name}/{rbd_image_name}: {msg}" + ) self.logger.exception(errmsg) - return BdevStatus(status=errcode, error_message=f"Failure creating bdev {name}: {errmsg}") + return BdevStatus( + status=errcode, + error_message=f"Failure creating bdev {name}: {errmsg}", + ) try: - cluster_name=self._get_cluster(anagrp) + cluster_name = self._get_cluster(anagrp) bdev_name = rpc_bdev.bdev_rbd_create( self.spdk_rpc_client, name=name, @@ -721,9 +867,17 @@ def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, bl ) with self.shared_state_lock: self.bdev_cluster[name] = cluster_name - self.bdev_params[name] = {'uuid':uuid, 'pool_name':rbd_pool_name, 'image_name':rbd_image_name, 'image_size':rbd_image_size, 'block_size': block_size} - - self.logger.debug(f"bdev_rbd_create: {bdev_name}, cluster_name {cluster_name}") + self.bdev_params[name] = { + "uuid": uuid, + "pool_name": rbd_pool_name, + "image_name": rbd_image_name, + "image_size": rbd_image_size, + "block_size": block_size, + } + + self.logger.debug( + f"bdev_rbd_create: {bdev_name}, cluster_name {cluster_name}" + ) except Exception as ex: self._put_cluster(cluster_name) errmsg = f"bdev_rbd_create {name} failed" @@ -742,14 +896,18 @@ def create_bdev(self, anagrp: int, name, uuid, rbd_pool_name, rbd_image_name, bl self.logger.error(errmsg) return BdevStatus(status=errno.ENODEV, error_message=errmsg) - assert name == bdev_name, f"Created bdev name {bdev_name} differs from requested name {name}" + assert ( + name == bdev_name + ), f"Created bdev name {bdev_name} differs from requested name {name}" return BdevStatus(status=0, error_message=os.strerror(0), bdev_name=name) - def resize_bdev(self, bdev_name, new_size, peer_msg = ""): + def resize_bdev(self, bdev_name, new_size, peer_msg=""): """Resizes a bdev.""" - self.logger.info(f"Received request to resize bdev {bdev_name} to {new_size} MiB{peer_msg}") + self.logger.info( + f"Received request to resize bdev {bdev_name} to {new_size} MiB{peer_msg}" + ) assert self.rpc_lock.locked(), "RPC is unlocked when calling resize_bdev()" rbd_pool_name = None rbd_image_name = None @@ -761,19 +919,29 @@ def resize_bdev(self, bdev_name, new_size, peer_msg = ""): rbd_pool_name = rbd_info["pool_name"] rbd_image_name = rbd_info["rbd_name"] except KeyError as err: - self.logger.warning(f"Key {err} is not found, will not check size for shrinkage") + self.logger.warning( + f"Key {err} is not found, will not check size for shrinkage" + ) pass else: - self.logger.warning(f"Can't get information for associated block device {bdev_name}, won't check size for shrinkage") + self.logger.warning( + f"Can't get information for associated block device {bdev_name}, won't check size for shrinkage" + ) if rbd_pool_name and rbd_image_name: try: - current_size = self.ceph_utils.get_image_size(rbd_pool_name, rbd_image_name) + current_size = self.ceph_utils.get_image_size( + rbd_pool_name, rbd_image_name + ) if current_size > new_size * 1024 * 1024: - return pb2.req_status(status=errno.EINVAL, - error_message=f"new size {new_size * 1024 * 1024} bytes is smaller than current size {current_size} bytes") + return pb2.req_status( + status=errno.EINVAL, + error_message=f"new size {new_size * 1024 * 1024} bytes is smaller than current size {current_size} bytes", + ) except Exception as ex: - self.logger.warning(f"Error trying to get the size of image {rbd_pool_name}/{rbd_image_name}, won't check size for shrinkage:\n{ex}") + self.logger.warning( + f"Error trying to get the size of image {rbd_pool_name}/{rbd_image_name}, won't check size for shrinkage:\n{ex}" + ) pass try: @@ -851,7 +1019,9 @@ def subsystem_already_exists(self, context, nqn) -> bool: if subnqn == nqn: return True except Exception: - self.logger.exception(f"Got exception while parsing {val}, will continue") + self.logger.exception( + f"Got exception while parsing {val}, will continue" + ) continue return False @@ -875,7 +1045,7 @@ def get_peer_message(self, context) -> str: if not context: return "" - if not hasattr(context, 'peer'): + if not hasattr(context, "peer"): return "" try: @@ -899,58 +1069,83 @@ def get_peer_message(self, context) -> str: def create_subsystem_safe(self, request, context): """Creates a subsystem.""" - create_subsystem_error_prefix = f"Failure creating subsystem {request.subsystem_nqn}" + create_subsystem_error_prefix = ( + f"Failure creating subsystem {request.subsystem_nqn}" + ) peer_msg = self.get_peer_message(context) self.logger.info( - f"Received request to create subsystem {request.subsystem_nqn}, enable_ha: {request.enable_ha}, max_namespaces: {request.max_namespaces}, no group append: {request.no_group_append}, dhchap_key: {request.dhchap_key}, context: {context}{peer_msg}") + f"Received request to create subsystem {request.subsystem_nqn}, enable_ha: {request.enable_ha}, max_namespaces: {request.max_namespaces}, no group append: {request.no_group_append}, dhchap_key: {request.dhchap_key}, context: {context}{peer_msg}" + ) if not request.enable_ha: - errmsg = f"{create_subsystem_error_prefix}: HA must be enabled for subsystems" + errmsg = ( + f"{create_subsystem_error_prefix}: HA must be enabled for subsystems" + ) self.logger.error(f"{errmsg}") - return pb2.subsys_status(status = errno.EINVAL, error_message = errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.EINVAL, error_message=errmsg, nqn=request.subsystem_nqn + ) if not request.subsystem_nqn: errmsg = f"Failure creating subsystem, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.subsys_status(status = errno.EINVAL, error_message = errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.EINVAL, error_message=errmsg, nqn=request.subsystem_nqn + ) if not request.max_namespaces: request.max_namespaces = self.max_namespaces else: if request.max_namespaces > self.max_namespaces: - self.logger.warning(f"The requested max number of namespaces for subsystem {request.subsystem_nqn} ({request.max_namespaces}) is greater than the global limit on the number of namespaces ({self.max_namespaces}), will continue") + self.logger.warning( + f"The requested max number of namespaces for subsystem {request.subsystem_nqn} ({request.max_namespaces}) is greater than the global limit on the number of namespaces ({self.max_namespaces}), will continue" + ) errmsg = "" if not GatewayState.is_key_element_valid(request.subsystem_nqn): - errmsg = f"{create_subsystem_error_prefix}: Invalid NQN \"{request.subsystem_nqn}\", contains invalid characters" + errmsg = f'{create_subsystem_error_prefix}: Invalid NQN "{request.subsystem_nqn}", contains invalid characters' self.logger.error(f"{errmsg}") - return pb2.subsys_status(status = errno.EINVAL, error_message = errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.EINVAL, error_message=errmsg, nqn=request.subsystem_nqn + ) if self.verify_nqns: rc = GatewayUtils.is_valid_nqn(request.subsystem_nqn) if rc[0] != 0: errmsg = f"{create_subsystem_error_prefix}: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.subsys_status(status = rc[0], error_message = errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=rc[0], error_message=errmsg, nqn=request.subsystem_nqn + ) if GatewayUtils.is_discovery_nqn(request.subsystem_nqn): - errmsg = f"{create_subsystem_error_prefix}: Can't create a discovery subsystem" + errmsg = ( + f"{create_subsystem_error_prefix}: Can't create a discovery subsystem" + ) self.logger.error(f"{errmsg}") - return pb2.subsys_status(status = errno.EINVAL, error_message = errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.EINVAL, error_message=errmsg, nqn=request.subsystem_nqn + ) if len(self.subsys_max_ns) >= self.max_subsystems: errmsg = f"{create_subsystem_error_prefix}: Maximal number of subsystems ({self.max_subsystems}) has already been reached" self.logger.error(f"{errmsg}") - return pb2.subsys_status(status = errno.E2BIG, error_message = errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.E2BIG, error_message=errmsg, nqn=request.subsystem_nqn + ) if context: if request.no_group_append or not self.gateway_group: self.logger.info(f"Subsystem NQN will not be changed") else: - group_name_to_use = self.gateway_group.replace(GatewayState.OMAP_KEY_DELIMITER, "-") + group_name_to_use = self.gateway_group.replace( + GatewayState.OMAP_KEY_DELIMITER, "-" + ) request.subsystem_nqn += f".{group_name_to_use}" - self.logger.info(f"Subsystem NQN was changed to {request.subsystem_nqn}, adding the group name") + self.logger.info( + f"Subsystem NQN was changed to {request.subsystem_nqn}, adding the group name" + ) # Set client ID range according to group id assigned by the monitor offset = self.group_id * CNTLID_RANGE_SIZE @@ -961,7 +1156,9 @@ def create_subsystem_safe(self, request, context): random.seed() randser = random.randint(2, 99999999999999) request.serial_number = f"Ceph{randser}" - self.logger.info(f"No serial number specified for {request.subsystem_nqn}, will use {request.serial_number}") + self.logger.info( + f"No serial number specified for {request.subsystem_nqn}, will use {request.serial_number}" + ) ret = False omap_lock = self.omap_lock.get_omap_lock_to_use(context) @@ -969,17 +1166,25 @@ def create_subsystem_safe(self, request, context): errmsg = "" try: subsys_using_serial = None - subsys_already_exists = self.subsystem_already_exists(context, request.subsystem_nqn) + subsys_already_exists = self.subsystem_already_exists( + context, request.subsystem_nqn + ) if subsys_already_exists: errmsg = f"Subsystem already exists" else: - subsys_using_serial = self.serial_number_already_used(context, request.serial_number) + subsys_using_serial = self.serial_number_already_used( + context, request.serial_number + ) if subsys_using_serial: errmsg = f"Serial number {request.serial_number} already used by subsystem {subsys_using_serial}" if subsys_already_exists or subsys_using_serial: errmsg = f"{create_subsystem_error_prefix}: {errmsg}" self.logger.error(f"{errmsg}") - return pb2.subsys_status(status=errno.EEXIST, error_message=errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.EEXIST, + error_message=errmsg, + nqn=request.subsystem_nqn, + ) ret = rpc_nvmf.nvmf_create_subsystem( self.spdk_rpc_client, nqn=request.subsystem_nqn, @@ -988,11 +1193,13 @@ def create_subsystem_safe(self, request, context): max_namespaces=request.max_namespaces, min_cntlid=min_cntlid, max_cntlid=max_cntlid, - ana_reporting = True, + ana_reporting=True, ) self.subsys_max_ns[request.subsystem_nqn] = request.max_namespaces if request.dhchap_key: - self.host_info.add_dhchap_key_to_subsystem(request.subsystem_nqn, request.dhchap_key) + self.host_info.add_dhchap_key_to_subsystem( + request.subsystem_nqn, request.dhchap_key + ) self.logger.debug(f"create_subsystem {request.subsystem_nqn}: {ret}") except Exception as ex: self.logger.exception(create_subsystem_error_prefix) @@ -1002,26 +1209,41 @@ def create_subsystem_safe(self, request, context): if resp: status = resp["code"] errmsg = f"{create_subsystem_error_prefix}: {resp['message']}" - return pb2.subsys_status(status=status, error_message=errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=status, error_message=errmsg, nqn=request.subsystem_nqn + ) # Just in case SPDK failed with no exception if not ret: self.logger.error(create_subsystem_error_prefix) - return pb2.subsys_status(status=errno.EINVAL, error_message=create_subsystem_error_prefix, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.EINVAL, + error_message=create_subsystem_error_prefix, + nqn=request.subsystem_nqn, + ) if context: # Update gateway state try: json_req = json_format.MessageToJson( - request, preserving_proto_field_name=True, including_default_value_fields=True) + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) self.gateway_state.add_subsystem(request.subsystem_nqn, json_req) except Exception as ex: errmsg = f"Error persisting subsystem {request.subsystem_nqn}" self.logger.exception(errmsg) errmsg = f"{errmsg}:\n{ex}" - return pb2.subsys_status(status=errno.EINVAL, error_message=errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.EINVAL, + error_message=errmsg, + nqn=request.subsystem_nqn, + ) - return pb2.subsys_status(status=0, error_message=os.strerror(0), nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=0, error_message=os.strerror(0), nqn=request.subsystem_nqn + ) def create_subsystem(self, request, context=None): return self.execute_grpc_function(self.create_subsystem_safe, request, context) @@ -1038,7 +1260,9 @@ def get_subsystem_namespaces(self, nqn) -> list: nsid = ns["nsid"] ns_list.append(nsid) except Exception: - self.logger.exception(f"Got exception trying to get subsystem {nqn} namespaces") + self.logger.exception( + f"Got exception trying to get subsystem {nqn} namespaces" + ) pass return ns_list @@ -1053,7 +1277,9 @@ def subsystem_has_listeners(self, nqn) -> bool: if lsnr["nqn"] == nqn: return True except Exception: - self.logger.exception(f"Got exception trying to get subsystem {nqn} listener") + self.logger.exception( + f"Got exception trying to get subsystem {nqn} listener" + ) pass return False @@ -1075,7 +1301,9 @@ def remove_subsystem_from_state(self, nqn, context): def delete_subsystem_safe(self, request, context): """Deletes a subsystem.""" - delete_subsystem_error_prefix = f"Failure deleting subsystem {request.subsystem_nqn}" + delete_subsystem_error_prefix = ( + f"Failure deleting subsystem {request.subsystem_nqn}" + ) ret = False omap_lock = self.omap_lock.get_omap_lock_to_use(context) @@ -1089,7 +1317,9 @@ def delete_subsystem_safe(self, request, context): if request.subsystem_nqn in self.subsystem_listeners: self.subsystem_listeners.pop(request.subsystem_nqn, None) self.host_info.clean_subsystem(request.subsystem_nqn) - self.subsystem_nsid_bdev_and_uuid.remove_namespace(request.subsystem_nqn) + self.subsystem_nsid_bdev_and_uuid.remove_namespace( + request.subsystem_nqn + ) self.remove_all_subsystem_key_files(request.subsystem_nqn) self.remove_all_subsystem_keys_from_keyring(request.subsystem_nqn) self.logger.debug(f"delete_subsystem {request.subsystem_nqn}: {ret}") @@ -1107,8 +1337,10 @@ def delete_subsystem_safe(self, request, context): # Just in case SPDK failed with no exception if not ret: self.logger.error(delete_subsystem_error_prefix) - self.remove_subsystem_from_state( request.subsystem_nqn, context) - return pb2.req_status(status=errno.EINVAL, error_message=delete_subsystem_error_prefix) + self.remove_subsystem_from_state(request.subsystem_nqn, context) + return pb2.req_status( + status=errno.EINVAL, error_message=delete_subsystem_error_prefix + ) return self.remove_subsystem_from_state(request.subsystem_nqn, context) @@ -1116,30 +1348,38 @@ def delete_subsystem(self, request, context=None): """Deletes a subsystem.""" peer_msg = self.get_peer_message(context) - delete_subsystem_error_prefix = f"Failure deleting subsystem {request.subsystem_nqn}" - self.logger.info(f"Received request to delete subsystem {request.subsystem_nqn}, context: {context}{peer_msg}") + delete_subsystem_error_prefix = ( + f"Failure deleting subsystem {request.subsystem_nqn}" + ) + self.logger.info( + f"Received request to delete subsystem {request.subsystem_nqn}, context: {context}{peer_msg}" + ) if not request.subsystem_nqn: errmsg = f"Failure deleting subsystem, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if self.verify_nqns: rc = GatewayUtils.is_valid_nqn(request.subsystem_nqn) if rc[0] != 0: errmsg = f"{delete_subsystem_error_prefix}: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc[0], error_message = errmsg) + return pb2.req_status(status=rc[0], error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.subsystem_nqn): - errmsg = f"{delete_subsystem_error_prefix}: Can't delete a discovery subsystem" + errmsg = ( + f"{delete_subsystem_error_prefix}: Can't delete a discovery subsystem" + ) self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) ns_list = [] if context: if self.subsystem_has_listeners(request.subsystem_nqn): - self.logger.warning(f"About to delete subsystem {request.subsystem_nqn} which has a listener defined") + self.logger.warning( + f"About to delete subsystem {request.subsystem_nqn} which has a listener defined" + ) ns_list = self.get_subsystem_namespaces(request.subsystem_nqn) # We found a namespace still using this subsystem and --force wasn't used fail with EBUSY @@ -1150,13 +1390,26 @@ def delete_subsystem(self, request, context=None): for nsid in ns_list: # We found a namespace still using this subsystem and --force was used so we will try to remove the namespace - self.logger.warning(f"Will remove namespace {nsid} from {request.subsystem_nqn}") - ret = self.namespace_delete(pb2.namespace_delete_req(subsystem_nqn=request.subsystem_nqn, nsid=nsid), context) + self.logger.warning( + f"Will remove namespace {nsid} from {request.subsystem_nqn}" + ) + ret = self.namespace_delete( + pb2.namespace_delete_req( + subsystem_nqn=request.subsystem_nqn, nsid=nsid + ), + context, + ) if ret.status == 0: - self.logger.info(f"Automatically removed namespace {nsid} from {request.subsystem_nqn}") + self.logger.info( + f"Automatically removed namespace {nsid} from {request.subsystem_nqn}" + ) else: - self.logger.error(f"Failure removing namespace {nsid} from {request.subsystem_nqn}:\n{ret.error_message}") - self.logger.warning(f"Will continue deleting {request.subsystem_nqn} anyway") + self.logger.error( + f"Failure removing namespace {nsid} from {request.subsystem_nqn}:\n{ret.error_message}" + ) + self.logger.warning( + f"Will continue deleting {request.subsystem_nqn} anyway" + ) return self.execute_grpc_function(self.delete_subsystem_safe, request, context) def check_if_image_used(self, pool_name, image_name): @@ -1172,20 +1425,31 @@ def check_if_image_used(self, pool_name, image_name): ns = json.loads(val) ns_pool = ns["rbd_pool_name"] ns_image = ns["rbd_image_name"] - if pool_name and pool_name == ns_pool and image_name and image_name == ns_image: + if ( + pool_name + and pool_name == ns_pool + and image_name + and image_name == ns_image + ): nqn = ns["subsystem_nqn"] errmsg = f"RBD image {ns_pool}/{ns_image} is already used by a namespace in subsystem {nqn}" break except Exception: - self.logger.exception(f"Got exception while parsing {val}, will continue") + self.logger.exception( + f"Got exception while parsing {val}, will continue" + ) continue return errmsg, nqn - def create_namespace(self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_auto_visible, context): + def create_namespace( + self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_auto_visible, context + ): """Adds a namespace to a subsystem.""" - + if context: - assert self.omap_lock.locked(), "OMAP is unlocked when calling create_namespace()" + assert ( + self.omap_lock.locked() + ), "OMAP is unlocked when calling create_namespace()" nsid_msg = "" if nsid: @@ -1194,12 +1458,16 @@ def create_namespace(self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_au if not subsystem_nqn: errmsg = f"Failure adding namespace, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.nsid_status(status=errno.EINVAL, error_message = errmsg) + return pb2.nsid_status(status=errno.EINVAL, error_message=errmsg) - add_namespace_error_prefix = f"Failure adding namespace{nsid_msg} to {subsystem_nqn}" + add_namespace_error_prefix = ( + f"Failure adding namespace{nsid_msg} to {subsystem_nqn}" + ) peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to add {bdev_name} to {subsystem_nqn} with ANA group id {anagrpid}{nsid_msg}, no_auto_visible: {no_auto_visible}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to add {bdev_name} to {subsystem_nqn} with ANA group id {anagrpid}{nsid_msg}, no_auto_visible: {no_auto_visible}, context: {context}{peer_msg}" + ) if anagrpid > self.subsys_max_ns[subsystem_nqn]: errmsg = f"{add_namespace_error_prefix}: Group ID {anagrpid} is bigger than configured maximum {self.subsys_max_ns[subsystem_nqn]}" @@ -1211,8 +1479,13 @@ def create_namespace(self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_au self.logger.error(errmsg) return pb2.nsid_status(status=errno.EINVAL, error_message=errmsg) - if no_auto_visible and self.subsystem_nsid_bdev_and_uuid.get_namespace_count(subsystem_nqn, - True, 0) >= self.max_namespaces_with_netmask: + if ( + no_auto_visible + and self.subsystem_nsid_bdev_and_uuid.get_namespace_count( + subsystem_nqn, True, 0 + ) + >= self.max_namespaces_with_netmask + ): errmsg = f"Failure adding namespace{nsid_msg} to {subsystem_nqn}: Maximal number of namespaces which are not auto visible ({self.max_namespaces_with_netmask}) has already been reached" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.E2BIG, error_message=errmsg) @@ -1222,13 +1495,21 @@ def create_namespace(self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_au self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.E2BIG, error_message=errmsg) - if not nsid and self.subsystem_nsid_bdev_and_uuid.get_namespace_count(subsystem_nqn, - None, 0) >= self.subsys_max_ns[subsystem_nqn]: + if ( + not nsid + and self.subsystem_nsid_bdev_and_uuid.get_namespace_count( + subsystem_nqn, None, 0 + ) + >= self.subsys_max_ns[subsystem_nqn] + ): errmsg = f"Failure adding namespace to {subsystem_nqn}: Subsystem's maximal number of namespaces ({self.subsys_max_ns[subsystem_nqn]}) has already been reached" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.E2BIG, error_message=errmsg) - if self.subsystem_nsid_bdev_and_uuid.get_namespace_count(None, None, 0) >= self.max_namespaces: + if ( + self.subsystem_nsid_bdev_and_uuid.get_namespace_count(None, None, 0) + >= self.max_namespaces + ): errmsg = f"Failure adding namespace to {subsystem_nqn}: Maximal number of namespaces ({self.max_namespaces}) has already been reached" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.E2BIG, error_message=errmsg) @@ -1243,7 +1524,9 @@ def create_namespace(self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_au uuid=uuid, no_auto_visible=no_auto_visible, ) - self.subsystem_nsid_bdev_and_uuid.add_namespace(subsystem_nqn, nsid, bdev_name, uuid, anagrpid, no_auto_visible) + self.subsystem_nsid_bdev_and_uuid.add_namespace( + subsystem_nqn, nsid, bdev_name, uuid, anagrpid, no_auto_visible + ) self.logger.debug(f"subsystem_add_ns: {nsid}") except Exception as ex: self.logger.exception(add_namespace_error_prefix) @@ -1259,7 +1542,9 @@ def create_namespace(self, subsystem_nqn, bdev_name, nsid, anagrpid, uuid, no_au # Just in case SPDK failed with no exception if not nsid: self.logger.error(add_namespace_error_prefix) - return pb2.nsid_status(status=errno.EINVAL, error_message=add_namespace_error_prefix) + return pb2.nsid_status( + status=errno.EINVAL, error_message=add_namespace_error_prefix + ) return pb2.nsid_status(nsid=nsid, status=0, error_message=os.strerror(0)) @@ -1273,7 +1558,9 @@ def set_ana_state(self, request, context=None): def set_ana_state_safe(self, ana_info: pb2.ana_info, context=None): peer_msg = self.get_peer_message(context) """Sets ana state for this gateway.""" - self.logger.info(f"Received request to set ana states {ana_info.states}, {peer_msg}") + self.logger.info( + f"Received request to set ana states {ana_info.states}, {peer_msg}" + ) state = self.gateway_state.local.get_state() inaccessible_ana_groups = {} @@ -1284,7 +1571,7 @@ def set_ana_state_safe(self, ana_info: pb2.ana_info, context=None): # fill the static gateway dictionary per nqn and grp_id nqn = nas.nqn for gs in nas.states: - self.ana_map[nqn][gs.grp_id] = gs.state + self.ana_map[nqn][gs.grp_id] = gs.state # If this is not set the subsystem was not created yet if not nqn in self.subsys_max_ns: @@ -1300,29 +1587,47 @@ def set_ana_state_safe(self, ana_info: pb2.ana_info, context=None): grp_id = gs.grp_id # The gateway's interface gRPC ana_state into SPDK JSON RPC values, # see nvmf_subsystem_listener_set_ana_state method https://spdk.io/doc/jsonrpc.html - ana_state = "optimized" if gs.state == pb2.ana_state.OPTIMIZED else "inaccessible" + ana_state = ( + "optimized" + if gs.state == pb2.ana_state.OPTIMIZED + else "inaccessible" + ) try: # Need to wait for the latest OSD map, for each RADOS # cluster context before becoming optimized, # part of blocklist logic if gs.state == pb2.ana_state.OPTIMIZED: # Go over the namespaces belonging to the ana group - for ns_info in self.subsystem_nsid_bdev_and_uuid.get_namespace_infos_for_anagrpid(nqn, grp_id): + for ( + ns_info + ) in self.subsystem_nsid_bdev_and_uuid.get_namespace_infos_for_anagrpid( + nqn, grp_id + ): # get the cluster name for this namespace with self.shared_state_lock: cluster = self.bdev_cluster[ns_info.bdev] if not cluster: - raise Exception(f"can not find cluster context name for bdev {ns_info.bdev}") + raise Exception( + f"can not find cluster context name for bdev {ns_info.bdev}" + ) if cluster in awaited_cluster_contexts: # this cluster context was already awaited continue - if not rpc_bdev.bdev_rbd_wait_for_latest_osdmap(self.spdk_rpc_client, name=cluster): - raise Exception(f"bdev_rbd_wait_for_latest_osdmap({cluster=}) error") - self.logger.debug(f"set_ana_state bdev_rbd_wait_for_latest_osdmap {cluster=}") + if not rpc_bdev.bdev_rbd_wait_for_latest_osdmap( + self.spdk_rpc_client, name=cluster + ): + raise Exception( + f"bdev_rbd_wait_for_latest_osdmap({cluster=}) error" + ) + self.logger.debug( + f"set_ana_state bdev_rbd_wait_for_latest_osdmap {cluster=}" + ) awaited_cluster_contexts.add(cluster) - self.logger.debug(f"set_ana_state nvmf_subsystem_listener_set_ana_state {nqn=} {listener=} {ana_state=} {grp_id=}") + self.logger.debug( + f"set_ana_state nvmf_subsystem_listener_set_ana_state {nqn=} {listener=} {ana_state=} {grp_id=}" + ) (adrfam, traddr, trsvcid, secure) = listener ret = rpc_nvmf.nvmf_subsystem_listener_set_ana_state( self.spdk_rpc_client, @@ -1332,12 +1637,17 @@ def set_ana_state_safe(self, ana_info: pb2.ana_info, context=None): trsvcid=str(trsvcid), adrfam=adrfam, ana_state=ana_state, - anagrpid=grp_id) - if ana_state == "inaccessible" : + anagrpid=grp_id, + ) + if ana_state == "inaccessible": inaccessible_ana_groups[grp_id] = True - self.logger.debug(f"set_ana_state nvmf_subsystem_listener_set_ana_state response {ret=}") + self.logger.debug( + f"set_ana_state nvmf_subsystem_listener_set_ana_state response {ret=}" + ) if not ret: - raise Exception(f"nvmf_subsystem_listener_set_ana_state({nqn=}, {listener=}, {ana_state=}, {grp_id=}) error") + raise Exception( + f"nvmf_subsystem_listener_set_ana_state({nqn=}, {listener=}, {ana_state=}, {grp_id=}) error" + ) except Exception as ex: self.logger.exception("nvmf_subsystem_listener_set_ana_state()") if context: @@ -1346,28 +1656,40 @@ def set_ana_state_safe(self, ana_info: pb2.ana_info, context=None): return pb2.req_status() return pb2.req_status(status=True) - def choose_anagrpid_for_namespace(self, nsid) ->int: - grps_list = self.ceph_utils.get_number_created_gateways(self.gateway_pool, self.gateway_group) + def choose_anagrpid_for_namespace(self, nsid) -> int: + grps_list = self.ceph_utils.get_number_created_gateways( + self.gateway_pool, self.gateway_group + ) for ana_grp in grps_list: - if not self.clusters[ana_grp]: # still no namespaces in this ana-group - probably the new GW added - self.logger.info(f"New GW created: chosen ana group {ana_grp} for ns {nsid} ") + if not self.clusters[ + ana_grp + ]: # still no namespaces in this ana-group - probably the new GW added + self.logger.info( + f"New GW created: chosen ana group {ana_grp} for ns {nsid} " + ) return ana_grp - #not found ana_grp .To calulate it. Find minimum loaded ana_grp cluster + # not found ana_grp .To calulate it. Find minimum loaded ana_grp cluster ana_load = {} min_load = 2000 chosen_ana_group = 0 for ana_grp in self.clusters: - if ana_grp in grps_list: #to take into consideration only valid groups - ana_load[ana_grp] = 0; + if ana_grp in grps_list: # to take into consideration only valid groups + ana_load[ana_grp] = 0 for name in self.clusters[ana_grp]: - ana_load[ana_grp] += self.clusters[ana_grp][name] # accumulate the total load per ana group for all valid ana_grp clusters - for ana_grp in ana_load : + ana_load[ana_grp] += self.clusters[ana_grp][ + name + ] # accumulate the total load per ana group for all valid ana_grp clusters + for ana_grp in ana_load: self.logger.info(f" ana group {ana_grp} load = {ana_load[ana_grp]} ") - if ana_load[ana_grp] <= min_load: + if ana_load[ana_grp] <= min_load: min_load = ana_load[ana_grp] chosen_ana_group = ana_grp - self.logger.info(f" ana group {ana_grp} load = {ana_load[ana_grp]} set as min {min_load} ") - self.logger.info(f"Found min loaded cluster: chosen ana group {chosen_ana_group} for ns {nsid} ") + self.logger.info( + f" ana group {ana_grp} load = {ana_load[ana_grp]} set as min {min_load} " + ) + self.logger.info( + f"Found min loaded cluster: chosen ana group {chosen_ana_group} for ns {nsid} " + ) return chosen_ana_group def namespace_add_safe(self, request, context): @@ -1376,7 +1698,7 @@ def namespace_add_safe(self, request, context): if not request.subsystem_nqn: errmsg = f"Failure adding namespace, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.nsid_status(status=errno.EINVAL, error_message = errmsg) + return pb2.nsid_status(status=errno.EINVAL, error_message=errmsg) grps_list = [] anagrp = 0 @@ -1384,49 +1706,65 @@ def namespace_add_safe(self, request, context): nsid_msg = "" if request.nsid: nsid_msg = f"{request.nsid} " - self.logger.info(f"Received request to add namespace {nsid_msg}to {request.subsystem_nqn}, ana group {request.anagrpid}, no_auto_visible: {request.no_auto_visible}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to add namespace {nsid_msg}to {request.subsystem_nqn}, ana group {request.anagrpid}, no_auto_visible: {request.no_auto_visible}, context: {context}{peer_msg}" + ) if not request.uuid: request.uuid = str(uuid.uuid4()) if context: if request.anagrpid != 0: - grps_list = self.ceph_utils.get_number_created_gateways(self.gateway_pool, self.gateway_group) + grps_list = self.ceph_utils.get_number_created_gateways( + self.gateway_pool, self.gateway_group + ) else: anagrp = self.choose_anagrpid_for_namespace(request.nsid) assert anagrp != 0, "Chosen ANA group is 0" if request.nsid: - ns = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) + ns = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, request.nsid + ) if not ns.empty(): errmsg = f"Failure adding namespace, NSID {request.nsid} is already in use" self.logger.error(f"{errmsg}") - return pb2.nsid_status(status=errno.EEXIST, error_message = errmsg) + return pb2.nsid_status(status=errno.EEXIST, error_message=errmsg) - ns = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, None, request.uuid) + ns = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, None, request.uuid + ) if not ns.empty(): - errmsg = f"Failure adding namespace, UUID {request.uuid} is already in use" - self.logger.error(f"{errmsg}") - return pb2.nsid_status(status=errno.EEXIST, error_message = errmsg) + errmsg = ( + f"Failure adding namespace, UUID {request.uuid} is already in use" + ) + self.logger.error(f"{errmsg}") + return pb2.nsid_status(status=errno.EEXIST, error_message=errmsg) omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: if context: - errmsg, ns_nqn = self.check_if_image_used(request.rbd_pool_name, request.rbd_image_name) + errmsg, ns_nqn = self.check_if_image_used( + request.rbd_pool_name, request.rbd_image_name + ) if errmsg and ns_nqn: if request.force: - self.logger.warning(f"{errmsg}, will continue as the \"force\" argument was used") + self.logger.warning( + f'{errmsg}, will continue as the "force" argument was used' + ) else: - errmsg = f"{errmsg}, either delete the namespace or use the \"force\" argument,\nyou can find the offending namespace by using the \"namespace list --subsystem {ns_nqn}\" CLI command" + errmsg = f'{errmsg}, either delete the namespace or use the "force" argument,\nyou can find the offending namespace by using the "namespace list --subsystem {ns_nqn}" CLI command' self.logger.error(errmsg) - return pb2.nsid_status(status=errno.EEXIST, error_message=errmsg) + return pb2.nsid_status( + status=errno.EEXIST, error_message=errmsg + ) bdev_name = GatewayService.find_unique_bdev_name(request.uuid) create_image = request.create_image if not context: create_image = False - else: # new namespace + else: # new namespace # If an explicit load balancing group was passed, make sure it exists if request.anagrpid != 0: if request.anagrpid not in grps_list: @@ -1435,11 +1773,21 @@ def namespace_add_safe(self, request, context): self.logger.error(errmsg) return pb2.req_status(status=errno.ENODEV, error_message=errmsg) else: - request.anagrpid = anagrp + request.anagrpid = anagrp anagrp = request.anagrpid - ret_bdev = self.create_bdev(anagrp, bdev_name, request.uuid, request.rbd_pool_name, - request.rbd_image_name, request.block_size, create_image, request.size, context, peer_msg) + ret_bdev = self.create_bdev( + anagrp, + bdev_name, + request.uuid, + request.rbd_pool_name, + request.rbd_image_name, + request.block_size, + create_image, + request.size, + context, + peer_msg, + ) if ret_bdev.status != 0: errmsg = f"Failure adding namespace {nsid_msg}to {request.subsystem_nqn}: {ret_bdev.error_message}" self.logger.error(errmsg) @@ -1448,18 +1796,32 @@ def namespace_add_safe(self, request, context): ns_bdev = self.get_bdev_info(bdev_name) if ns_bdev != None: try: - ret_del = self.delete_bdev(bdev_name, peer_msg = peer_msg) - self.logger.debug(f"delete_bdev({bdev_name}): {ret_del.status}") + ret_del = self.delete_bdev(bdev_name, peer_msg=peer_msg) + self.logger.debug( + f"delete_bdev({bdev_name}): {ret_del.status}" + ) except AssertionError: - self.logger.exception(f"Got an assert while trying to delete bdev {bdev_name}") + self.logger.exception( + f"Got an assert while trying to delete bdev {bdev_name}" + ) raise except Exception: - self.logger.exception(f"Got exception while trying to delete bdev {bdev_name}") + self.logger.exception( + f"Got exception while trying to delete bdev {bdev_name}" + ) return pb2.nsid_status(status=ret_bdev.status, error_message=errmsg) # If we got here we asserted that ret_bdev.bdev_name == bdev_name - ret_ns = self.create_namespace(request.subsystem_nqn, bdev_name, request.nsid, anagrp, request.uuid, request.no_auto_visible, context) + ret_ns = self.create_namespace( + request.subsystem_nqn, + bdev_name, + request.nsid, + anagrp, + request.uuid, + request.no_auto_visible, + context, + ) if ret_ns.status == 0 and request.nsid and ret_ns.nsid != request.nsid: errmsg = f"Returned NSID {ret_ns.nsid} differs from requested one {request.nsid}" self.logger.error(errmsg) @@ -1468,14 +1830,20 @@ def namespace_add_safe(self, request, context): if ret_ns.status != 0: try: - ret_del = self.delete_bdev(bdev_name, peer_msg = peer_msg) + ret_del = self.delete_bdev(bdev_name, peer_msg=peer_msg) if ret_del.status != 0: - self.logger.warning(f"Failure {ret_del.status} deleting bdev {bdev_name}: {ret_del.error_message}") + self.logger.warning( + f"Failure {ret_del.status} deleting bdev {bdev_name}: {ret_del.error_message}" + ) except AssertionError: - self.logger.exception(f"Got an assert while trying to delete bdev {bdev_name}") + self.logger.exception( + f"Got an assert while trying to delete bdev {bdev_name}" + ) raise except Exception: - self.logger.exception(f"Got exception while trying to delete bdev {bdev_name}") + self.logger.exception( + f"Got exception while trying to delete bdev {bdev_name}" + ) errmsg = f"Failure adding namespace {nsid_msg}to {request.subsystem_nqn}: {ret_ns.error_message}" self.logger.error(errmsg) return pb2.nsid_status(status=ret_ns.status, error_message=errmsg) @@ -1485,8 +1853,13 @@ def namespace_add_safe(self, request, context): request.nsid = ret_ns.nsid try: json_req = json_format.MessageToJson( - request, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_namespace(request.subsystem_nqn, ret_ns.nsid, json_req) + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + self.gateway_state.add_namespace( + request.subsystem_nqn, ret_ns.nsid, json_req + ) except Exception as ex: errmsg = f"Error persisting namespace {nsid_msg}on {request.subsystem_nqn}" self.logger.exception(errmsg) @@ -1505,7 +1878,9 @@ def namespace_change_load_balancing_group_safe(self, request, context): grps_list = [] peer_msg = self.get_peer_message(context) change_lb_group_failure_prefix = f"Failure changing load balancing group for namespace with NSID {request.nsid} in {request.subsystem_nqn}" - self.logger.info(f"Received request to change load balancing group for namespace with NSID {request.nsid} in {request.subsystem_nqn} to {request.anagrpid}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to change load balancing group for namespace with NSID {request.nsid} in {request.subsystem_nqn} to {request.anagrpid}, context: {context}{peer_msg}" + ) if not request.subsystem_nqn: errmsg = f"Failure changing load balancing group for namespace, missing subsystem NQN" @@ -1517,14 +1892,18 @@ def namespace_change_load_balancing_group_safe(self, request, context): self.logger.error(errmsg) return pb2.req_status(status=errno.ENODEV, error_message=errmsg) - grps_list = self.ceph_utils.get_number_created_gateways(self.gateway_pool, self.gateway_group) + grps_list = self.ceph_utils.get_number_created_gateways( + self.gateway_pool, self.gateway_group + ) if request.anagrpid not in grps_list: self.logger.debug(f"ANA groups: {grps_list}") errmsg = f"{change_lb_group_failure_prefix}: Load balancing group {request.anagrpid} doesn't exist" self.logger.error(errmsg) return pb2.req_status(status=errno.ENODEV, error_message=errmsg) - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, request.nsid + ) omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: @@ -1533,7 +1912,9 @@ def namespace_change_load_balancing_group_safe(self, request, context): # notice that the local state might not be up to date in case we're in the middle of update() but as the # context is not None, we are not in an update(), the omap lock made sure that we got here with an updated local state state = self.gateway_state.local.get_state() - ns_key = GatewayState.build_namespace_key(request.subsystem_nqn, request.nsid) + ns_key = GatewayState.build_namespace_key( + request.subsystem_nqn, request.nsid + ) try: state_ns = state[ns_key] ns_entry = json.loads(state_ns) @@ -1542,8 +1923,12 @@ def namespace_change_load_balancing_group_safe(self, request, context): self.logger.error(errmsg) return pb2.req_status(status=errno.ENOENT, error_message=errmsg) anagrp = ns_entry["anagrpid"] - gw_id = self.ceph_utils.get_gw_id_owner_ana_group(self.gateway_pool, self.gateway_group, anagrp) - self.logger.debug(f"ANA group of ns#{request.nsid} - {anagrp} is owned by gateway {gw_id}, self.name is {self.gateway_name}") + gw_id = self.ceph_utils.get_gw_id_owner_ana_group( + self.gateway_pool, self.gateway_group, anagrp + ) + self.logger.debug( + f"ANA group of ns#{request.nsid} - {anagrp} is owned by gateway {gw_id}, self.name is {self.gateway_name}" + ) if self.gateway_name != gw_id: errmsg = f"ANA group of ns#{request.nsid} - {anagrp} is owned by gateway {gw_id} so try this command from it, this gateway name is {self.gateway_name}" self.logger.error(errmsg) @@ -1571,26 +1956,35 @@ def namespace_change_load_balancing_group_safe(self, request, context): # Just in case SPDK failed with no exception if not ret: self.logger.error(change_lb_group_failure_prefix) - return pb2.req_status(status=errno.EINVAL, error_message=change_lb_group_failure_prefix) + return pb2.req_status( + status=errno.EINVAL, error_message=change_lb_group_failure_prefix + ) if context: assert ns_entry, "Namespace entry is None for non-update call" # Update gateway state try: - add_req = pb2.namespace_add_req(rbd_pool_name=ns_entry["rbd_pool_name"], - rbd_image_name=ns_entry["rbd_image_name"], - subsystem_nqn=ns_entry["subsystem_nqn"], - nsid=ns_entry["nsid"], - block_size=ns_entry["block_size"], - uuid=ns_entry["uuid"], - anagrpid=request.anagrpid, - create_image=ns_entry["create_image"], - size=int(ns_entry["size"]), - force=ns_entry["force"], - no_auto_visible=ns_entry["no_auto_visible"]) + add_req = pb2.namespace_add_req( + rbd_pool_name=ns_entry["rbd_pool_name"], + rbd_image_name=ns_entry["rbd_image_name"], + subsystem_nqn=ns_entry["subsystem_nqn"], + nsid=ns_entry["nsid"], + block_size=ns_entry["block_size"], + uuid=ns_entry["uuid"], + anagrpid=request.anagrpid, + create_image=ns_entry["create_image"], + size=int(ns_entry["size"]), + force=ns_entry["force"], + no_auto_visible=ns_entry["no_auto_visible"], + ) json_req = json_format.MessageToJson( - add_req, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_namespace(request.subsystem_nqn, request.nsid, json_req) + add_req, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + self.gateway_state.add_namespace( + request.subsystem_nqn, request.nsid, json_req + ) except Exception as ex: errmsg = f"Error persisting namespace load balancing group for namespace with NSID {request.nsid} in {request.subsystem_nqn}" self.logger.exception(errmsg) @@ -1601,14 +1995,18 @@ def namespace_change_load_balancing_group_safe(self, request, context): def namespace_change_load_balancing_group(self, request, context=None): """Changes a namespace load balancing group.""" - return self.execute_grpc_function(self.namespace_change_load_balancing_group_safe, request, context) + return self.execute_grpc_function( + self.namespace_change_load_balancing_group_safe, request, context + ) def remove_namespace_from_state(self, nqn, nsid, context): if not context: return pb2.req_status(status=0, error_message=os.strerror(0)) # If we got here context is not None, so we must hold the OMAP lock - assert self.omap_lock.locked(), "OMAP is unlocked when calling remove_namespace_from_state()" + assert ( + self.omap_lock.locked() + ), "OMAP is unlocked when calling remove_namespace_from_state()" # Update gateway state try: @@ -1638,13 +2036,19 @@ def remove_namespace(self, subsystem_nqn, nsid, context): """Removes a namespace from a subsystem.""" if context: - assert self.omap_lock.locked(), "OMAP is unlocked when calling remove_namespace()" + assert ( + self.omap_lock.locked() + ), "OMAP is unlocked when calling remove_namespace()" peer_msg = self.get_peer_message(context) - namespace_failure_prefix = f"Failure removing namespace {nsid} from {subsystem_nqn}" - self.logger.info(f"Received request to remove namespace {nsid} from {subsystem_nqn}{peer_msg}") + namespace_failure_prefix = ( + f"Failure removing namespace {nsid} from {subsystem_nqn}" + ) + self.logger.info( + f"Received request to remove namespace {nsid} from {subsystem_nqn}{peer_msg}" + ) if GatewayUtils.is_discovery_nqn(subsystem_nqn): - errmsg=f"{namespace_failure_prefix}: Can't remove a namespace from a discovery subsystem" + errmsg = f"{namespace_failure_prefix}: Can't remove a namespace from a discovery subsystem" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -1668,7 +2072,9 @@ def remove_namespace(self, subsystem_nqn, nsid, context): # Just in case SPDK failed with no exception if not ret: self.logger.error(namespace_failure_prefix) - return pb2.req_status(status=errno.EINVAL, error_message=namespace_failure_prefix) + return pb2.req_status( + status=errno.EINVAL, error_message=namespace_failure_prefix + ) return pb2.req_status(status=0, error_message=os.strerror(0)) @@ -1679,8 +2085,10 @@ def get_bdev_info(self, bdev_name): ret_bdev = None try: bdevs = rpc_bdev.bdev_get_bdevs(self.spdk_rpc_client, name=bdev_name) - if (len(bdevs) > 1): - self.logger.warning(f"Got {len(bdevs)} bdevs for bdev name {bdev_name}, will use the first one") + if len(bdevs) > 1: + self.logger.warning( + f"Got {len(bdevs)} bdevs for bdev name {bdev_name}, will use the first one" + ) ret_bdev = bdevs[0] except Exception: self.logger.exception(f"Got exception while getting bdev {bdev_name} info") @@ -1701,16 +2109,25 @@ def list_namespaces(self, request, context=None): nsid_msg = f"namespace with NSID {request.nsid} and UUID {request.uuid}" else: nsid_msg = f"namespace with NSID {request.nsid}" - self.logger.info(f"Received request to list {nsid_msg} for {request.subsystem}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to list {nsid_msg} for {request.subsystem}, context: {context}{peer_msg}" + ) if not request.subsystem: errmsg = f"Failure listing namespaces, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.namespaces_info(status=errno.EINVAL, error_message=errmsg, subsystem_nqn=request.subsystem, namespaces=[]) + return pb2.namespaces_info( + status=errno.EINVAL, + error_message=errmsg, + subsystem_nqn=request.subsystem, + namespaces=[], + ) with self.rpc_lock: try: - ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem) + ret = rpc_nvmf.nvmf_get_subsystems( + self.spdk_rpc_client, nqn=request.subsystem + ) self.logger.debug(f"list_namespaces: {ret}") except Exception as ex: errmsg = f"Failure listing namespaces" @@ -1721,13 +2138,20 @@ def list_namespaces(self, request, context=None): if resp: status = resp["code"] errmsg = f"Failure listing namespaces: {resp['message']}" - return pb2.namespaces_info(status=status, error_message=errmsg, subsystem_nqn=request.subsystem, namespaces=[]) + return pb2.namespaces_info( + status=status, + error_message=errmsg, + subsystem_nqn=request.subsystem, + namespaces=[], + ) namespaces = [] for s in ret: try: if s["nqn"] != request.subsystem: - self.logger.warning(f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore') + self.logger.warning( + f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore' + ) continue try: ns_list = s["namespaces"] @@ -1735,35 +2159,49 @@ def list_namespaces(self, request, context=None): ns_list = [] pass if not ns_list: - self.subsystem_nsid_bdev_and_uuid.remove_namespace(request.subsystem) + self.subsystem_nsid_bdev_and_uuid.remove_namespace( + request.subsystem + ) for n in ns_list: nsid = n["nsid"] bdev_name = n["bdev_name"] if request.nsid and request.nsid != n["nsid"]: - self.logger.debug(f'Filter out namespace {n["nsid"]} which is different than requested nsid {request.nsid}') + self.logger.debug( + f'Filter out namespace {n["nsid"]} which is different than requested nsid {request.nsid}' + ) continue if request.uuid and request.uuid != n["uuid"]: - self.logger.debug(f'Filter out namespace with UUID {n["uuid"]} which is different than requested UUID {request.uuid}') + self.logger.debug( + f'Filter out namespace with UUID {n["uuid"]} which is different than requested UUID {request.uuid}' + ) continue lb_group = 0 try: lb_group = n["anagrpid"] except KeyError: pass - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem, nsid) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem, nsid + ) if find_ret.empty(): - self.logger.warning(f"Can't find info of namesapce {nsid} in {request.subsystem}. Visibility status will be inaccurate") + self.logger.warning( + f"Can't find info of namesapce {nsid} in {request.subsystem}. Visibility status will be inaccurate" + ) no_auto_visible = find_ret.no_auto_visible - one_ns = pb2.namespace_cli(nsid = nsid, - bdev_name = bdev_name, - uuid = n["uuid"], - load_balancing_group = lb_group, - no_auto_visible = no_auto_visible, - hosts = find_ret.host_list) + one_ns = pb2.namespace_cli( + nsid=nsid, + bdev_name=bdev_name, + uuid=n["uuid"], + load_balancing_group=lb_group, + no_auto_visible=no_auto_visible, + hosts=find_ret.host_list, + ) with self.rpc_lock: ns_bdev = self.get_bdev_info(bdev_name) if ns_bdev == None: - self.logger.warning(f"Can't find namespace's bdev {bdev_name}, will not list bdev's information") + self.logger.warning( + f"Can't find namespace's bdev {bdev_name}, will not list bdev's information" + ) else: try: drv_specific_info = ns_bdev["driver_specific"] @@ -1771,17 +2209,27 @@ def list_namespaces(self, request, context=None): one_ns.rbd_image_name = rbd_info["rbd_name"] one_ns.rbd_pool_name = rbd_info["pool_name"] one_ns.block_size = ns_bdev["block_size"] - one_ns.rbd_image_size = ns_bdev["block_size"] * ns_bdev["num_blocks"] + one_ns.rbd_image_size = ( + ns_bdev["block_size"] * ns_bdev["num_blocks"] + ) assigned_limits = ns_bdev["assigned_rate_limits"] - one_ns.rw_ios_per_second=assigned_limits["rw_ios_per_sec"] - one_ns.rw_mbytes_per_second=assigned_limits["rw_mbytes_per_sec"] - one_ns.r_mbytes_per_second=assigned_limits["r_mbytes_per_sec"] - one_ns.w_mbytes_per_second=assigned_limits["w_mbytes_per_sec"] + one_ns.rw_ios_per_second = assigned_limits["rw_ios_per_sec"] + one_ns.rw_mbytes_per_second = assigned_limits[ + "rw_mbytes_per_sec" + ] + one_ns.r_mbytes_per_second = assigned_limits[ + "r_mbytes_per_sec" + ] + one_ns.w_mbytes_per_second = assigned_limits[ + "w_mbytes_per_sec" + ] except KeyError as err: - self.logger.warning(f"Key {err} is not found, will not list bdev's information") + self.logger.warning( + f"Key {err} is not found, will not list bdev's information" + ) pass except Exception: - self.logger.exception(f"{ns_bdev=} parse error") + self.logger.exception(f"{ns_bdev=} parse error") pass namespaces.append(one_ns) break @@ -1789,35 +2237,52 @@ def list_namespaces(self, request, context=None): self.logger.exception(f"{s=} parse error") pass - return pb2.namespaces_info(status = 0, error_message = os.strerror(0), subsystem_nqn=request.subsystem, namespaces=namespaces) + return pb2.namespaces_info( + status=0, + error_message=os.strerror(0), + subsystem_nqn=request.subsystem, + namespaces=namespaces, + ) def namespace_get_io_stats(self, request, context=None): """Get namespace's IO stats.""" peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to get IO stats for namespace {request.nsid} on {request.subsystem_nqn}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to get IO stats for namespace {request.nsid} on {request.subsystem_nqn}, context: {context}{peer_msg}" + ) if not request.nsid: errmsg = f"Failure getting IO stats for namespace, missing NSID" self.logger.error(f"{errmsg}") - return pb2.namespace_io_stats_info(status=errno.EINVAL, error_message=errmsg) + return pb2.namespace_io_stats_info( + status=errno.EINVAL, error_message=errmsg + ) if not request.subsystem_nqn: errmsg = f"Failure getting IO stats for namespace {request.nsid}, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.namespace_io_stats_info(status=errno.EINVAL, error_message=errmsg) + return pb2.namespace_io_stats_info( + status=errno.EINVAL, error_message=errmsg + ) with self.rpc_lock: - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, request.nsid + ) if find_ret.empty(): errmsg = f"Failure getting IO stats for namespace {request.nsid} on {request.subsystem_nqn}: Can't find namespace" self.logger.error(errmsg) - return pb2.namespace_io_stats_info(status=errno.ENODEV, error_message=errmsg) + return pb2.namespace_io_stats_info( + status=errno.ENODEV, error_message=errmsg + ) uuid = find_ret.uuid bdev_name = find_ret.bdev if not bdev_name: errmsg = f"Failure getting IO stats for namespace {request.nsid} on {request.subsystem_nqn}: Can't find associated block device" self.logger.error(errmsg) - return pb2.namespace_io_stats_info(status=errno.ENODEV, error_message=errmsg) + return pb2.namespace_io_stats_info( + status=errno.ENODEV, error_message=errmsg + ) try: ret = rpc_bdev.bdev_get_iostat( @@ -1840,71 +2305,89 @@ def namespace_get_io_stats(self, request, context=None): if not ret: errmsg = f"Failure getting IO stats for namespace {request.nsid} on {request.subsystem_nqn}" self.logger.error(errmsg) - return pb2.namespace_io_stats_info(status=errno.EINVAL, error_message=errmsg) + return pb2.namespace_io_stats_info( + status=errno.EINVAL, error_message=errmsg + ) exmsg = "" try: bdevs = ret["bdevs"] if not bdevs: - return pb2.namespace_io_stats_info(status=errno.ENODEV, - error_message=f"Failure getting IO stats for namespace {request.nsid} on {request.subsystem_nqn}: No associated block device found") + return pb2.namespace_io_stats_info( + status=errno.ENODEV, + error_message=f"Failure getting IO stats for namespace {request.nsid} on {request.subsystem_nqn}: No associated block device found", + ) if len(bdevs) > 1: - self.logger.warning(f"More than one associated block device found for namespace, will use the first one") + self.logger.warning( + f"More than one associated block device found for namespace, will use the first one" + ) bdev = bdevs[0] io_errs = [] try: - io_error=bdev["io_error"] + io_error = bdev["io_error"] for err_name in io_error.keys(): - one_error = pb2.namespace_io_error(name=err_name, value=io_error[err_name]) + one_error = pb2.namespace_io_error( + name=err_name, value=io_error[err_name] + ) io_errs.append(one_error) except Exception: self.logger.exception(f"failure getting io errors") - io_stats = pb2.namespace_io_stats_info(status=0, - error_message=os.strerror(0), - subsystem_nqn=request.subsystem_nqn, - nsid=request.nsid, - uuid=uuid, - bdev_name=bdev_name, - tick_rate=ret["tick_rate"], - ticks=ret["ticks"], - bytes_read=bdev["bytes_read"], - num_read_ops=bdev["num_read_ops"], - bytes_written=bdev["bytes_written"], - num_write_ops=bdev["num_write_ops"], - bytes_unmapped=bdev["bytes_unmapped"], - num_unmap_ops=bdev["num_unmap_ops"], - read_latency_ticks=bdev["read_latency_ticks"], - max_read_latency_ticks=bdev["max_read_latency_ticks"], - min_read_latency_ticks=bdev["min_read_latency_ticks"], - write_latency_ticks=bdev["write_latency_ticks"], - max_write_latency_ticks=bdev["max_write_latency_ticks"], - min_write_latency_ticks=bdev["min_write_latency_ticks"], - unmap_latency_ticks=bdev["unmap_latency_ticks"], - max_unmap_latency_ticks=bdev["max_unmap_latency_ticks"], - min_unmap_latency_ticks=bdev["min_unmap_latency_ticks"], - copy_latency_ticks=bdev["copy_latency_ticks"], - max_copy_latency_ticks=bdev["max_copy_latency_ticks"], - min_copy_latency_ticks=bdev["min_copy_latency_ticks"], - io_error=io_errs) + io_stats = pb2.namespace_io_stats_info( + status=0, + error_message=os.strerror(0), + subsystem_nqn=request.subsystem_nqn, + nsid=request.nsid, + uuid=uuid, + bdev_name=bdev_name, + tick_rate=ret["tick_rate"], + ticks=ret["ticks"], + bytes_read=bdev["bytes_read"], + num_read_ops=bdev["num_read_ops"], + bytes_written=bdev["bytes_written"], + num_write_ops=bdev["num_write_ops"], + bytes_unmapped=bdev["bytes_unmapped"], + num_unmap_ops=bdev["num_unmap_ops"], + read_latency_ticks=bdev["read_latency_ticks"], + max_read_latency_ticks=bdev["max_read_latency_ticks"], + min_read_latency_ticks=bdev["min_read_latency_ticks"], + write_latency_ticks=bdev["write_latency_ticks"], + max_write_latency_ticks=bdev["max_write_latency_ticks"], + min_write_latency_ticks=bdev["min_write_latency_ticks"], + unmap_latency_ticks=bdev["unmap_latency_ticks"], + max_unmap_latency_ticks=bdev["max_unmap_latency_ticks"], + min_unmap_latency_ticks=bdev["min_unmap_latency_ticks"], + copy_latency_ticks=bdev["copy_latency_ticks"], + max_copy_latency_ticks=bdev["max_copy_latency_ticks"], + min_copy_latency_ticks=bdev["min_copy_latency_ticks"], + io_error=io_errs, + ) return io_stats except Exception as ex: self.logger.exception(f"parse error") exmsg = str(ex) pass - return pb2.namespace_io_stats_info(status=errno.EINVAL, - error_message=f"Failure getting IO stats for namespace {nsid_msg}on {request.subsystem_nqn}: Error parsing returned stats:\n{exmsg}") + return pb2.namespace_io_stats_info( + status=errno.EINVAL, + error_message=f"Failure getting IO stats for namespace {nsid_msg}on {request.subsystem_nqn}: Error parsing returned stats:\n{exmsg}", + ) def get_qos_limits_string(self, request): limits_to_set = "" if request.HasField("rw_ios_per_second"): limits_to_set += f" R/W IOs per second: {request.rw_ios_per_second}" if request.HasField("rw_mbytes_per_second"): - limits_to_set += f" R/W megabytes per second: {request.rw_mbytes_per_second}" + limits_to_set += ( + f" R/W megabytes per second: {request.rw_mbytes_per_second}" + ) if request.HasField("r_mbytes_per_second"): - limits_to_set += f" Read megabytes per second: {request.r_mbytes_per_second}" + limits_to_set += ( + f" Read megabytes per second: {request.r_mbytes_per_second}" + ) if request.HasField("w_mbytes_per_second"): - limits_to_set += f" Write megabytes per second: {request.w_mbytes_per_second}" + limits_to_set += ( + f" Write megabytes per second: {request.w_mbytes_per_second}" + ) return limits_to_set @@ -1913,19 +2396,27 @@ def namespace_set_qos_limits_safe(self, request, context): peer_msg = self.get_peer_message(context) limits_to_set = self.get_qos_limits_string(request) - self.logger.info(f"Received request to set QOS limits for namespace {request.nsid} on {request.subsystem_nqn},{limits_to_set}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to set QOS limits for namespace {request.nsid} on {request.subsystem_nqn},{limits_to_set}, context: {context}{peer_msg}" + ) if not request.nsid: errmsg = f"Failure setting QOS limits for namespace, missing NSID" self.logger.error(f"{errmsg}") - return pb2.namespace_io_stats_info(status=errno.EINVAL, error_message=errmsg) + return pb2.namespace_io_stats_info( + status=errno.EINVAL, error_message=errmsg + ) if not request.subsystem_nqn: errmsg = f"Failure setting QOS limits for namespace {request.nsid}, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.namespace_io_stats_info(status=errno.EINVAL, error_message=errmsg) + return pb2.namespace_io_stats_info( + status=errno.EINVAL, error_message=errmsg + ) - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, request.nsid + ) if find_ret.empty(): errmsg = f"Failure setting QOS limits for namespace {request.nsid} on {request.subsystem_nqn}: Can't find namespace" self.logger.error(errmsg) @@ -1950,33 +2441,51 @@ def namespace_set_qos_limits_safe(self, request, context): ns_qos_entry = None if context: state = self.gateway_state.local.get_state() - ns_qos_key = GatewayState.build_namespace_qos_key(request.subsystem_nqn, request.nsid) + ns_qos_key = GatewayState.build_namespace_qos_key( + request.subsystem_nqn, request.nsid + ) try: state_ns_qos = state[ns_qos_key] ns_qos_entry = json.loads(state_ns_qos) except Exception as ex: - self.logger.info(f"No previous QOS limits found, this is the first time the limits are set for namespace {request.nsid} on {request.subsystem_nqn}") + self.logger.info( + f"No previous QOS limits found, this is the first time the limits are set for namespace {request.nsid} on {request.subsystem_nqn}" + ) # Merge current limits with previous ones, if exist if ns_qos_entry: - if not request.HasField("rw_ios_per_second") and ns_qos_entry.get("rw_ios_per_second") != None: + if ( + not request.HasField("rw_ios_per_second") + and ns_qos_entry.get("rw_ios_per_second") != None + ): request.rw_ios_per_second = int(ns_qos_entry["rw_ios_per_second"]) - if not request.HasField("rw_mbytes_per_second") and ns_qos_entry.get("rw_mbytes_per_second") != None: + if ( + not request.HasField("rw_mbytes_per_second") + and ns_qos_entry.get("rw_mbytes_per_second") != None + ): request.rw_mbytes_per_second = int(ns_qos_entry["rw_mbytes_per_second"]) - if not request.HasField("r_mbytes_per_second") and ns_qos_entry.get("r_mbytes_per_second") != None: + if ( + not request.HasField("r_mbytes_per_second") + and ns_qos_entry.get("r_mbytes_per_second") != None + ): request.r_mbytes_per_second = int(ns_qos_entry["r_mbytes_per_second"]) - if not request.HasField("w_mbytes_per_second") and ns_qos_entry.get("w_mbytes_per_second") != None: + if ( + not request.HasField("w_mbytes_per_second") + and ns_qos_entry.get("w_mbytes_per_second") != None + ): request.w_mbytes_per_second = int(ns_qos_entry["w_mbytes_per_second"]) limits_to_set = self.get_qos_limits_string(request) - self.logger.debug(f"After merging current QOS limits with previous ones for namespace {request.nsid} on {request.subsystem_nqn},{limits_to_set}") + self.logger.debug( + f"After merging current QOS limits with previous ones for namespace {request.nsid} on {request.subsystem_nqn},{limits_to_set}" + ) omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: try: ret = rpc_bdev.bdev_set_qos_limit( - self.spdk_rpc_client, - **set_qos_limits_args) + self.spdk_rpc_client, **set_qos_limits_args + ) self.logger.debug(f"bdev_set_qos_limit {bdev_name}: {ret}") except Exception as ex: errmsg = f"Failure setting QOS limits for namespace {request.nsid} on {request.subsystem_nqn}" @@ -1986,7 +2495,9 @@ def namespace_set_qos_limits_safe(self, request, context): status = errno.EINVAL if resp: status = resp["code"] - errmsg = f"Failure setting namespace's QOS limits: {resp['message']}" + errmsg = ( + f"Failure setting namespace's QOS limits: {resp['message']}" + ) return pb2.req_status(status=status, error_message=errmsg) # Just in case SPDK failed with no exception @@ -1999,8 +2510,13 @@ def namespace_set_qos_limits_safe(self, request, context): # Update gateway state try: json_req = json_format.MessageToJson( - request, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_namespace_qos(request.subsystem_nqn, request.nsid, json_req) + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + self.gateway_state.add_namespace_qos( + request.subsystem_nqn, request.nsid, json_req + ) except Exception as ex: errmsg = f"Error persisting namespace QOS settings {request.nsid} on {request.subsystem_nqn}" self.logger.exception(errmsg) @@ -2011,30 +2527,38 @@ def namespace_set_qos_limits_safe(self, request, context): def namespace_set_qos_limits(self, request, context=None): """Set namespace's qos limits.""" - return self.execute_grpc_function(self.namespace_set_qos_limits_safe, request, context) + return self.execute_grpc_function( + self.namespace_set_qos_limits_safe, request, context + ) def namespace_resize_safe(self, request, context=None): """Resize a namespace.""" peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to resize namespace {request.nsid} on {request.subsystem_nqn} to {request.new_size} MiB, context: {context}{peer_msg}") + self.logger.info( + f"Received request to resize namespace {request.nsid} on {request.subsystem_nqn} to {request.new_size} MiB, context: {context}{peer_msg}" + ) if not request.nsid: errmsg = f"Failure resizing namespace, missing NSID" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not request.subsystem_nqn: errmsg = f"Failure resizing namespace {request.nsid}, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if request.new_size <= 0: - errmsg = f"Failure resizing namespace {request.nsid}: New size must be positive" + errmsg = ( + f"Failure resizing namespace {request.nsid}: New size must be positive" + ) self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, request.nsid + ) if find_ret.empty(): errmsg = f"Failure resizing namespace {request.nsid} on {request.subsystem_nqn}: Can't find namespace" self.logger.error(errmsg) @@ -2065,24 +2589,30 @@ def namespace_delete_safe(self, request, context): if not request.nsid: errmsg = f"Failure deleting namespace, missing NSID" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not request.subsystem_nqn: errmsg = f"Failure deleting namespace {request.nsid}, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to delete namespace {request.nsid} from {request.subsystem_nqn}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to delete namespace {request.nsid} from {request.subsystem_nqn}, context: {context}{peer_msg}" + ) - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, request.nsid + ) if find_ret.empty(): errmsg = f"Failure deleting namespace: Can't find namespace" self.logger.error(errmsg) return pb2.req_status(status=errno.ENODEV, error_message=errmsg) bdev_name = find_ret.bdev if not bdev_name: - self.logger.warning(f"Can't find namespace's bdev name, will try to delete namespace anyway") + self.logger.warning( + f"Can't find namespace's bdev name, will try to delete namespace anyway" + ) omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: @@ -2090,10 +2620,14 @@ def namespace_delete_safe(self, request, context): if ret.status != 0: return ret - self.remove_namespace_from_state(request.subsystem_nqn, request.nsid, context) - self.subsystem_nsid_bdev_and_uuid.remove_namespace(request.subsystem_nqn, request.nsid) + self.remove_namespace_from_state( + request.subsystem_nqn, request.nsid, context + ) + self.subsystem_nsid_bdev_and_uuid.remove_namespace( + request.subsystem_nqn, request.nsid + ) if bdev_name: - ret_del = self.delete_bdev(bdev_name, peer_msg = peer_msg) + ret_del = self.delete_bdev(bdev_name, peer_msg=peer_msg) if ret_del.status != 0: errmsg = f"Failure deleting namespace {request.nsid} from {request.subsystem_nqn}: {ret_del.error_message}" self.logger.error(errmsg) @@ -2109,25 +2643,27 @@ def namespace_add_host_safe(self, request, context): """Add a host to a namespace.""" peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to add host {request.host_nqn} to namespace {request.nsid} on {request.subsystem_nqn}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to add host {request.host_nqn} to namespace {request.nsid} on {request.subsystem_nqn}, context: {context}{peer_msg}" + ) if not request.nsid: errmsg = f"Failure adding host to namespace, missing NSID" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not request.subsystem_nqn: errmsg = f"Failure adding host to namespace {request.nsid}, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not request.host_nqn: errmsg = f"Failure adding host to namespace {request.nsid} on {request.subsystem_nqn}, missing host NQN" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if request.host_nqn == "*": - errmsg = f"Failure adding host to namespace {request.nsid} on {request.subsystem_nqn}, host can't be \"*\"" + errmsg = f'Failure adding host to namespace {request.nsid} on {request.subsystem_nqn}, host can\'t be "*"' self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -2136,12 +2672,12 @@ def namespace_add_host_safe(self, request, context): if rc[0] != 0: errmsg = f"Failure adding host {request.host_nqn} to namespace {request.nsid} on {request.subsystem_nqn}, invalid subsystem NQN: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc[0], error_message = errmsg) + return pb2.req_status(status=rc[0], error_message=errmsg) rc = GatewayUtils.is_valid_nqn(request.host_nqn) if rc[0] != 0: errmsg = f"Failure adding host {request.host_nqn} to namespace {request.nsid} on {request.subsystem_nqn}, invalid host NQN: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc[0], error_message = errmsg) + return pb2.req_status(status=rc[0], error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.subsystem_nqn): errmsg = f"Failure adding host to namespace {request.nsid} on {request.subsystem_nqn}, subsystem NQN can't be a discovery NQN" @@ -2153,7 +2689,9 @@ def namespace_add_host_safe(self, request, context): self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, request.nsid + ) if not find_ret.empty(): if not find_ret.no_auto_visible: errmsg = f"Failure adding host {request.host_nqn} to namespace {request.nsid} on {request.subsystem_nqn}, namespace is visible to all hosts" @@ -2172,7 +2710,7 @@ def namespace_add_host_safe(self, request, context): self.spdk_rpc_client, nqn=request.subsystem_nqn, nsid=request.nsid, - host=request.host_nqn + host=request.host_nqn, ) self.logger.debug(f"ns_visible {request.host_nqn}: {ret}") if not find_ret.empty(): @@ -2188,8 +2726,13 @@ def namespace_add_host_safe(self, request, context): # Update gateway state try: json_req = json_format.MessageToJson( - request, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_namespace_host(request.subsystem_nqn, request.nsid, request.host_nqn, json_req) + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + self.gateway_state.add_namespace_host( + request.subsystem_nqn, request.nsid, request.host_nqn, json_req + ) except Exception as ex: errmsg = f"Error persisting host {request.host_nqn} for namespace {request.nsid} on {request.subsystem_nqn}" self.logger.exception(errmsg) @@ -2200,31 +2743,35 @@ def namespace_add_host_safe(self, request, context): def namespace_add_host(self, request, context=None): """Add a host to a namespace.""" - return self.execute_grpc_function(self.namespace_add_host_safe, request, context) + return self.execute_grpc_function( + self.namespace_add_host_safe, request, context + ) def namespace_delete_host_safe(self, request, context): """Delete a host from a namespace.""" peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to delete host {request.host_nqn} from namespace {request.nsid} on {request.subsystem_nqn}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to delete host {request.host_nqn} from namespace {request.nsid} on {request.subsystem_nqn}, context: {context}{peer_msg}" + ) if not request.nsid: errmsg = f"Failure deleting host from namespace, missing NSID" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not request.subsystem_nqn: errmsg = f"Failure deleting host from namespace {request.nsid}, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not request.host_nqn: errmsg = f"Failure deleting host from namespace {request.nsid} on {request.subsystem_nqn}, missing host NQN" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if request.host_nqn == "*": - errmsg = f"Failure deleting host from namespace {request.nsid} on {request.subsystem_nqn}, host can't be \"*\"" + errmsg = f'Failure deleting host from namespace {request.nsid} on {request.subsystem_nqn}, host can\'t be "*"' self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -2233,12 +2780,12 @@ def namespace_delete_host_safe(self, request, context): if rc[0] != 0: errmsg = f"Failure deleting host {request.host_nqn} from namespace {request.nsid} on {request.subsystem_nqn}, invalid subsystem NQN: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc[0], error_message = errmsg) + return pb2.req_status(status=rc[0], error_message=errmsg) rc = GatewayUtils.is_valid_nqn(request.host_nqn) if rc[0] != 0: errmsg = f"Failure deleting host {request.host_nqn} from namespace {request.nsid} on {request.subsystem_nqn}, invalid host NQN: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc[0], error_message = errmsg) + return pb2.req_status(status=rc[0], error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.subsystem_nqn): errmsg = f"Failure deleting host from namespace {request.nsid} on {request.subsystem_nqn}, subsystem NQN can't be a discovery NQN" @@ -2250,7 +2797,9 @@ def namespace_delete_host_safe(self, request, context): self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(request.subsystem_nqn, request.nsid) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + request.subsystem_nqn, request.nsid + ) if not find_ret.empty() and not find_ret.no_auto_visible: errmsg = f"Failure deleting host from namespace {request.nsid} on {request.subsystem_nqn}, namespace is visible to all hosts" self.logger.error(f"{errmsg}") @@ -2263,7 +2812,7 @@ def namespace_delete_host_safe(self, request, context): self.spdk_rpc_client, nqn=request.subsystem_nqn, nsid=request.nsid, - host=request.host_nqn + host=request.host_nqn, ) self.logger.debug(f"ns_visible {request.host_nqn}: {ret}") if not find_ret.empty(): @@ -2278,7 +2827,9 @@ def namespace_delete_host_safe(self, request, context): if context: # Update gateway state try: - self.gateway_state.remove_namespace_host(request.subsystem_nqn, request.nsid, request.host_nqn) + self.gateway_state.remove_namespace_host( + request.subsystem_nqn, request.nsid, request.host_nqn + ) except Exception as ex: errmsg = f"Error persisting deletion of host {request.host_nqn} for namespace {request.nsid} on {request.subsystem_nqn}" self.logger.exception(errmsg) @@ -2289,7 +2840,9 @@ def namespace_delete_host_safe(self, request, context): def namespace_delete_host(self, request, context=None): """Delete a host from a namespace.""" - return self.execute_grpc_function(self.namespace_delete_host_safe, request, context) + return self.execute_grpc_function( + self.namespace_delete_host_safe, request, context + ) def matching_host_exists(self, context, subsys_nqn, host_nqn) -> bool: if not context: @@ -2315,32 +2868,47 @@ def get_subsystem_hosts(self, subsys_nqn): pass return hosts - def _create_dhchap_key_files(self, subsystem_nqn, host_nqn, dhchap_key, dhchap_ctrlr_key, err_prefix): + def _create_dhchap_key_files( + self, subsystem_nqn, host_nqn, dhchap_key, dhchap_ctrlr_key, err_prefix + ): assert dhchap_key, "DH-HMAC-CHAP key value can't be empty" dhchap_file = None dhchap_key_name = None if dhchap_key: - dhchap_file = self.create_host_dhchap_file(subsystem_nqn, host_nqn, dhchap_key) + dhchap_file = self.create_host_dhchap_file( + subsystem_nqn, host_nqn, dhchap_key + ) if not dhchap_file: - errmsg=f"{err_prefix}: Can't write DH-HMAC-CHAP file" + errmsg = f"{err_prefix}: Can't write DH-HMAC-CHAP file" self.logger.error(f"{errmsg}") return (errno.ENOENT, errmsg, None, None, None, None) dhchap_key_name = GatewayService.construct_key_name_for_keyring( - subsystem_nqn, host_nqn, GatewayService.DHCHAP_PREFIX) + subsystem_nqn, host_nqn, GatewayService.DHCHAP_PREFIX + ) dhchap_ctrlr_file = None dhchap_ctrlr_key_name = None if dhchap_ctrlr_key: - dhchap_ctrlr_file = self.create_host_dhchap_file(subsystem_nqn, host_nqn, dhchap_ctrlr_key) + dhchap_ctrlr_file = self.create_host_dhchap_file( + subsystem_nqn, host_nqn, dhchap_ctrlr_key + ) if not dhchap_ctrlr_file: - errmsg=f"{err_prefix}: Can't write DH-HMAC-CHAP controller file" + errmsg = f"{err_prefix}: Can't write DH-HMAC-CHAP controller file" self.logger.error(f"{errmsg}") if dhchap_file: self.remove_host_dhchap_file(subsystem_nqn, host_nqn) return (errno.ENOENT, errmsg, None, None, None, None) dhchap_ctrlr_key_name = GatewayService.construct_key_name_for_keyring( - subsystem_nqn, host_nqn, GatewayService.DHCHAP_CONTROLLER_PREFIX) + subsystem_nqn, host_nqn, GatewayService.DHCHAP_CONTROLLER_PREFIX + ) - return (0, "", dhchap_file, dhchap_key_name, dhchap_ctrlr_file, dhchap_ctrlr_key_name) + return ( + 0, + "", + dhchap_file, + dhchap_key_name, + dhchap_ctrlr_file, + dhchap_ctrlr_key_name, + ) def _add_key_to_keyring(self, keytype, filename, keyname): if not keyname or not filename: @@ -2367,8 +2935,12 @@ def _add_key_to_keyring(self, keytype, filename, keyname): pass try: - ret = rpc_keyring.keyring_file_add_key(self.spdk_rpc_client, keyname, filename) - self.logger.debug(f"keyring_file_add_key {keyname} and file {filename}: {ret}") + ret = rpc_keyring.keyring_file_add_key( + self.spdk_rpc_client, keyname, filename + ) + self.logger.debug( + f"keyring_file_add_key {keyname} and file {filename}: {ret}" + ) self.logger.info(f"Added {keytype} key {keyname} to keyring") except Exception: pass @@ -2378,61 +2950,78 @@ def add_host_safe(self, request, context): peer_msg = self.get_peer_message(context) if request.host_nqn == "*": - self.logger.info(f"Received request to allow any host access for {request.subsystem_nqn}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to allow any host access for {request.subsystem_nqn}, context: {context}{peer_msg}" + ) else: self.logger.info( - f"Received request to add host {request.host_nqn} to {request.subsystem_nqn}, psk: {request.psk}, dhchap: {request.dhchap_key}, context: {context}{peer_msg}") + f"Received request to add host {request.host_nqn} to {request.subsystem_nqn}, psk: {request.psk}, dhchap: {request.dhchap_key}, context: {context}{peer_msg}" + ) - all_host_failure_prefix=f"Failure allowing open host access to {request.subsystem_nqn}" - host_failure_prefix=f"Failure adding host {request.host_nqn} to {request.subsystem_nqn}" + all_host_failure_prefix = ( + f"Failure allowing open host access to {request.subsystem_nqn}" + ) + host_failure_prefix = ( + f"Failure adding host {request.host_nqn} to {request.subsystem_nqn}" + ) if not GatewayState.is_key_element_valid(request.host_nqn): - errmsg = f"{host_failure_prefix}: Invalid host NQN \"{request.host_nqn}\", contains invalid characters" + errmsg = f'{host_failure_prefix}: Invalid host NQN "{request.host_nqn}", contains invalid characters' self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not GatewayState.is_key_element_valid(request.subsystem_nqn): - errmsg = f"{host_failure_prefix}: Invalid subsystem NQN \"{request.subsystem_nqn}\", contains invalid characters" + errmsg = f'{host_failure_prefix}: Invalid subsystem NQN "{request.subsystem_nqn}", contains invalid characters' self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - if request.host_nqn == "*" and self.host_info.does_subsystem_have_dhchap_key(request.subsystem_nqn): - errmsg=f"{all_host_failure_prefix}: Can't allow any host access on a subsystem having a DH-HMAC-CHAP key" + if request.host_nqn == "*" and self.host_info.does_subsystem_have_dhchap_key( + request.subsystem_nqn + ): + errmsg = f"{all_host_failure_prefix}: Can't allow any host access on a subsystem having a DH-HMAC-CHAP key" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if self.verify_nqns: rc = GatewayService.is_valid_host_nqn(request.host_nqn) if rc.status != 0: errmsg = f"{host_failure_prefix}: {rc.error_message}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc.status, error_message = errmsg) + return pb2.req_status(status=rc.status, error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.subsystem_nqn): if request.host_nqn == "*": - errmsg=f"{all_host_failure_prefix}: Can't allow host access to a discovery subsystem" + errmsg = f"{all_host_failure_prefix}: Can't allow host access to a discovery subsystem" else: - errmsg=f"{host_failure_prefix}: Can't add host to a discovery subsystem" + errmsg = ( + f"{host_failure_prefix}: Can't add host to a discovery subsystem" + ) self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.host_nqn): - errmsg=f"{host_failure_prefix}: Can't use a discovery NQN as host's" + errmsg = f"{host_failure_prefix}: Can't use a discovery NQN as host's" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if request.psk and request.host_nqn == "*": - errmsg=f"{all_host_failure_prefix}: PSK is only allowed for specific hosts" + errmsg = ( + f"{all_host_failure_prefix}: PSK is only allowed for specific hosts" + ) self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if request.dhchap_key and request.host_nqn == "*": - errmsg=f"{all_host_failure_prefix}: DH-HMAC-CHAP key is only allowed for specific hosts" + errmsg = f"{all_host_failure_prefix}: DH-HMAC-CHAP key is only allowed for specific hosts" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - if request.dhchap_key and not self.host_info.does_subsystem_have_dhchap_key(request.subsystem_nqn): - self.logger.warning(f"Host {request.host_nqn} has a DH-HMAC-CHAP key but subsystem {request.subsystem_nqn} has no key, a unidirectional authentication will be used") + if request.dhchap_key and not self.host_info.does_subsystem_have_dhchap_key( + request.subsystem_nqn + ): + self.logger.warning( + f"Host {request.host_nqn} has a DH-HMAC-CHAP key but subsystem {request.subsystem_nqn} has no key, a unidirectional authentication will be used" + ) if request.host_nqn == "*": secure = False @@ -2440,16 +3029,20 @@ def add_host_safe(self, request, context): for listener in self.subsystem_listeners[request.subsystem_nqn]: (_, _, _, secure) = listener if secure: - errmsg=f"{all_host_failure_prefix}: Can't allow any host on a subsystem with secure listeners" + errmsg = f"{all_host_failure_prefix}: Can't allow any host on a subsystem with secure listeners" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) except Exception: pass - host_already_exist = self.matching_host_exists(context, request.subsystem_nqn, request.host_nqn) + host_already_exist = self.matching_host_exists( + context, request.subsystem_nqn, request.host_nqn + ) if host_already_exist: if request.host_nqn == "*": - errmsg = f"{all_host_failure_prefix}: Open host access is already allowed" + errmsg = ( + f"{all_host_failure_prefix}: Open host access is already allowed" + ) self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EEXIST, error_message=errmsg) else: @@ -2457,32 +3050,45 @@ def add_host_safe(self, request, context): self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EEXIST, error_message=errmsg) - if request.host_nqn != "*" and self.host_info.get_host_count(request.subsystem_nqn) >= self.max_hosts_per_subsystem: + if ( + request.host_nqn != "*" + and self.host_info.get_host_count(request.subsystem_nqn) + >= self.max_hosts_per_subsystem + ): errmsg = f"{host_failure_prefix}: Maximal number of hosts for subsystem ({self.max_hosts_per_subsystem}) has already been reached" self.logger.error(f"{errmsg}") - return pb2.subsys_status(status = errno.E2BIG, error_message = errmsg, nqn = request.subsystem_nqn) + return pb2.subsys_status( + status=errno.E2BIG, error_message=errmsg, nqn=request.subsystem_nqn + ) - dhchap_ctrlr_key = self.host_info.get_subsystem_dhchap_key(request.subsystem_nqn) + dhchap_ctrlr_key = self.host_info.get_subsystem_dhchap_key( + request.subsystem_nqn + ) if dhchap_ctrlr_key: - self.logger.info(f"Got DHCHAP key {dhchap_ctrlr_key} for subsystem {request.subsystem_nqn}") + self.logger.info( + f"Got DHCHAP key {dhchap_ctrlr_key} for subsystem {request.subsystem_nqn}" + ) if dhchap_ctrlr_key and not request.dhchap_key: - errmsg=f"{host_failure_prefix}: Host must have a DH-HMAC-CHAP key if the subsystem has one" + errmsg = f"{host_failure_prefix}: Host must have a DH-HMAC-CHAP key if the subsystem has one" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) psk_file = None psk_key_name = None if request.psk: - psk_file = self.create_host_psk_file(request.subsystem_nqn, request.host_nqn, request.psk) + psk_file = self.create_host_psk_file( + request.subsystem_nqn, request.host_nqn, request.psk + ) if not psk_file: - errmsg=f"{host_failure_prefix}: Can't write PSK file" + errmsg = f"{host_failure_prefix}: Can't write PSK file" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.ENOENT, error_message=errmsg) psk_key_name = GatewayService.construct_key_name_for_keyring( - request.subsystem_nqn, request.host_nqn, GatewayService.PSK_PREFIX) + request.subsystem_nqn, request.host_nqn, GatewayService.PSK_PREFIX + ) if len(psk_key_name) >= SubsystemHostAuth.MAX_PSK_KEY_NAME_LENGTH: - errmsg=f"{host_failure_prefix}: PSK key name {psk_key_name} is too long, max length is {SubsystemHostAuth.MAX_PSK_KEY_NAME_LENGTH}" + errmsg = f"{host_failure_prefix}: PSK key name {psk_key_name} is too long, max length is {SubsystemHostAuth.MAX_PSK_KEY_NAME_LENGTH}" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.E2BIG, error_message=errmsg) @@ -2491,18 +3097,26 @@ def add_host_safe(self, request, context): dhchap_ctrlr_file = None dhchap_ctrlr_key_name = None if request.dhchap_key: - (key_files_status, - key_file_errmsg, - dhchap_file, - dhchap_key_name, - dhchap_ctrlr_file, - dhchap_ctrlr_key_name) = self._create_dhchap_key_files( - request.subsystem_nqn, request.host_nqn, - request.dhchap_key, dhchap_ctrlr_key, host_failure_prefix) + ( + key_files_status, + key_file_errmsg, + dhchap_file, + dhchap_key_name, + dhchap_ctrlr_file, + dhchap_ctrlr_key_name, + ) = self._create_dhchap_key_files( + request.subsystem_nqn, + request.host_nqn, + request.dhchap_key, + dhchap_ctrlr_key, + host_failure_prefix, + ) if key_files_status != 0: if psk_file: self.remove_host_psk_file(request.subsystem_nqn, request.host_nqn) - return pb2.req_status(status=key_files_status, error_message=key_file_errmsg) + return pb2.req_status( + status=key_files_status, error_message=key_file_errmsg + ) omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: @@ -2517,8 +3131,14 @@ def add_host_safe(self, request, context): self.host_info.allow_any_host(request.subsystem_nqn) else: # Allow single host access to subsystem self._add_key_to_keyring("PSK", psk_file, psk_key_name) - self._add_key_to_keyring("DH-HMAC-CHAP", dhchap_file, dhchap_key_name) - self._add_key_to_keyring("DH-HMAC-CHAP controller", dhchap_ctrlr_file, dhchap_ctrlr_key_name) + self._add_key_to_keyring( + "DH-HMAC-CHAP", dhchap_file, dhchap_key_name + ) + self._add_key_to_keyring( + "DH-HMAC-CHAP controller", + dhchap_ctrlr_file, + dhchap_ctrlr_key_name, + ) ret = rpc_nvmf.nvmf_subsystem_add_host( self.spdk_rpc_client, nqn=request.subsystem_nqn, @@ -2529,22 +3149,34 @@ def add_host_safe(self, request, context): ) self.logger.debug(f"add_host {request.host_nqn}: {ret}") if psk_file: - self.host_info.add_psk_host(request.subsystem_nqn, request.host_nqn, request.psk) - self.remove_host_psk_file(request.subsystem_nqn, request.host_nqn) + self.host_info.add_psk_host( + request.subsystem_nqn, request.host_nqn, request.psk + ) + self.remove_host_psk_file( + request.subsystem_nqn, request.host_nqn + ) try: - rpc_keyring.keyring_file_remove_key(self.spdk_rpc_client, psk_key_name) + rpc_keyring.keyring_file_remove_key( + self.spdk_rpc_client, psk_key_name + ) except Exception: pass if dhchap_file: - self.host_info.add_dhchap_host(request.subsystem_nqn, request.host_nqn, request.dhchap_key) + self.host_info.add_dhchap_host( + request.subsystem_nqn, request.host_nqn, request.dhchap_key + ) self.host_info.add_host_nqn(request.subsystem_nqn, request.host_nqn) except Exception as ex: if request.host_nqn == "*": self.logger.exception(all_host_failure_prefix) errmsg = f"{all_host_failure_prefix}:\n{ex}" else: - self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) - self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_key_files( + request.subsystem_nqn, request.host_nqn + ) + self.remove_all_host_keys_from_keyring( + request.subsystem_nqn, request.host_nqn + ) self.logger.exception(host_failure_prefix) errmsg = f"{host_failure_prefix}:\n{ex}" resp = self.parse_json_exeption(ex) @@ -2563,8 +3195,12 @@ def add_host_safe(self, request, context): errmsg = all_host_failure_prefix else: errmsg = host_failure_prefix - self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) - self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_key_files( + request.subsystem_nqn, request.host_nqn + ) + self.remove_all_host_keys_from_keyring( + request.subsystem_nqn, request.host_nqn + ) self.logger.error(errmsg) return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -2572,14 +3208,23 @@ def add_host_safe(self, request, context): # Update gateway state try: json_req = json_format.MessageToJson( - request, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_host(request.subsystem_nqn, request.host_nqn, json_req) + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + self.gateway_state.add_host( + request.subsystem_nqn, request.host_nqn, json_req + ) except Exception as ex: errmsg = f"Error persisting host {request.host_nqn} access addition" self.logger.exception(errmsg) errmsg = f"{errmsg}:\n{ex}" - self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) - self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_key_files( + request.subsystem_nqn, request.host_nqn + ) + self.remove_all_host_keys_from_keyring( + request.subsystem_nqn, request.host_nqn + ) return pb2.req_status(status=errno.EINVAL, error_message=errmsg) return pb2.req_status(status=0, error_message=os.strerror(0)) @@ -2592,7 +3237,9 @@ def remove_host_from_state(self, subsystem_nqn, host_nqn, context): return pb2.req_status(status=0, error_message=os.strerror(0)) if context: - assert self.omap_lock.locked(), "OMAP is unlocked when calling remove_host_from_state()" + assert ( + self.omap_lock.locked() + ), "OMAP is unlocked when calling remove_host_from_state()" # Update gateway state try: self.gateway_state.remove_host(subsystem_nqn, host_nqn) @@ -2607,29 +3254,33 @@ def remove_host_safe(self, request, context): """Removes a host from a subsystem.""" peer_msg = self.get_peer_message(context) - all_host_failure_prefix=f"Failure disabling open host access to {request.subsystem_nqn}" - host_failure_prefix=f"Failure removing host {request.host_nqn} access from {request.subsystem_nqn}" + all_host_failure_prefix = ( + f"Failure disabling open host access to {request.subsystem_nqn}" + ) + host_failure_prefix = f"Failure removing host {request.host_nqn} access from {request.subsystem_nqn}" if self.verify_nqns: rc = GatewayService.is_valid_host_nqn(request.host_nqn) if rc.status != 0: errmsg = f"{host_failure_prefix}: {rc.error_message}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc.status, error_message = errmsg) + return pb2.req_status(status=rc.status, error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.subsystem_nqn): if request.host_nqn == "*": - errmsg=f"{all_host_failure_prefix}: Can't disable open host access to a discovery subsystem" + errmsg = f"{all_host_failure_prefix}: Can't disable open host access to a discovery subsystem" else: - errmsg=f"{host_failure_prefix}: Can't remove host access from a discovery subsystem" + errmsg = f"{host_failure_prefix}: Can't remove host access from a discovery subsystem" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.host_nqn): if request.host_nqn == "*": - errmsg=f"{all_host_failure_prefix}: Can't use a discovery NQN as host's" + errmsg = ( + f"{all_host_failure_prefix}: Can't use a discovery NQN as host's" + ) else: - errmsg=f"{host_failure_prefix}: Can't use a discovery NQN as host's" + errmsg = f"{host_failure_prefix}: Can't use a discovery NQN as host's" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -2639,7 +3290,8 @@ def remove_host_safe(self, request, context): if request.host_nqn == "*": # Disable allow any host access self.logger.info( f"Received request to disable open host access to" - f" {request.subsystem_nqn}, context: {context}{peer_msg}") + f" {request.subsystem_nqn}, context: {context}{peer_msg}" + ) ret = rpc_nvmf.nvmf_subsystem_allow_any_host( self.spdk_rpc_client, nqn=request.subsystem_nqn, @@ -2650,18 +3302,29 @@ def remove_host_safe(self, request, context): else: # Remove single host access to subsystem self.logger.info( f"Received request to remove host {request.host_nqn} access from" - f" {request.subsystem_nqn}, context: {context}{peer_msg}") + f" {request.subsystem_nqn}, context: {context}{peer_msg}" + ) ret = rpc_nvmf.nvmf_subsystem_remove_host( self.spdk_rpc_client, nqn=request.subsystem_nqn, host=request.host_nqn, ) self.logger.debug(f"remove_host {request.host_nqn}: {ret}") - self.host_info.remove_psk_host(request.subsystem_nqn, request.host_nqn) - self.host_info.remove_dhchap_host(request.subsystem_nqn, request.host_nqn) - self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) - self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) - self.host_info.remove_host_nqn(request.subsystem_nqn, request.host_nqn) + self.host_info.remove_psk_host( + request.subsystem_nqn, request.host_nqn + ) + self.host_info.remove_dhchap_host( + request.subsystem_nqn, request.host_nqn + ) + self.remove_all_host_key_files( + request.subsystem_nqn, request.host_nqn + ) + self.remove_all_host_keys_from_keyring( + request.subsystem_nqn, request.host_nqn + ) + self.host_info.remove_host_nqn( + request.subsystem_nqn, request.host_nqn + ) except Exception as ex: if request.host_nqn == "*": self.logger.exception(all_host_failure_prefix) @@ -2670,7 +3333,9 @@ def remove_host_safe(self, request, context): self.logger.exception(host_failure_prefix) errmsg = f"{host_failure_prefix}:\n{ex}" self.logger.error(errmsg) - self.remove_host_from_state(request.subsystem_nqn, request.host_nqn, context) + self.remove_host_from_state( + request.subsystem_nqn, request.host_nqn, context + ) resp = self.parse_json_exeption(ex) status = errno.EINVAL if resp: @@ -2688,10 +3353,14 @@ def remove_host_safe(self, request, context): else: errmsg = host_failure_prefix self.logger.error(errmsg) - self.remove_host_from_state(request.subsystem_nqn, request.host_nqn, context) + self.remove_host_from_state( + request.subsystem_nqn, request.host_nqn, context + ) return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - return self.remove_host_from_state(request.subsystem_nqn, request.host_nqn, context) + return self.remove_host_from_state( + request.subsystem_nqn, request.host_nqn, context + ) def remove_host(self, request, context=None): return self.execute_grpc_function(self.remove_host_safe, request, context) @@ -2700,60 +3369,67 @@ def change_host_key_safe(self, request, context): """Changes host's inband authentication key.""" peer_msg = self.get_peer_message(context) - failure_prefix=f"Failure changing DH-HMAC-CHAP key for host {request.host_nqn} on subsystem {request.subsystem_nqn}" + failure_prefix = f"Failure changing DH-HMAC-CHAP key for host {request.host_nqn} on subsystem {request.subsystem_nqn}" self.logger.info( - f"Received request to change inband authentication key for host {request.host_nqn} on subsystem {request.subsystem_nqn}, dhchap: {request.dhchap_key}, context: {context}{peer_msg}") + f"Received request to change inband authentication key for host {request.host_nqn} on subsystem {request.subsystem_nqn}, dhchap: {request.dhchap_key}, context: {context}{peer_msg}" + ) if request.host_nqn == "*": - errmsg=f"{failure_prefix}: Host NQN can't be '*'" + errmsg = f"{failure_prefix}: Host NQN can't be '*'" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not GatewayState.is_key_element_valid(request.host_nqn): - errmsg = f"{failure_prefix}: Invalid host NQN \"{request.host_nqn}\", contains invalid characters" + errmsg = f'{failure_prefix}: Invalid host NQN "{request.host_nqn}", contains invalid characters' self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not GatewayState.is_key_element_valid(request.subsystem_nqn): - errmsg = f"{failure_prefix}: Invalid subsystem NQN \"{request.subsystem_nqn}\", contains invalid characters" + errmsg = f'{failure_prefix}: Invalid subsystem NQN "{request.subsystem_nqn}", contains invalid characters' self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if self.verify_nqns: rc = GatewayUtils.is_valid_nqn(request.subsystem_nqn) if rc[0] != 0: errmsg = f"{failure_prefix}: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc[0], error_message = errmsg) + return pb2.req_status(status=rc[0], error_message=errmsg) rc = GatewayUtils.is_valid_nqn(request.host_nqn) if rc[0] != 0: errmsg = f"{failure_prefix}: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc[0], error_message = errmsg) + return pb2.req_status(status=rc[0], error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.subsystem_nqn): - errmsg=f"{failure_prefix}: Can't use a discovery NQN as subsystem's" + errmsg = f"{failure_prefix}: Can't use a discovery NQN as subsystem's" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.host_nqn): - errmsg=f"{failure_prefix}: Can't use a discovery NQN as host's" + errmsg = f"{failure_prefix}: Can't use a discovery NQN as host's" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) - dhchap_ctrlr_key = self.host_info.get_subsystem_dhchap_key(request.subsystem_nqn) + dhchap_ctrlr_key = self.host_info.get_subsystem_dhchap_key( + request.subsystem_nqn + ) if dhchap_ctrlr_key and not request.dhchap_key: - errmsg=f"{failure_prefix}: Host must have a DH-HMAC-CHAP key if the subsystem has one" + errmsg = f"{failure_prefix}: Host must have a DH-HMAC-CHAP key if the subsystem has one" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if request.dhchap_key and not dhchap_ctrlr_key: - self.logger.warning(f"Host {request.host_nqn} has a DH-HMAC-CHAP key but subsystem {request.subsystem_nqn} has no key, a unidirectional authentication will be used") + self.logger.warning( + f"Host {request.host_nqn} has a DH-HMAC-CHAP key but subsystem {request.subsystem_nqn} has no key, a unidirectional authentication will be used" + ) - host_already_exist = self.matching_host_exists(context, request.subsystem_nqn, request.host_nqn) + host_already_exist = self.matching_host_exists( + context, request.subsystem_nqn, request.host_nqn + ) if not host_already_exist and context: - errmsg=f"{failure_prefix}: Can't find host on subsystem" + errmsg = f"{failure_prefix}: Can't find host on subsystem" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -2762,27 +3438,39 @@ def change_host_key_safe(self, request, context): dhchap_ctrlr_file = None dhchap_ctrlr_key_name = None if request.dhchap_key: - (key_files_status, - key_file_errmsg, - dhchap_file, - dhchap_key_name, - dhchap_ctrlr_file, - dhchap_ctrlr_key_name) = self._create_dhchap_key_files( - request.subsystem_nqn, request.host_nqn, - request.dhchap_key, dhchap_ctrlr_key, failure_prefix) + ( + key_files_status, + key_file_errmsg, + dhchap_file, + dhchap_key_name, + dhchap_ctrlr_file, + dhchap_ctrlr_key_name, + ) = self._create_dhchap_key_files( + request.subsystem_nqn, + request.host_nqn, + request.dhchap_key, + dhchap_ctrlr_key, + failure_prefix, + ) if key_files_status != 0: - return pb2.req_status(status=key_files_status, error_message=key_file_errmsg) + return pb2.req_status( + status=key_files_status, error_message=key_file_errmsg + ) omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: host_psk = None if context: - host_psk = self.host_info.get_host_psk_key(request.subsystem_nqn, request.host_nqn) + host_psk = self.host_info.get_host_psk_key( + request.subsystem_nqn, request.host_nqn + ) try: self._add_key_to_keyring("DH-HMAC-CHAP", dhchap_file, dhchap_key_name) - self._add_key_to_keyring("DH-HMAC-CHAP controller", dhchap_ctrlr_file, dhchap_ctrlr_key_name) + self._add_key_to_keyring( + "DH-HMAC-CHAP controller", dhchap_ctrlr_file, dhchap_ctrlr_key_name + ) ret = rpc_nvmf.nvmf_subsystem_set_keys( self.spdk_rpc_client, request.subsystem_nqn, @@ -2808,22 +3496,35 @@ def change_host_key_safe(self, request, context): return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if dhchap_key_name: - self.host_info.add_dhchap_host(request.subsystem_nqn, request.host_nqn, request.dhchap_key) + self.host_info.add_dhchap_host( + request.subsystem_nqn, request.host_nqn, request.dhchap_key + ) else: - self.host_info.remove_dhchap_host(request.subsystem_nqn, request.host_nqn) + self.host_info.remove_dhchap_host( + request.subsystem_nqn, request.host_nqn + ) self.remove_all_host_key_files(request.subsystem_nqn, request.host_nqn) - self.remove_all_host_keys_from_keyring(request.subsystem_nqn, request.host_nqn) + self.remove_all_host_keys_from_keyring( + request.subsystem_nqn, request.host_nqn + ) if context: # Update gateway state try: - add_req = pb2.add_host_req(subsystem_nqn=request.subsystem_nqn, - host_nqn=request.host_nqn, - psk=host_psk, - dhchap_key=request.dhchap_key) + add_req = pb2.add_host_req( + subsystem_nqn=request.subsystem_nqn, + host_nqn=request.host_nqn, + psk=host_psk, + dhchap_key=request.dhchap_key, + ) json_req = json_format.MessageToJson( - add_req, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_host(request.subsystem_nqn, request.host_nqn, json_req) + add_req, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + self.gateway_state.add_host( + request.subsystem_nqn, request.host_nqn, json_req + ) except Exception as ex: errmsg = f"Error persisting host change key for host {request.host_nqn} in {request.subsystem_nqn}" self.logger.exception(errmsg) @@ -2840,9 +3541,13 @@ def list_hosts_safe(self, request, context): """List hosts.""" peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to list hosts for {request.subsystem}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to list hosts for {request.subsystem}, context: {context}{peer_msg}" + ) try: - ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem) + ret = rpc_nvmf.nvmf_get_subsystems( + self.spdk_rpc_client, nqn=request.subsystem + ) self.logger.debug(f"list_hosts: {ret}") except Exception as ex: errmsg = f"Failure listing hosts, can't get subsystems" @@ -2852,7 +3557,9 @@ def list_hosts_safe(self, request, context): status = errno.EINVAL if resp: status = resp["code"] - errmsg = f"Failure listing hosts, can't get subsystems: {resp['message']}" + errmsg = ( + f"Failure listing hosts, can't get subsystems: {resp['message']}" + ) return pb2.hosts_info(status=status, error_message=errmsg, hosts=[]) hosts = [] @@ -2860,7 +3567,9 @@ def list_hosts_safe(self, request, context): for s in ret: try: if s["nqn"] != request.subsystem: - self.logger.warning(f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore') + self.logger.warning( + f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore' + ) continue try: allow_any_host = s["allow_any_host"] @@ -2872,15 +3581,20 @@ def list_hosts_safe(self, request, context): host_nqn = h["nqn"] psk = self.host_info.is_psk_host(request.subsystem, host_nqn) dhchap = self.host_info.is_dhchap_host(request.subsystem, host_nqn) - one_host = pb2.host(nqn = host_nqn, use_psk = psk, use_dhchap = dhchap) + one_host = pb2.host(nqn=host_nqn, use_psk=psk, use_dhchap=dhchap) hosts.append(one_host) break except Exception: self.logger.exception(f"{s=} parse error") pass - return pb2.hosts_info(status = 0, error_message = os.strerror(0), allow_any_host=allow_any_host, - subsystem_nqn=request.subsystem, hosts=hosts) + return pb2.hosts_info( + status=0, + error_message=os.strerror(0), + allow_any_host=allow_any_host, + subsystem_nqn=request.subsystem, + hosts=hosts, + ) def list_hosts(self, request, context=None): return self.execute_grpc_function(self.list_hosts_safe, request, context) @@ -2890,15 +3604,22 @@ def list_connections_safe(self, request, context): peer_msg = self.get_peer_message(context) log_level = logging.INFO if context else logging.DEBUG - self.logger.log(log_level, f"Received request to list connections for {request.subsystem}, context: {context}{peer_msg}") + self.logger.log( + log_level, + f"Received request to list connections for {request.subsystem}, context: {context}{peer_msg}", + ) if not request.subsystem: errmsg = f"Failure listing connections, missing subsystem NQN" self.logger.error(f"{errmsg}") - return pb2.connections_info(status=errno.EINVAL, error_message = errmsg, connections=[]) + return pb2.connections_info( + status=errno.EINVAL, error_message=errmsg, connections=[] + ) try: - qpair_ret = rpc_nvmf.nvmf_subsystem_get_qpairs(self.spdk_rpc_client, nqn=request.subsystem) + qpair_ret = rpc_nvmf.nvmf_subsystem_get_qpairs( + self.spdk_rpc_client, nqn=request.subsystem + ) self.logger.debug(f"list_connections get_qpairs: {qpair_ret}") except Exception as ex: errmsg = f"Failure listing connections, can't get qpairs" @@ -2908,11 +3629,17 @@ def list_connections_safe(self, request, context): status = errno.EINVAL if resp: status = resp["code"] - errmsg = f"Failure listing connections, can't get qpairs: {resp['message']}" - return pb2.connections_info(status=status, error_message=errmsg, connections=[]) + errmsg = ( + f"Failure listing connections, can't get qpairs: {resp['message']}" + ) + return pb2.connections_info( + status=status, error_message=errmsg, connections=[] + ) try: - ctrl_ret = rpc_nvmf.nvmf_subsystem_get_controllers(self.spdk_rpc_client, nqn=request.subsystem) + ctrl_ret = rpc_nvmf.nvmf_subsystem_get_controllers( + self.spdk_rpc_client, nqn=request.subsystem + ) self.logger.debug(f"list_connections get_controllers: {ctrl_ret}") except Exception as ex: errmsg = f"Failure listing connections, can't get controllers" @@ -2923,10 +3650,14 @@ def list_connections_safe(self, request, context): if resp: status = resp["code"] errmsg = f"Failure listing connections, can't get controllers: {resp['message']}" - return pb2.connections_info(status=status, error_message=errmsg, connections=[]) + return pb2.connections_info( + status=status, error_message=errmsg, connections=[] + ) try: - subsys_ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem) + subsys_ret = rpc_nvmf.nvmf_get_subsystems( + self.spdk_rpc_client, nqn=request.subsystem + ) self.logger.debug(f"list_connections subsystems: {subsys_ret}") except Exception as ex: errmsg = f"Failure listing connections, can't get subsystems" @@ -2937,14 +3668,18 @@ def list_connections_safe(self, request, context): if resp: status = resp["code"] errmsg = f"Failure listing connections, can't get subsystems: {resp['message']}" - return pb2.connections_info(status=status, error_message=errmsg, connections=[]) + return pb2.connections_info( + status=status, error_message=errmsg, connections=[] + ) connections = [] host_nqns = [] for s in subsys_ret: try: if s["nqn"] != request.subsystem: - self.logger.warning(f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore') + self.logger.warning( + f'Got subsystem {s["nqn"]} instead of {request.subsystem}, ignore' + ) continue try: subsys_hosts = s["hosts"] @@ -2999,7 +3734,9 @@ def list_connections_safe(self, request, context): found = True break except Exception: - self.logger.exception(f"Got exception while parsing qpair: {qp}") + self.logger.exception( + f"Got exception while parsing qpair: {qp}" + ) pass if not found: @@ -3010,17 +3747,28 @@ def list_connections_safe(self, request, context): dhchap = self.host_info.is_dhchap_host(request.subsystem, hostnqn) if request.subsystem in self.subsystem_listeners: - if (adrfam, traddr, trsvcid, True) in self.subsystem_listeners[request.subsystem]: + if (adrfam, traddr, trsvcid, True) in self.subsystem_listeners[ + request.subsystem + ]: secure = True if not trtype: trtype = "TCP" if not adrfam: adrfam = "ipv4" - one_conn = pb2.connection(nqn=hostnqn, connected=True, - traddr=traddr, trsvcid=trsvcid, trtype=trtype, adrfam=adrfam, - qpairs_count=conn["num_io_qpairs"], controller_id=conn["cntlid"], - secure=secure, use_psk=psk, use_dhchap=dhchap) + one_conn = pb2.connection( + nqn=hostnqn, + connected=True, + traddr=traddr, + trsvcid=trsvcid, + trtype=trtype, + adrfam=adrfam, + qpairs_count=conn["num_io_qpairs"], + controller_id=conn["cntlid"], + secure=secure, + use_psk=psk, + use_dhchap=dhchap, + ) connections.append(one_conn) if hostnqn in host_nqns: host_nqns.remove(hostnqn) @@ -3033,12 +3781,24 @@ def list_connections_safe(self, request, context): dhchap = False psk = self.host_info.is_psk_host(request.subsystem, nqn) dhchap = self.host_info.is_dhchap_host(request.subsystem, nqn) - one_conn = pb2.connection(nqn=nqn, connected=False, traddr="", trsvcid=0, - qpairs_count=-1, controller_id=-1, use_psk=psk, use_dhchap=dhchap) + one_conn = pb2.connection( + nqn=nqn, + connected=False, + traddr="", + trsvcid=0, + qpairs_count=-1, + controller_id=-1, + use_psk=psk, + use_dhchap=dhchap, + ) connections.append(one_conn) - return pb2.connections_info(status = 0, error_message = os.strerror(0), - subsystem_nqn=request.subsystem, connections=connections) + return pb2.connections_info( + status=0, + error_message=os.strerror(0), + subsystem_nqn=request.subsystem, + connections=connections, + ) def list_connections(self, request, context=None): return self.execute_grpc_function(self.list_connections_safe, request, context) @@ -3051,29 +3811,31 @@ def create_listener_safe(self, request, context): adrfam = GatewayEnumUtils.get_key_from_value(pb2.AddressFamily, request.adrfam) if adrfam == None: - errmsg=f"{create_listener_error_prefix}: Unknown address family {request.adrfam}" + errmsg = f"{create_listener_error_prefix}: Unknown address family {request.adrfam}" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to create {request.host_name}" - f" TCP {adrfam} listener for {request.nqn} at" - f" {request.traddr}:{request.trsvcid}, secure: {request.secure}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to create {request.host_name}" + f" TCP {adrfam} listener for {request.nqn} at" + f" {request.traddr}:{request.trsvcid}, secure: {request.secure}, context: {context}{peer_msg}" + ) traddr = GatewayUtils.unescape_address_if_ipv6(request.traddr, adrfam) if GatewayUtils.is_discovery_nqn(request.nqn): - errmsg=f"{create_listener_error_prefix}: Can't create a listener for a discovery subsystem" + errmsg = f"{create_listener_error_prefix}: Can't create a listener for a discovery subsystem" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if not GatewayState.is_key_element_valid(request.host_name): - errmsg=f"{create_listener_error_prefix}: Host name \"{request.host_name}\" contains invalid characters" + errmsg = f'{create_listener_error_prefix}: Host name "{request.host_name}" contains invalid characters' self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if request.secure and self.host_info.is_any_host_allowed(request.nqn): - errmsg=f"{create_listener_error_prefix}: Secure channel is only allowed for subsystems in which \"allow any host\" is off" + errmsg = f'{create_listener_error_prefix}: Secure channel is only allowed for subsystems in which "allow any host" is off' self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -3090,20 +3852,40 @@ def create_listener_safe(self, request, context): with omap_lock: try: if request.host_name == self.host_name: - if (adrfam, traddr, request.trsvcid, False) in self.subsystem_listeners[request.nqn] or (adrfam, traddr, request.trsvcid, True) in self.subsystem_listeners[request.nqn]: - self.logger.error(f"{request.nqn} already listens on address {request.traddr}:{request.trsvcid}") - return pb2.req_status(status=errno.EEXIST, - error_message=f"{create_listener_error_prefix}: Subsystem already listens on this address") - ret = rpc_nvmf.nvmf_subsystem_add_listener(self.spdk_rpc_client, **add_listener_args) + if ( + adrfam, + traddr, + request.trsvcid, + False, + ) in self.subsystem_listeners[request.nqn] or ( + adrfam, + traddr, + request.trsvcid, + True, + ) in self.subsystem_listeners[ + request.nqn + ]: + self.logger.error( + f"{request.nqn} already listens on address {request.traddr}:{request.trsvcid}" + ) + return pb2.req_status( + status=errno.EEXIST, + error_message=f"{create_listener_error_prefix}: Subsystem already listens on this address", + ) + ret = rpc_nvmf.nvmf_subsystem_add_listener( + self.spdk_rpc_client, **add_listener_args + ) self.logger.debug(f"create_listener: {ret}") - self.subsystem_listeners[request.nqn].add((adrfam, traddr, request.trsvcid, request.secure)) + self.subsystem_listeners[request.nqn].add( + (adrfam, traddr, request.trsvcid, request.secure) + ) else: if context: - errmsg=f"{create_listener_error_prefix}: Gateway's host name must match current host ({self.host_name})" + errmsg = f"{create_listener_error_prefix}: Gateway's host name must match current host ({self.host_name})" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.ENODEV, error_message=errmsg) else: - errmsg=f"Listener not created as gateway's host name {self.host_name} differs from requested host {request.host_name}" + errmsg = f"Listener not created as gateway's host name {self.host_name} differs from requested host {request.host_name}" self.logger.debug(f"{errmsg}") return pb2.req_status(status=0, error_message=errmsg) except Exception as ex: @@ -3119,44 +3901,60 @@ def create_listener_safe(self, request, context): # Just in case SPDK failed with no exception if not ret: self.logger.error(create_listener_error_prefix) - return pb2.req_status(status=errno.EINVAL, error_message=create_listener_error_prefix) + return pb2.req_status( + status=errno.EINVAL, error_message=create_listener_error_prefix + ) try: - self.logger.debug(f"create_listener nvmf_subsystem_listener_set_ana_state {request=} set inaccessible for all ana groups") + self.logger.debug( + f"create_listener nvmf_subsystem_listener_set_ana_state {request=} set inaccessible for all ana groups" + ) _ana_state = "inaccessible" ret = rpc_nvmf.nvmf_subsystem_listener_set_ana_state( - self.spdk_rpc_client, - nqn=request.nqn, - ana_state=_ana_state, - trtype="TCP", - traddr=traddr, - trsvcid=str(request.trsvcid), - adrfam=adrfam) - self.logger.debug(f"create_listener nvmf_subsystem_listener_set_ana_state response {ret=}") + self.spdk_rpc_client, + nqn=request.nqn, + ana_state=_ana_state, + trtype="TCP", + traddr=traddr, + trsvcid=str(request.trsvcid), + adrfam=adrfam, + ) + self.logger.debug( + f"create_listener nvmf_subsystem_listener_set_ana_state response {ret=}" + ) # have been provided with ana state for this nqn prior to creation # update optimized ana groups if self.ana_map[request.nqn]: - for x in range (self.subsys_max_ns[request.nqn]): - ana_grp = x+1 - if ana_grp in self.ana_map[request.nqn] and self.ana_map[request.nqn][ana_grp] == pb2.ana_state.OPTIMIZED: + for x in range(self.subsys_max_ns[request.nqn]): + ana_grp = x + 1 + if ( + ana_grp in self.ana_map[request.nqn] + and self.ana_map[request.nqn][ana_grp] + == pb2.ana_state.OPTIMIZED + ): _ana_state = "optimized" - self.logger.debug(f"using ana_map: set listener on nqn : {request.nqn} ana state : {_ana_state} for group : {ana_grp}") + self.logger.debug( + f"using ana_map: set listener on nqn : {request.nqn} ana state : {_ana_state} for group : {ana_grp}" + ) ret = rpc_nvmf.nvmf_subsystem_listener_set_ana_state( - self.spdk_rpc_client, - nqn=request.nqn, - ana_state=_ana_state, - trtype="TCP", - traddr=traddr, - trsvcid=str(request.trsvcid), - adrfam=adrfam, - anagrpid=ana_grp ) - self.logger.debug(f"create_listener nvmf_subsystem_listener_set_ana_state response {ret=}") + self.spdk_rpc_client, + nqn=request.nqn, + ana_state=_ana_state, + trtype="TCP", + traddr=traddr, + trsvcid=str(request.trsvcid), + adrfam=adrfam, + anagrpid=ana_grp, + ) + self.logger.debug( + f"create_listener nvmf_subsystem_listener_set_ana_state response {ret=}" + ) except Exception as ex: - errmsg=f"{create_listener_error_prefix}: Error setting ANA state" + errmsg = f"{create_listener_error_prefix}: Error setting ANA state" self.logger.exception(errmsg) - errmsg=f"{errmsg}:\n{ex}" + errmsg = f"{errmsg}:\n{ex}" resp = self.parse_json_exeption(ex) status = errno.EINVAL if resp: @@ -3168,13 +3966,22 @@ def create_listener_safe(self, request, context): # Update gateway state try: json_req = json_format.MessageToJson( - request, preserving_proto_field_name=True, including_default_value_fields=True) - self.gateway_state.add_listener(request.nqn, - request.host_name, - "TCP", request.traddr, - request.trsvcid, json_req) + request, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + self.gateway_state.add_listener( + request.nqn, + request.host_name, + "TCP", + request.traddr, + request.trsvcid, + json_req, + ) except Exception as ex: - errmsg = f"Error persisting listener {request.traddr}:{request.trsvcid}" + errmsg = ( + f"Error persisting listener {request.traddr}:{request.trsvcid}" + ) self.logger.exception(errmsg) errmsg = f"{errmsg}:\n{ex}" return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -3189,7 +3996,9 @@ def remove_listener_from_state(self, nqn, host_name, traddr, port, context): return pb2.req_status(status=0, error_message=os.strerror(0)) if context: - assert self.omap_lock.locked(), "OMAP is unlocked when calling remove_listener_from_state()" + assert ( + self.omap_lock.locked() + ), "OMAP is unlocked when calling remove_listener_from_state()" host_name = host_name.strip() listener_hosts = [] @@ -3203,7 +4012,9 @@ def remove_listener_from_state(self, nqn, host_name, traddr, port, context): listener = json.loads(val) listener_nqn = listener["nqn"] if listener_nqn != nqn: - self.logger.warning(f"Got subsystem {listener_nqn} instead of {nqn}, ignore") + self.logger.warning( + f"Got subsystem {listener_nqn} instead of {nqn}, ignore" + ) continue if listener["traddr"] != traddr: continue @@ -3226,7 +4037,9 @@ def remove_listener_from_state(self, nqn, host_name, traddr, port, context): self.logger.exception(errmsg) if not req_status: errmsg = f"{errmsg}:\n{ex}" - req_status = pb2.req_status(status=errno.EINVAL, error_message=errmsg) + req_status = pb2.req_status( + status=errno.EINVAL, error_message=errmsg + ) if not req_status: req_status = pb2.req_status(status=0, error_message=os.strerror(0)) @@ -3241,7 +4054,7 @@ def delete_listener_safe(self, request, context): adrfam = GatewayEnumUtils.get_key_from_value(pb2.AddressFamily, request.adrfam) if adrfam == None: - errmsg=f"{delete_listener_error_prefix}. Unknown address family {request.adrfam}" + errmsg = f"{delete_listener_error_prefix}. Unknown address family {request.adrfam}" self.logger.error(errmsg) return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) @@ -3249,19 +4062,23 @@ def delete_listener_safe(self, request, context): peer_msg = self.get_peer_message(context) force_msg = " forcefully" if request.force else "" - host_msg = "all hosts" if request.host_name == "*" else f"host {request.host_name}" + host_msg = ( + "all hosts" if request.host_name == "*" else f"host {request.host_name}" + ) - self.logger.info(f"Received request to delete TCP listener of {host_msg}" - f" for subsystem {request.nqn} at" - f" {esc_traddr}:{request.trsvcid}{force_msg}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to delete TCP listener of {host_msg}" + f" for subsystem {request.nqn} at" + f" {esc_traddr}:{request.trsvcid}{force_msg}, context: {context}{peer_msg}" + ) if request.host_name == "*" and not request.force: - errmsg=f"{delete_listener_error_prefix}. Must use the \"--force\" parameter when setting the host name to \"*\"." + errmsg = f'{delete_listener_error_prefix}. Must use the "--force" parameter when setting the host name to "*".' self.logger.error(errmsg) return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.nqn): - errmsg=f"{delete_listener_error_prefix}. Can't delete a listener from a discovery subsystem" + errmsg = f"{delete_listener_error_prefix}. Can't delete a listener from a discovery subsystem" self.logger.error(errmsg) return pb2.req_status(status=errno.EINVAL, error_message=errmsg) @@ -3269,7 +4086,7 @@ def delete_listener_safe(self, request, context): list_conn_req = pb2.list_connections_req(subsystem=request.nqn) list_conn_ret = self.list_connections_safe(list_conn_req, context) if list_conn_ret.status != 0: - errmsg=f"{delete_listener_error_prefix}. Can't verify there are no active connections for this address" + errmsg = f"{delete_listener_error_prefix}. Can't verify there are no active connections for this address" self.logger.error(errmsg) return pb2.req_status(status=errno.ENOTEMPTY, error_message=errmsg) for conn in list_conn_ret.connections: @@ -3279,7 +4096,7 @@ def delete_listener_safe(self, request, context): continue if conn.trsvcid != request.trsvcid: continue - errmsg=f"{delete_listener_error_prefix} due to active connections for {esc_traddr}:{request.trsvcid}. Deleting the listener terminates active connections. You can continue to delete the listener by adding the `--force` parameter." + errmsg = f"{delete_listener_error_prefix} due to active connections for {esc_traddr}:{request.trsvcid}. Deleting the listener terminates active connections. You can continue to delete the listener by adding the `--force` parameter." self.logger.error(errmsg) return pb2.req_status(status=errno.ENOTEMPTY, error_message=errmsg) @@ -3297,12 +4114,26 @@ def delete_listener_safe(self, request, context): ) self.logger.debug(f"delete_listener: {ret}") if request.nqn in self.subsystem_listeners: - if (adrfam, traddr, request.trsvcid, False) in self.subsystem_listeners[request.nqn]: - self.subsystem_listeners[request.nqn].remove((adrfam, traddr, request.trsvcid, False)) - if (adrfam, traddr, request.trsvcid, True) in self.subsystem_listeners[request.nqn]: - self.subsystem_listeners[request.nqn].remove((adrfam, traddr, request.trsvcid, True)) + if ( + adrfam, + traddr, + request.trsvcid, + False, + ) in self.subsystem_listeners[request.nqn]: + self.subsystem_listeners[request.nqn].remove( + (adrfam, traddr, request.trsvcid, False) + ) + if ( + adrfam, + traddr, + request.trsvcid, + True, + ) in self.subsystem_listeners[request.nqn]: + self.subsystem_listeners[request.nqn].remove( + (adrfam, traddr, request.trsvcid, True) + ) else: - errmsg=f"{delete_listener_error_prefix}. Gateway's host name must match current host ({self.host_name}). You can continue to delete the listener by adding the `--force` parameter." + errmsg = f"{delete_listener_error_prefix}. Gateway's host name must match current host ({self.host_name}). You can continue to delete the listener by adding the `--force` parameter." self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.ENOENT, error_message=errmsg) except Exception as ex: @@ -3310,8 +4141,9 @@ def delete_listener_safe(self, request, context): # It's OK for SPDK to fail in case we used a different host name, just continue to remove from OMAP if request.host_name == self.host_name: errmsg = f"{delete_listener_error_prefix}:\n{ex}" - self.remove_listener_from_state(request.nqn, request.host_name, - traddr, request.trsvcid, context) + self.remove_listener_from_state( + request.nqn, request.host_name, traddr, request.trsvcid, context + ) resp = self.parse_json_exeption(ex) status = errno.EINVAL if resp: @@ -3323,12 +4155,16 @@ def delete_listener_safe(self, request, context): # Just in case SPDK failed with no exception if not ret: self.logger.error(delete_listener_error_prefix) - self.remove_listener_from_state(request.nqn, request.host_name, - traddr, request.trsvcid, context) - return pb2.req_status(status=errno.EINVAL, error_message=delete_listener_error_prefix) + self.remove_listener_from_state( + request.nqn, request.host_name, traddr, request.trsvcid, context + ) + return pb2.req_status( + status=errno.EINVAL, error_message=delete_listener_error_prefix + ) - return self.remove_listener_from_state(request.nqn, request.host_name, - traddr, request.trsvcid, context) + return self.remove_listener_from_state( + request.nqn, request.host_name, traddr, request.trsvcid, context + ) def delete_listener(self, request, context=None): return self.execute_grpc_function(self.delete_listener_safe, request, context) @@ -3337,13 +4173,17 @@ def list_listeners_safe(self, request, context): """List listeners.""" peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to list listeners for {request.subsystem}, context: {context}{peer_msg}") + self.logger.info( + f"Received request to list listeners for {request.subsystem}, context: {context}{peer_msg}" + ) listeners = [] omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: state = self.gateway_state.local.get_state() - listener_prefix = GatewayState.build_partial_listener_key(request.subsystem, None) + listener_prefix = GatewayState.build_partial_listener_key( + request.subsystem, None + ) for key, val in state.items(): if not key.startswith(listener_prefix): continue @@ -3351,23 +4191,29 @@ def list_listeners_safe(self, request, context): listener = json.loads(val) nqn = listener["nqn"] if nqn != request.subsystem: - self.logger.warning(f"Got subsystem {nqn} instead of {request.subsystem}, ignore") + self.logger.warning( + f"Got subsystem {nqn} instead of {request.subsystem}, ignore" + ) continue secure = False if "secure" in listener: secure = listener["secure"] - one_listener = pb2.listener_info(host_name = listener["host_name"], - trtype = "TCP", - adrfam = listener["adrfam"], - traddr = listener["traddr"], - trsvcid = listener["trsvcid"], - secure = secure) + one_listener = pb2.listener_info( + host_name=listener["host_name"], + trtype="TCP", + adrfam=listener["adrfam"], + traddr=listener["traddr"], + trsvcid=listener["trsvcid"], + secure=secure, + ) listeners.append(one_listener) except Exception: self.logger.exception(f"Got exception while parsing {val}") continue - return pb2.listeners_info(status = 0, error_message = os.strerror(0), listeners=listeners) + return pb2.listeners_info( + status=0, error_message=os.strerror(0), listeners=listeners + ) def list_listeners(self, request, context=None): return self.execute_grpc_function(self.list_listeners_safe, request, context) @@ -3378,17 +4224,28 @@ def list_subsystems_safe(self, request, context): peer_msg = self.get_peer_message(context) log_level = logging.INFO if context else logging.DEBUG if request.subsystem_nqn: - self.logger.log(log_level, f"Received request to list subsystem {request.subsystem_nqn}, context: {context}{peer_msg}") + self.logger.log( + log_level, + f"Received request to list subsystem {request.subsystem_nqn}, context: {context}{peer_msg}", + ) else: if request.serial_number: - self.logger.log(log_level, f"Received request to list the subsystem with serial number {request.serial_number}, context: {context}{peer_msg}") + self.logger.log( + log_level, + f"Received request to list the subsystem with serial number {request.serial_number}, context: {context}{peer_msg}", + ) else: - self.logger.log(log_level, f"Received request to list all subsystems, context: {context}{peer_msg}") + self.logger.log( + log_level, + f"Received request to list all subsystems, context: {context}{peer_msg}", + ) subsystems = [] try: if request.subsystem_nqn: - ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client, nqn=request.subsystem_nqn) + ret = rpc_nvmf.nvmf_get_subsystems( + self.spdk_rpc_client, nqn=request.subsystem_nqn + ) else: ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_client) self.logger.debug(f"list_subsystems: {ret}") @@ -3401,7 +4258,9 @@ def list_subsystems_safe(self, request, context): if resp: status = resp["code"] errmsg = f"Failure listing subsystems: {resp['message']}" - return pb2.subsystems_info_cli(status=status, error_message=errmsg, subsystems=[]) + return pb2.subsystems_info_cli( + status=status, error_message=errmsg, subsystems=[] + ) for s in ret: try: @@ -3414,7 +4273,9 @@ def list_subsystems_safe(self, request, context): self.subsystem_nsid_bdev_and_uuid.remove_namespace(s["nqn"]) s["namespace_count"] = ns_count s["enable_ha"] = True - s["has_dhchap_key"] = self.host_info.does_subsystem_have_dhchap_key(s["nqn"]) + s["has_dhchap_key"] = self.host_info.does_subsystem_have_dhchap_key( + s["nqn"] + ) else: s["namespace_count"] = 0 s["enable_ha"] = False @@ -3427,13 +4288,17 @@ def list_subsystems_safe(self, request, context): self.logger.exception(f"{s=} parse error") pass - return pb2.subsystems_info_cli(status = 0, error_message = os.strerror(0), subsystems=subsystems) + return pb2.subsystems_info_cli( + status=0, error_message=os.strerror(0), subsystems=subsystems + ) def get_subsystems_safe(self, request, context): """Gets subsystems.""" peer_msg = self.get_peer_message(context) - self.logger.debug(f"Received request to get subsystems, context: {context}{peer_msg}") + self.logger.debug( + f"Received request to get subsystems, context: {context}{peer_msg}" + ) subsystems = [] try: ret = rpc_nvmf.nvmf_get_subsystems(self.spdk_rpc_subsystems_client) @@ -3445,7 +4310,9 @@ def get_subsystems_safe(self, request, context): for s in ret: try: - s["has_dhchap_key"] = self.host_info.does_subsystem_have_dhchap_key(s["nqn"]) + s["has_dhchap_key"] = self.host_info.does_subsystem_have_dhchap_key( + s["nqn"] + ) ns_key = "namespaces" if ns_key in s: for n in s[ns_key]: @@ -3453,7 +4320,9 @@ def get_subsystems_safe(self, request, context): with self.shared_state_lock: nonce = self.cluster_nonce[self.bdev_cluster[bdev]] n["nonce"] = nonce - find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace(s["nqn"], n["nsid"]) + find_ret = self.subsystem_nsid_bdev_and_uuid.find_namespace( + s["nqn"], n["nsid"] + ) n["no_auto_visible"] = find_ret.no_auto_visible n["hosts"] = find_ret.host_list # Parse the JSON dictionary into the protobuf message @@ -3476,26 +4345,29 @@ def list_subsystems(self, request, context=None): def change_subsystem_key_safe(self, request, context): """Change subsystem key.""" peer_msg = self.get_peer_message(context) - failure_prefix=f"Failure changing DH-HMAC-CHAP key for subsystem {request.subsystem_nqn}" + failure_prefix = ( + f"Failure changing DH-HMAC-CHAP key for subsystem {request.subsystem_nqn}" + ) self.logger.info( - f"Received request to change inband authentication key for subsystem {request.subsystem_nqn}, dhchap: {request.dhchap_key}, context: {context}{peer_msg}") + f"Received request to change inband authentication key for subsystem {request.subsystem_nqn}, dhchap: {request.dhchap_key}, context: {context}{peer_msg}" + ) if not GatewayState.is_key_element_valid(request.subsystem_nqn): - errmsg = f"{failure_prefix}: Invalid subsystem NQN \"{request.subsystem_nqn}\", contains invalid characters" + errmsg = f'{failure_prefix}: Invalid subsystem NQN "{request.subsystem_nqn}", contains invalid characters' self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) if self.verify_nqns: rc = GatewayUtils.is_valid_nqn(request.subsystem_nqn) if rc[0] != 0: errmsg = f"{failure_prefix}: {rc[1]}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = rc[0], error_message = errmsg) + return pb2.req_status(status=rc[0], error_message=errmsg) if GatewayUtils.is_discovery_nqn(request.subsystem_nqn): errmsg = f"{failure_prefix}: Can't change DH-HMAC-CHAP key for a discovery subsystem" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status(status=errno.EINVAL, error_message=errmsg) omap_lock = self.omap_lock.get_omap_lock_to_use(context) with omap_lock: @@ -3509,10 +4381,14 @@ def change_subsystem_key_safe(self, request, context): all_subsys_hosts = self.get_subsystem_hosts(request.subsystem_nqn) for hostnqn in all_subsys_hosts: assert hostnqn, "Shouldn't get an empty host NQN" - if not self.host_info.is_dhchap_host(request.subsystem_nqn, hostnqn): + if not self.host_info.is_dhchap_host( + request.subsystem_nqn, hostnqn + ): errmsg = f"{failure_prefix}: Can't set a subsystem's DH-HMAC-CHAP key when it has hosts with no key, like host {hostnqn}" self.logger.error(f"{errmsg}") - return pb2.req_status(status = errno.EINVAL, error_message = errmsg) + return pb2.req_status( + status=errno.EINVAL, error_message=errmsg + ) subsys_key = GatewayState.build_subsystem_key(request.subsystem_nqn) try: @@ -3523,16 +4399,23 @@ def change_subsystem_key_safe(self, request, context): self.logger.error(errmsg) return pb2.req_status(status=errno.ENOENT, error_message=errmsg) - assert subsys_entry, f"Can't find entry for subsystem {request.subsystem_nqn}" + assert ( + subsys_entry + ), f"Can't find entry for subsystem {request.subsystem_nqn}" try: - create_req = pb2.create_subsystem_req(subsystem_nqn=request.subsystem_nqn, - serial_number=subsys_entry["serial_number"], - max_namespaces=subsys_entry["max_namespaces"], - enable_ha=subsys_entry["enable_ha"], - no_group_append=subsys_entry["no_group_append"], - dhchap_key=request.dhchap_key) + create_req = pb2.create_subsystem_req( + subsystem_nqn=request.subsystem_nqn, + serial_number=subsys_entry["serial_number"], + max_namespaces=subsys_entry["max_namespaces"], + enable_ha=subsys_entry["enable_ha"], + no_group_append=subsys_entry["no_group_append"], + dhchap_key=request.dhchap_key, + ) json_req = json_format.MessageToJson( - create_req, preserving_proto_field_name=True, including_default_value_fields=True) + create_req, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) self.gateway_state.add_subsystem(request.subsystem_nqn, json_req) except Exception as ex: errmsg = f"Error persisting subsystem key change for {request.subsystem_nqn}" @@ -3544,41 +4427,53 @@ def change_subsystem_key_safe(self, request, context): # We need to change the subsystem key before calling the host change key functions, so the new subsystem key will be used # As we change the list now, we have to use a copy having the old values if request.dhchap_key: - self.host_info.add_dhchap_key_to_subsystem(request.subsystem_nqn, request.dhchap_key) + self.host_info.add_dhchap_key_to_subsystem( + request.subsystem_nqn, request.dhchap_key + ) else: self.host_info.remove_dhchap_key_from_subsystem(request.subsystem_nqn) for hnqn in hosts.keys(): - change_req = pb2.change_host_key_req(subsystem_nqn=request.subsystem_nqn, - host_nqn=hnqn, - dhchap_key=hosts[hnqn]) + change_req = pb2.change_host_key_req( + subsystem_nqn=request.subsystem_nqn, + host_nqn=hnqn, + dhchap_key=hosts[hnqn], + ) try: self.change_host_key_safe(change_req, context) except Excpetion: pass - return pb2.req_status(status=0, error_message=os.strerror(0)) def change_subsystem_key(self, request, context=None): """Change subsystem key.""" - return self.execute_grpc_function(self.change_subsystem_key_safe, request, context) + return self.execute_grpc_function( + self.change_subsystem_key_safe, request, context + ) def get_spdk_nvmf_log_flags_and_level_safe(self, request, context): """Gets spdk nvmf log flags, log level and log print level""" peer_msg = self.get_peer_message(context) - self.logger.info(f"Received request to get SPDK nvmf log flags and level{peer_msg}") + self.logger.info( + f"Received request to get SPDK nvmf log flags and level{peer_msg}" + ) log_flags = [] try: - nvmf_log_flags = {key: value for key, value in rpc_log.log_get_flags( - self.spdk_rpc_client).items() if key.startswith('nvmf')} + nvmf_log_flags = { + key: value + for key, value in rpc_log.log_get_flags(self.spdk_rpc_client).items() + if key.startswith("nvmf") + } for flag, flagvalue in nvmf_log_flags.items(): - pb2_log_flag = pb2.spdk_log_flag_info(name = flag, enabled = flagvalue) + pb2_log_flag = pb2.spdk_log_flag_info(name=flag, enabled=flagvalue) log_flags.append(pb2_log_flag) spdk_log_level = rpc_log.log_get_level(self.spdk_rpc_client) spdk_log_print_level = rpc_log.log_get_print_level(self.spdk_rpc_client) - self.logger.debug(f"spdk log flags: {nvmf_log_flags}, " - f"spdk log level: {spdk_log_level}, " - f"spdk log print level: {spdk_log_print_level}") + self.logger.debug( + f"spdk log flags: {nvmf_log_flags}, " + f"spdk log level: {spdk_log_level}, " + f"spdk log print level: {spdk_log_print_level}" + ) except Exception as ex: errmsg = f"Failure getting SPDK log levels and nvmf log flags" self.logger.exception(errmsg) @@ -3588,17 +4483,22 @@ def get_spdk_nvmf_log_flags_and_level_safe(self, request, context): if resp: status = resp["code"] errmsg = f"Failure getting SPDK log levels and nvmf log flags: {resp['message']}" - return pb2.spdk_nvmf_log_flags_and_level_info(status = status, error_message = errmsg) + return pb2.spdk_nvmf_log_flags_and_level_info( + status=status, error_message=errmsg + ) return pb2.spdk_nvmf_log_flags_and_level_info( nvmf_log_flags=log_flags, - log_level = spdk_log_level, - log_print_level = spdk_log_print_level, - status = 0, - error_message = os.strerror(0)) + log_level=spdk_log_level, + log_print_level=spdk_log_print_level, + status=0, + error_message=os.strerror(0), + ) def get_spdk_nvmf_log_flags_and_level(self, request, context=None): - return self.execute_grpc_function(self.get_spdk_nvmf_log_flags_and_level_safe, request, context) + return self.execute_grpc_function( + self.get_spdk_nvmf_log_flags_and_level_safe, request, context + ) def set_spdk_nvmf_logs_safe(self, request, context): """Enables spdk nvmf logs""" @@ -3609,37 +4509,52 @@ def set_spdk_nvmf_logs_safe(self, request, context): peer_msg = self.get_peer_message(context) if request.HasField("log_level"): - log_level = GatewayEnumUtils.get_key_from_value(pb2.LogLevel, request.log_level) + log_level = GatewayEnumUtils.get_key_from_value( + pb2.LogLevel, request.log_level + ) if log_level == None: - errmsg=f"Unknown log level {request.log_level}" + errmsg = f"Unknown log level {request.log_level}" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) if request.HasField("print_level"): - print_level = GatewayEnumUtils.get_key_from_value(pb2.LogLevel, request.print_level) + print_level = GatewayEnumUtils.get_key_from_value( + pb2.LogLevel, request.print_level + ) if print_level == None: - errmsg=f"Unknown print level {request.print_level}" + errmsg = f"Unknown print level {request.print_level}" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) - self.logger.info(f"Received request to set SPDK nvmf logs: log_level: {log_level}, print_level: {print_level}{peer_msg}") + self.logger.info( + f"Received request to set SPDK nvmf logs: log_level: {log_level}, print_level: {print_level}{peer_msg}" + ) try: - nvmf_log_flags = [key for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() if key.startswith('nvmf')] - ret = [rpc_log.log_set_flag( - self.spdk_rpc_client, flag=flag) for flag in nvmf_log_flags] - self.logger.debug(f"Set SPDK nvmf log flags {nvmf_log_flags} to TRUE: {ret}") + nvmf_log_flags = [ + key + for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() + if key.startswith("nvmf") + ] + ret = [ + rpc_log.log_set_flag(self.spdk_rpc_client, flag=flag) + for flag in nvmf_log_flags + ] + self.logger.debug( + f"Set SPDK nvmf log flags {nvmf_log_flags} to TRUE: {ret}" + ) if log_level != None: ret_log = rpc_log.log_set_level(self.spdk_rpc_client, level=log_level) self.logger.debug(f"Set log level to {log_level}: {ret_log}") if print_level != None: ret_print = rpc_log.log_set_print_level( - self.spdk_rpc_client, level=print_level) + self.spdk_rpc_client, level=print_level + ) self.logger.debug(f"Set log print level to {print_level}: {ret_print}") except Exception as ex: - errmsg="Failure setting SPDK log levels" + errmsg = "Failure setting SPDK log levels" self.logger.exception(errmsg) - errmsg="{errmsg}:\n{ex}" + errmsg = "{errmsg}:\n{ex}" for flag in nvmf_log_flags: rpc_log.log_clear_flag(self.spdk_rpc_client, flag=flag) resp = self.parse_json_exeption(ex) @@ -3663,7 +4578,9 @@ def set_spdk_nvmf_logs_safe(self, request, context): return pb2.req_status(status=status, error_message=errmsg) def set_spdk_nvmf_logs(self, request, context=None): - return self.execute_grpc_function(self.set_spdk_nvmf_logs_safe, request, context) + return self.execute_grpc_function( + self.set_spdk_nvmf_logs_safe, request, context + ) def disable_spdk_nvmf_logs_safe(self, request, context): """Disables spdk nvmf logs""" @@ -3671,10 +4588,19 @@ def disable_spdk_nvmf_logs_safe(self, request, context): self.logger.info(f"Received request to disable SPDK nvmf logs{peer_msg}") try: - nvmf_log_flags = [key for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() if key.startswith('nvmf')] - ret = [rpc_log.log_clear_flag(self.spdk_rpc_client, flag=flag) for flag in nvmf_log_flags] - logs_level = [rpc_log.log_set_level(self.spdk_rpc_client, level='NOTICE'), - rpc_log.log_set_print_level(self.spdk_rpc_client, level='INFO')] + nvmf_log_flags = [ + key + for key in rpc_log.log_get_flags(self.spdk_rpc_client).keys() + if key.startswith("nvmf") + ] + ret = [ + rpc_log.log_clear_flag(self.spdk_rpc_client, flag=flag) + for flag in nvmf_log_flags + ] + logs_level = [ + rpc_log.log_set_level(self.spdk_rpc_client, level="NOTICE"), + rpc_log.log_set_print_level(self.spdk_rpc_client, level="INFO"), + ] ret.extend(logs_level) except Exception as ex: errmsg = f"Failure in disable SPDK nvmf log flags" @@ -3695,7 +4621,9 @@ def disable_spdk_nvmf_logs_safe(self, request, context): return pb2.req_status(status=status, error_message=errmsg) def disable_spdk_nvmf_logs(self, request, context=None): - return self.execute_grpc_function(self.disable_spdk_nvmf_logs_safe, request, context) + return self.execute_grpc_function( + self.disable_spdk_nvmf_logs_safe, request, context + ) def parse_version(self, version): if not version: @@ -3708,7 +4636,7 @@ def parse_version(self, version): v2 = int(vlist[1]) v3 = int(vlist[2]) except Exception: - self.logger.exception(f"Can't parse version \"{version}\"") + self.logger.exception(f'Can\'t parse version "{version}"') return None return (v1, v2, v3) @@ -3722,21 +4650,23 @@ def get_gateway_info_safe(self, request, context): cli_version_string = request.cli_version addr = self.config.get_with_default("gateway", "addr", "") port = self.config.get_with_default("gateway", "port", "") - ret = pb2.gateway_info(cli_version = request.cli_version, - version = gw_version_string, - spdk_version = spdk_version_string, - name = self.gateway_name, - group = self.gateway_group, - addr = addr, - port = port, - load_balancing_group = self.group_id + 1, - bool_status = True, - hostname = self.host_name, - max_subsystems = self.max_subsystems, - max_namespaces = self.max_namespaces, - max_hosts_per_subsystem = self.max_hosts_per_subsystem, - status = 0, - error_message = os.strerror(0)) + ret = pb2.gateway_info( + cli_version=request.cli_version, + version=gw_version_string, + spdk_version=spdk_version_string, + name=self.gateway_name, + group=self.gateway_group, + addr=addr, + port=port, + load_balancing_group=self.group_id + 1, + bool_status=True, + hostname=self.host_name, + max_subsystems=self.max_subsystems, + max_namespaces=self.max_namespaces, + max_hosts_per_subsystem=self.max_hosts_per_subsystem, + status=0, + error_message=os.strerror(0), + ) cli_ver = self.parse_version(cli_version_string) gw_ver = self.parse_version(gw_version_string) if cli_ver != None and gw_ver != None and cli_ver < gw_ver: @@ -3752,9 +4682,13 @@ def get_gateway_info_safe(self, request, context): ret.status = errno.EINVAL ret.error_message = f"Invalid gateway's version {gw_version_string}" if not cli_version_string: - self.logger.warning(f"No CLI version specified, can't check version compatibility") + self.logger.warning( + f"No CLI version specified, can't check version compatibility" + ) elif not cli_ver: - self.logger.warning(f"Invalid CLI version {cli_version_string}, can't check version compatibility") + self.logger.warning( + f"Invalid CLI version {cli_version_string}, can't check version compatibility" + ) if ret.status == 0: log_func = self.logger.debug else: @@ -3770,26 +4704,39 @@ def get_gateway_log_level(self, request, context=None): """Get gateway's log level""" peer_msg = self.get_peer_message(context) try: - log_level = GatewayEnumUtils.get_key_from_value(pb2.GwLogLevel, self.logger.level) + log_level = GatewayEnumUtils.get_key_from_value( + pb2.GwLogLevel, self.logger.level + ) except Exception: - self.logger.exception(f"Can't get string value for log level {self.logger.level}") - return pb2.gateway_log_level_info(status = errno.ENOKEY, - error_message=f"Invalid gateway log level") - self.logger.info(f"Received request to get gateway's log level. Level is {log_level}{peer_msg}") - return pb2.gateway_log_level_info(status = 0, error_message=os.strerror(0), log_level=log_level) + self.logger.exception( + f"Can't get string value for log level {self.logger.level}" + ) + return pb2.gateway_log_level_info( + status=errno.ENOKEY, error_message=f"Invalid gateway log level" + ) + self.logger.info( + f"Received request to get gateway's log level. Level is {log_level}{peer_msg}" + ) + return pb2.gateway_log_level_info( + status=0, error_message=os.strerror(0), log_level=log_level + ) def set_gateway_log_level(self, request, context=None): """Set gateway's log level""" peer_msg = self.get_peer_message(context) - log_level = GatewayEnumUtils.get_key_from_value(pb2.GwLogLevel, request.log_level) + log_level = GatewayEnumUtils.get_key_from_value( + pb2.GwLogLevel, request.log_level + ) if log_level == None: - errmsg=f"Unknown log level {request.log_level}" + errmsg = f"Unknown log level {request.log_level}" self.logger.error(f"{errmsg}") return pb2.req_status(status=errno.ENOKEY, error_message=errmsg) log_level = log_level.upper() - self.logger.info(f"Received request to set gateway's log level to {log_level}{peer_msg}") + self.logger.info( + f"Received request to set gateway's log level to {log_level}{peer_msg}" + ) self.gw_logger_object.set_log_level(request.log_level) try: @@ -3797,12 +4744,16 @@ def set_gateway_log_level(self, request, context=None): except FileNotFoundError: pass except Exception: - self.logger.exception(f"Failure removing \"{GatewayLogger.NVME_GATEWAY_LOG_LEVEL_FILE_PATH}\"") + self.logger.exception( + f'Failure removing "{GatewayLogger.NVME_GATEWAY_LOG_LEVEL_FILE_PATH}"' + ) try: with open(GatewayLogger.NVME_GATEWAY_LOG_LEVEL_FILE_PATH, "w") as f: f.write(str(request.log_level)) except Exception: - self.logger.exception(f"Failure writing log level to \"{GatewayLogger.NVME_GATEWAY_LOG_LEVEL_FILE_PATH}\"") + self.logger.exception( + f'Failure writing log level to "{GatewayLogger.NVME_GATEWAY_LOG_LEVEL_FILE_PATH}"' + ) return pb2.req_status(status=0, error_message=os.strerror(0)) diff --git a/control/prometheus.py b/control/prometheus.py index 106ead59..9b050759 100644 --- a/control/prometheus.py +++ b/control/prometheus.py @@ -14,17 +14,25 @@ import spdk.rpc as rpc from .proto import gateway_pb2 as pb2 -from prometheus_client.core import REGISTRY, GaugeMetricFamily, CounterMetricFamily, InfoMetricFamily +from prometheus_client.core import ( + REGISTRY, + GaugeMetricFamily, + CounterMetricFamily, + InfoMetricFamily, +) from prometheus_client import start_http_server, GC_COLLECTOR from typing import NamedTuple from functools import wraps from .utils import NICS -COLLECTION_ELAPSED_WARNING = 0.8 # Percentage of the refresh interval before a warning message is issued +COLLECTION_ELAPSED_WARNING = ( + 0.8 # Percentage of the refresh interval before a warning message is issued +) REGISTRY.unregister(GC_COLLECTOR) # Turn off garbage collector metrics logger = None + class RBD(NamedTuple): pool: str namespace: str @@ -44,9 +52,10 @@ def wrapped(self, *args, **kwargs): self.metrics_cache.append(metric) yield metric else: - logger.debug('Returning content from cache') + logger.debug("Returning content from cache") for metric in self.metrics_cache: yield metric + return wrapped @@ -56,9 +65,10 @@ def call(self, *args, **kwargs): st = time.time() result = method(self, *args, **kwargs) elapsed = time.time() - st - if hasattr(self, 'method_timings'): + if hasattr(self, "method_timings"): self.method_timings[method.__name__] = elapsed return result + return call @@ -79,23 +89,29 @@ def start_exporter(spdk_rpc_client, config, gateway_rpc, logger_to_use): logger = logger_to_use port = config.getint_with_default("gateway", "prometheus_port", 10008) ssl = config.getboolean_with_default("gateway", "prometheus_exporter_ssl", True) - mode = 'https' if ssl else 'http' + mode = "https" if ssl else "http" if ssl: - cert_filepath = config.get('mtls', 'server_cert') - key_filepath = config.get('mtls', 'server_key') + cert_filepath = config.get("mtls", "server_cert") + key_filepath = config.get("mtls", "server_key") if os.path.exists(cert_filepath) and os.path.exists(key_filepath): - httpd_ok = start_httpd(port=port, certfile=cert_filepath, keyfile=key_filepath) + httpd_ok = start_httpd( + port=port, certfile=cert_filepath, keyfile=key_filepath + ) else: httpd_ok = False - logger.error("Unable to start prometheus exporter - missing cert/key file(s)") + logger.error( + "Unable to start prometheus exporter - missing cert/key file(s)" + ) else: # SSL mode explicitly disabled by config option httpd_ok = start_httpd(port=port) if httpd_ok: - logger.info(f"Prometheus exporter running in {mode} mode, listening on port {port}") + logger.info( + f"Prometheus exporter running in {mode} mode, listening on port {port}" + ) REGISTRY.register(NVMeOFCollector(spdk_rpc_client, config, gateway_rpc)) @@ -107,11 +123,13 @@ def __init__(self, spdk_rpc_client, config, gateway_rpc): self.gateway_rpc = gateway_rpc self.metric_prefix = "ceph_nvmeof" self.gw_config = config - _bdev_pools = config.get_with_default('gateway', 'prometheus_bdev_pools', '') - self.bdev_pools = _bdev_pools.split(',') if _bdev_pools else [] - self.interval = config.getint_with_default('gateway', 'prometheus_stats_inteval', 10) + _bdev_pools = config.get_with_default("gateway", "prometheus_bdev_pools", "") + self.bdev_pools = _bdev_pools.split(",") if _bdev_pools else [] + self.interval = config.getint_with_default( + "gateway", "prometheus_stats_inteval", 10 + ) self.lock = threading.Lock() - self.hostname = os.getenv('NODE_NAME') or os.getenv('HOSTNAME') + self.hostname = os.getenv("NODE_NAME") or os.getenv("HOSTNAME") # gw metadata is static, so fetch the data only at startup self.gw_metadata = self._get_gw_metadata() # proto.gateway_pb2.gateway_info @@ -126,7 +144,9 @@ def __init__(self, spdk_rpc_client, config, gateway_rpc): self.method_timings = {} if self.bdev_pools: - logger.info(f"Stats restricted to bdevs in the following pool(s): {','.join(self.bdev_pools)}") + logger.info( + f"Stats restricted to bdevs in the following pool(s): {','.join(self.bdev_pools)}" + ) else: logger.info("Stats for all bdevs will be provided") @@ -178,7 +198,9 @@ def _get_spdk_thread_stats(self): @timer def _get_subsystems(self): """Fetch aggregated subsystem information""" - subsystems_info = self.gateway_rpc.get_subsystems(pb2.get_subsystems_req(), None) + subsystems_info = self.gateway_rpc.get_subsystems( + pb2.get_subsystems_req(), None + ) return subsystems_info.subsystems @timer @@ -186,7 +208,9 @@ def _list_subsystems(self): """Fetch abbreviated subsystem information used by the CLI""" resp = self.gateway_rpc.list_subsystems(pb2.list_subsystems_req()) if resp.status != 0: - logger.error(f"Exporter failed to execute list_subsystems: {resp.error_message}") + logger.error( + f"Exporter failed to execute list_subsystems: {resp.error_message}" + ) return {} return {subsys.nqn: subsys for subsys in resp.subsystems} @@ -196,9 +220,13 @@ def _get_connection_map(self, subsystem_list): """Fetch connection information for all defined subsystems""" connection_map = {} for subsys in subsystem_list: - resp = self.gateway_rpc.list_connections(pb2.list_connections_req(subsystem=subsys.nqn)) + resp = self.gateway_rpc.list_connections( + pb2.list_connections_req(subsystem=subsys.nqn) + ) if resp.status != 0: - logger.error(f"Exporter failed to fetch connection info for {subsys.nqn}: {resp.error_message}") + logger.error( + f"Exporter failed to fetch connection info for {subsys.nqn}: {resp.error_message}" + ) continue connection_map[subsys.nqn] = resp return connection_map @@ -228,7 +256,9 @@ def collect(self): if elapsed > self.interval: logger.error(f"Stats refresh time > interval time of {self.interval} secs") elif elapsed > self.interval * COLLECTION_ELAPSED_WARNING: - logger.warning(f"Stats refresh of {elapsed:.2f}s is close to exceeding the interval {self.interval}s") + logger.warning( + f"Stats refresh of {elapsed:.2f}s is close to exceeding the interval {self.interval}s" + ) else: logger.debug(f"Stats refresh completed in {elapsed:.3f} secs.") @@ -236,47 +266,58 @@ def collect(self): f"{self.metric_prefix}_gateway", "Gateway information", value={ - 'spdk_version': self.gw_metadata.spdk_version, - 'version': self.gw_metadata.version, - 'addr': self.gw_metadata.addr, - 'port': self.gw_metadata.port, - 'name': self.gw_metadata.name, - 'hostname': self.hostname, - 'group': self.gw_metadata.group - }) + "spdk_version": self.gw_metadata.spdk_version, + "version": self.gw_metadata.version, + "addr": self.gw_metadata.addr, + "port": self.gw_metadata.port, + "name": self.gw_metadata.name, + "hostname": self.hostname, + "group": self.gw_metadata.group, + }, + ) yield gateway_info bdev_metadata = GaugeMetricFamily( f"{self.metric_prefix}_bdev_metadata", "BDEV Metadata", - labels=["bdev_name", "pool_name", "namespace", "rbd_name", "block_size"]) + labels=["bdev_name", "pool_name", "namespace", "rbd_name", "block_size"], + ) bdev_capacity = GaugeMetricFamily( f"{self.metric_prefix}_bdev_capacity_bytes", "BDEV Capacity", - labels=["bdev_name"]) + labels=["bdev_name"], + ) for bdev in self.bdev_info: - bdev_name = bdev.get('name') + bdev_name = bdev.get("name") try: rbd_info = bdev["driver_specific"]["rbd"] except KeyError: - logger.debug(f"no rbd information present for bdev {bdev.get('name')}, skipping") + logger.debug( + f"no rbd information present for bdev {bdev.get('name')}, skipping" + ) continue - rbd_pool = rbd_info.get('pool_name') - rbd_namespace = rbd_info.get('namespace', '') # namespace is not currently present - rbd_image = rbd_info.get('rbd_name') + rbd_pool = rbd_info.get("pool_name") + rbd_namespace = rbd_info.get( + "namespace", "" + ) # namespace is not currently present + rbd_image = rbd_info.get("rbd_name") if self.bdev_pools: if rbd_pool not in self.bdev_pools: continue bdev_lookup[bdev_name] = RBD(rbd_pool, rbd_namespace, rbd_image) - bdev_metadata.add_metric([ - bdev_name, - rbd_pool, - rbd_namespace, - rbd_image, - str(bdev.get('block_size'))], 1) + bdev_metadata.add_metric( + [ + bdev_name, + rbd_pool, + rbd_namespace, + rbd_image, + str(bdev.get("block_size")), + ], + 1, + ) bdev_size = bdev.get("block_size") * bdev.get("num_blocks") bdev_capacity.add_metric([bdev.get("name")], bdev_size) @@ -288,32 +329,40 @@ def collect(self): bdev_read_ops = CounterMetricFamily( f"{self.metric_prefix}_bdev_reads_completed_total", "Total number of read operations completed", - labels=["bdev_name"]) + labels=["bdev_name"], + ) bdev_write_ops = CounterMetricFamily( f"{self.metric_prefix}_bdev_writes_completed_total", "Total number of write operations completed", - labels=["bdev_name"]) + labels=["bdev_name"], + ) bdev_read_bytes = CounterMetricFamily( f"{self.metric_prefix}_bdev_read_bytes_total", "Total number of bytes read successfully", - labels=["bdev_name"]) + labels=["bdev_name"], + ) bdev_write_bytes = CounterMetricFamily( f"{self.metric_prefix}_bdev_written_bytes_total", "Total number of bytes written successfully", - labels=["bdev_name"]) + labels=["bdev_name"], + ) bdev_read_seconds = CounterMetricFamily( f"{self.metric_prefix}_bdev_read_seconds_total", "Total time spent servicing READ I/O", - labels=["bdev_name"]) + labels=["bdev_name"], + ) bdev_write_seconds = CounterMetricFamily( f"{self.metric_prefix}_bdev_write_seconds_total", "Total time spent servicing WRITE I/O", - labels=["bdev_name"]) + labels=["bdev_name"], + ) for bdev in self.bdev_io_stats.get("bdevs", []): - bdev_name = bdev.get('name') + bdev_name = bdev.get("name") if bdev_name not in bdev_lookup: - logger.debug(f"i/o stats for bdev {bdev_name} skipped. Either not an rbd bdev, or excluded by 'prometheus_bdev_pools'") + logger.debug( + f"i/o stats for bdev {bdev_name} skipped. Either not an rbd bdev, or excluded by 'prometheus_bdev_pools'" + ) continue bdev_read_ops.add_metric([bdev_name], bdev.get("num_read_ops")) @@ -322,8 +371,12 @@ def collect(self): bdev_write_bytes.add_metric([bdev_name], bdev.get("bytes_written")) if tick_rate: - bdev_read_seconds.add_metric([bdev_name], (bdev.get("read_latency_ticks") / tick_rate)) - bdev_write_seconds.add_metric([bdev_name], (bdev.get("write_latency_ticks") / tick_rate)) + bdev_read_seconds.add_metric( + [bdev_name], (bdev.get("read_latency_ticks") / tick_rate) + ) + bdev_write_seconds.add_metric( + [bdev_name], (bdev.get("write_latency_ticks") / tick_rate) + ) yield bdev_read_ops yield bdev_write_ops @@ -335,45 +388,66 @@ def collect(self): reactor_utilization = CounterMetricFamily( f"{self.metric_prefix}_reactor_seconds_total", "time reactor thread active with I/O", - labels=["name", "mode"]) + labels=["name", "mode"], + ) for spdk_thread in self.spdk_thread_stats.get("threads", []): if "poll" not in spdk_thread["name"]: continue if tick_rate: - reactor_utilization.add_metric([spdk_thread.get("name"), "busy"], (spdk_thread.get("busy") / tick_rate)) - reactor_utilization.add_metric([spdk_thread.get("name"), "idle"], (spdk_thread.get("idle") / tick_rate)) + reactor_utilization.add_metric( + [spdk_thread.get("name"), "busy"], + (spdk_thread.get("busy") / tick_rate), + ) + reactor_utilization.add_metric( + [spdk_thread.get("name"), "idle"], + (spdk_thread.get("idle") / tick_rate), + ) yield reactor_utilization subsystem_metadata = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_metadata", "Metadata describing the subsystem configuration", - labels=["nqn", "serial_number", "model_number", "allow_any_host", "ha_enabled", "group"]) + labels=[ + "nqn", + "serial_number", + "model_number", + "allow_any_host", + "ha_enabled", + "group", + ], + ) subsystem_listeners = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_listener_count", "Number of listener addresses used by the subsystem", - labels=["nqn"]) + labels=["nqn"], + ) subsystem_host_count = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_host_count", "Number of hosts defined to the subsystem", - labels=["nqn"]) + labels=["nqn"], + ) subsystem_namespace_limit = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_namespace_limit", "Maximum namespaces supported", - labels=["nqn"]) + labels=["nqn"], + ) subsystem_namespace_count = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_namespace_count", "Number of namespaces associated with the subsystem", - labels=["nqn"]) + labels=["nqn"], + ) subsystem_namespace_metadata = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_namespace_metadata", "Namespace information for the subsystem", - labels=["nqn", "nsid", "bdev_name", "anagrpid"]) + labels=["nqn", "nsid", "bdev_name", "anagrpid"], + ) host_connection_state = GaugeMetricFamily( f"{self.metric_prefix}_host_connection_state", "Host connection state 0=disconnected, 1=connected", - labels=["gw_name", "nqn", "host_nqn", "host_addr"]) + labels=["gw_name", "nqn", "host_nqn", "host_addr"], + ) listener_map = {} @@ -392,18 +466,25 @@ def collect(self): else: listener_map[listener.traddr] = [nqn] - subsystem_metadata.add_metric([nqn, subsys.serial_number, subsys.model_number, subsys_is_open, ha_enabled, self.gw_metadata.group], 1) + subsystem_metadata.add_metric( + [ + nqn, + subsys.serial_number, + subsys.model_number, + subsys_is_open, + ha_enabled, + self.gw_metadata.group, + ], + 1, + ) subsystem_listeners.add_metric([nqn], len(subsys.listen_addresses)) subsystem_host_count.add_metric([nqn], len(subsys.hosts)) subsystem_namespace_count.add_metric([nqn], len(subsys.namespaces)) subsystem_namespace_limit.add_metric([nqn], subsys.max_namespaces) for ns in subsys.namespaces: - subsystem_namespace_metadata.add_metric([ - nqn, - str(ns.nsid), - ns.bdev_name, - str(ns.anagrpid) - ], 1) + subsystem_namespace_metadata.add_metric( + [nqn, str(ns.nsid), ns.bdev_name, str(ns.anagrpid)], 1 + ) try: conn_info = self.connections[nqn] @@ -411,12 +492,15 @@ def collect(self): logger.debug(f"couldn't find {nqn} in connection list, skipping") continue for conn in conn_info.connections: - host_connection_state.add_metric([ - self.gw_metadata.name, - nqn, - conn.nqn, - f"{conn.traddr}:{conn.trsvcid}" if conn.connected else "" - ], 1 if conn.connected else 0) + host_connection_state.add_metric( + [ + self.gw_metadata.name, + nqn, + conn.nqn, + f"{conn.traddr}:{conn.trsvcid}" if conn.connected else "", + ], + 1 if conn.connected else 0, + ) yield subsystem_metadata yield subsystem_listeners @@ -428,15 +512,18 @@ def collect(self): subsystem_listener_iface_info = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_listener_iface_info", "Interface information", - labels=["device", "operstate", "duplex", "mac_address"]) + labels=["device", "operstate", "duplex", "mac_address"], + ) subsystem_listener_iface_speed_bytes = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_listener_iface_speed_bytes", "Link speed of the Listener interface", - labels=["device"]) + labels=["device"], + ) subsystem_listener_iface_nqn_info = GaugeMetricFamily( f"{self.metric_prefix}_subsystem_listener_iface_nqn_info", "Subsystem usage of a NIC device", - labels=["device", "nqn"]) + labels=["device", "nqn"], + ) nics = NICS() for addr in listener_map.keys(): @@ -448,21 +535,13 @@ def collect(self): nic = nics.adapters[device_name] except KeyError: continue - subsystem_listener_iface_info.add_metric([ - device_name, - nic.operstate, - nic.duplex, - nic.mac_address - ], 1) - subsystem_listener_iface_speed_bytes.add_metric([ - device_name - ], nic.speed) + subsystem_listener_iface_info.add_metric( + [device_name, nic.operstate, nic.duplex, nic.mac_address], 1 + ) + subsystem_listener_iface_speed_bytes.add_metric([device_name], nic.speed) for nqn in listener_map[addr]: - subsystem_listener_iface_nqn_info.add_metric([ - device_name, - nqn - ], 1) + subsystem_listener_iface_nqn_info.add_metric([device_name, nqn], 1) yield subsystem_listener_iface_info yield subsystem_listener_iface_speed_bytes @@ -471,7 +550,8 @@ def collect(self): method_runtimes = GaugeMetricFamily( f"{self.metric_prefix}_rpc_method_seconds", "Run times of the RPC method calls", - labels=["method"]) + labels=["method"], + ) for name, value in self.method_timings.items(): method_runtimes.add_metric([name], value) yield method_runtimes diff --git a/control/server.py b/control/server.py index 16d05b9c..9ab30534 100644 --- a/control/server.py +++ b/control/server.py @@ -27,7 +27,13 @@ from .proto import gateway_pb2 as pb2 from .proto import gateway_pb2_grpc as pb2_grpc from .proto import monitor_pb2_grpc -from .state import GatewayState, LocalGatewayState, OmapLock, OmapGatewayState, GatewayStateHandler +from .state import ( + GatewayState, + LocalGatewayState, + OmapLock, + OmapGatewayState, + GatewayStateHandler, +) from .grpc import GatewayService, MonitorGroupService from .discovery import DiscoveryService from .config import GatewayConfig @@ -36,12 +42,14 @@ from .cephutils import CephUtils from .prometheus import start_exporter + def sigterm_handler(signum, frame): """Handle SIGTERM, runs when a gateway is terminated gracefully.""" logger = GatewayLogger().logger logger.info(f"GatewayServer: SIGTERM received {signum=}") raise SystemExit(0) + def sigchld_handler(signum, frame): """Handle SIGCHLD, runs when a child process, like the spdk, terminates.""" logger = GatewayLogger().logger @@ -60,10 +68,12 @@ def sigchld_handler(signum, frame): # GW process should exit now raise SystemExit(f"Gateway subprocess terminated {pid=} {exit_code=}") + def int_to_bitmask(n): """Converts an integer n to a bitmask string""" return f"0x{hex((1 << n) - 1)[2:].upper()}" + def cpumask_set(args): """Check if reactor cpu mask is set in command line args""" @@ -73,11 +83,12 @@ def cpumask_set(args): # Check for the presence of "--cpumask=" for arg in args: - if arg.startswith('--cpumask='): + if arg.startswith("--cpumask="): return True return False + class GatewayServer: """Runs SPDK and receives client requests for the gateway service. @@ -109,7 +120,7 @@ def __init__(self, config: GatewayConfig): self.ceph_utils = None self.rpc_lock = threading.Lock() self.group_id = 0 - self.monitor_client = '/usr/bin/ceph-nvmeof-monitor-client' + self.monitor_client = "/usr/bin/ceph-nvmeof-monitor-client" self.monitor_client_log_file = None self.monitor_client_log_file_path = None self.omap_state = None @@ -148,7 +159,9 @@ def __exit__(self, exc_type, exc_value, traceback): self.spdk_log_file = None if self.spdk_log_file_path: - GatewayLogger.compress_file(self.spdk_log_file_path, f"{self.spdk_log_file_path}.gz") + GatewayLogger.compress_file( + self.spdk_log_file_path, f"{self.spdk_log_file_path}.gz" + ) self.spdk_log_file_path = None if self.monitor_client_log_file: @@ -159,7 +172,10 @@ def __exit__(self, exc_type, exc_value, traceback): self.monitor_client_log_file = None if self.monitor_client_log_file_path: - GatewayLogger.compress_file(self.monitor_client_log_file_path, f"{self.monitor_client_log_file_path}.gz") + GatewayLogger.compress_file( + self.monitor_client_log_file_path, + f"{self.monitor_client_log_file_path}.gz", + ) self.monitor_client_log_file_path = None if self.server: @@ -188,13 +204,19 @@ def set_group_id(self, id: int): def _wait_for_group_id(self): """Waits for the monitor notification of this gatway's group id""" self.monitor_server = self._grpc_server(self._monitor_address()) - monitor_pb2_grpc.add_MonitorGroupServicer_to_server(MonitorGroupService(self.set_group_id), self.monitor_server) + monitor_pb2_grpc.add_MonitorGroupServicer_to_server( + MonitorGroupService(self.set_group_id), self.monitor_server + ) self.monitor_server.start() - self.logger.info(f"MonitorGroup server is listening on {self._monitor_address()} for group id") + self.logger.info( + f"MonitorGroup server is listening on {self._monitor_address()} for group id" + ) self.monitor_event.wait() self.monitor_event = None self.logger.info("Stopping the MonitorGroup server...") - grace = self.config.getfloat_with_default("gateway", "monitor_stop_grace", 1/1000) + grace = self.config.getfloat_with_default( + "gateway", "monitor_stop_grace", 1 / 1000 + ) self.monitor_server.stop(grace).wait() self.logger.info("The MonitorGroup gRPC server stopped...") self.monitor_server = None @@ -202,15 +224,23 @@ def _wait_for_group_id(self): def start_prometheus(self): ###Starts the prometheus endpoint if enabled by the config.### - if self.config.getboolean_with_default("gateway", "enable_prometheus_exporter", True): + if self.config.getboolean_with_default( + "gateway", "enable_prometheus_exporter", True + ): self.logger.info("Prometheus endpoint is enabled") - start_exporter(self.spdk_rpc_client, self.config, self.gateway_rpc, self.logger) + start_exporter( + self.spdk_rpc_client, self.config, self.gateway_rpc, self.logger + ) else: - self.logger.info(f"Prometheus endpoint is disabled. To enable, set the config option 'enable_prometheus_exporter = True'") + self.logger.info( + f"Prometheus endpoint is disabled. To enable, set the config option 'enable_prometheus_exporter = True'" + ) def serve(self): """Starts gateway server.""" - self.logger.info(f"Starting serve, monitor client version: {self._monitor_client_version()}") + self.logger.info( + f"Starting serve, monitor client version: {self._monitor_client_version()}" + ) omap_state = OmapGatewayState(self.config, f"gateway-{self.name}") self.omap_state = omap_state @@ -235,9 +265,24 @@ def serve(self): self._start_discovery_service() # Register service implementation with server - gateway_state = GatewayStateHandler(self.config, local_state, omap_state, self.gateway_rpc_caller, f"gateway-{self.name}") + gateway_state = GatewayStateHandler( + self.config, + local_state, + omap_state, + self.gateway_rpc_caller, + f"gateway-{self.name}", + ) self.omap_lock = OmapLock(omap_state, gateway_state, self.rpc_lock) - self.gateway_rpc = GatewayService(self.config, gateway_state, self.rpc_lock, self.omap_lock, self.group_id, self.spdk_rpc_client, self.spdk_rpc_subsystems_client, self.ceph_utils) + self.gateway_rpc = GatewayService( + self.config, + gateway_state, + self.rpc_lock, + self.omap_lock, + self.group_id, + self.spdk_rpc_client, + self.spdk_rpc_subsystems_client, + self.ceph_utils, + ) self.server = self._grpc_server(self._gateway_address()) pb2_grpc.add_GatewayServicer_to_server(self.gateway_rpc, self.server) @@ -252,9 +297,11 @@ def serve(self): log_level = self.config.get_with_default("spdk", "log_level", None) if log_level and log_level.strip(): log_level = log_level.strip().upper() - log_req = pb2.set_spdk_nvmf_logs_req(log_level=log_level, print_level=log_level) + log_req = pb2.set_spdk_nvmf_logs_req( + log_level=log_level, print_level=log_level + ) self.gateway_rpc.set_spdk_nvmf_logs(log_req) - + self._register_service_map() # This should be at the end of the function, after the server is up @@ -266,10 +313,10 @@ def _register_service_map(self): metadata = { "id": self.name.removeprefix("client.nvmeof."), "pool_name": self.config.get("ceph", "pool"), - "daemon_type": "gateway", # "nvmeof: 3 active (3 hosts)" + "daemon_type": "gateway", # "nvmeof: 3 active (3 hosts)" "group": self.config.get_with_default("gateway", "group", ""), - } - self.ceph_utils.service_daemon_register(conn, metadata) + } + self.ceph_utils.service_daemon_register(conn, metadata) def _monitor_client_version(self) -> str: """Return monitor client version string.""" @@ -279,7 +326,9 @@ def _monitor_client_version(self) -> str: try: # Execute the command and capture its output signal.signal(signal.SIGCHLD, signal.SIG_IGN) - completed_process = subprocess.run([self.monitor_client, "--version"], capture_output=True, text=True) + completed_process = subprocess.run( + [self.monitor_client, "--version"], capture_output=True, text=True + ) finally: # Restore the original SIGCHLD handler signal.signal(signal.SIGCHLD, original_sigchld_handler) @@ -290,7 +339,9 @@ def _monitor_client_version(self) -> str: def _start_monitor_client(self): """Runs CEPH NVMEOF Monitor Client.""" - enable_monitor_client = self.config.getboolean_with_default("gateway", "enable_monitor_client", True) + enable_monitor_client = self.config.getboolean_with_default( + "gateway", "enable_monitor_client", True + ) if not enable_monitor_client: self.logger.info("CEPH monitor client is disabled") return @@ -298,29 +349,47 @@ def _start_monitor_client(self): rados_id = self.config.get_with_default("ceph", "id", "client.admin") if not rados_id.startswith(client_prefix): rados_id = client_prefix + rados_id - cmd = [ self.monitor_client, - "--gateway-name", self.name, - "--gateway-address", self._gateway_address(), - "--gateway-pool", self.config.get("ceph", "pool"), - "--gateway-group", self.config.get_with_default("gateway", "group", ""), - "--monitor-group-address", self._monitor_address(), - '-c', '/etc/ceph/ceph.conf', - '-n', rados_id, - '-k', '/etc/ceph/keyring'] + cmd = [ + self.monitor_client, + "--gateway-name", + self.name, + "--gateway-address", + self._gateway_address(), + "--gateway-pool", + self.config.get("ceph", "pool"), + "--gateway-group", + self.config.get_with_default("gateway", "group", ""), + "--monitor-group-address", + self._monitor_address(), + "-c", + "/etc/ceph/ceph.conf", + "-n", + rados_id, + "-k", + "/etc/ceph/keyring", + ] if self.config.getboolean("gateway", "enable_auth"): cmd += [ - "--server-cert", self.config.get("mtls", "server_cert"), - "--client-key", self.config.get("mtls", "client_key"), - "--client-cert", self.config.get("mtls", "client_cert") ] + "--server-cert", + self.config.get("mtls", "server_cert"), + "--client-key", + self.config.get("mtls", "client_key"), + "--client-cert", + self.config.get("mtls", "client_cert"), + ] self.monitor_client_log_file = None self.monitor_client_log_file_path = None log_stderr = None log_file_dir = self.config.get_with_default("monitor", "log_file_dir", None) - self.monitor_client_log_file_path = self.handle_process_output_file(log_file_dir, "monitor-client") + self.monitor_client_log_file_path = self.handle_process_output_file( + log_file_dir, "monitor-client" + ) if self.monitor_client_log_file_path: try: - self.monitor_client_log_file = open(self.monitor_client_log_file_path, "wt") + self.monitor_client_log_file = open( + self.monitor_client_log_file_path, "wt" + ) log_stderr = subprocess.STDOUT except Exception: pass @@ -328,10 +397,16 @@ def _start_monitor_client(self): self.logger.info(f"Starting {' '.join(cmd)}") try: # start monitor client process - self.monitor_client_process = subprocess.Popen(cmd, stdout=self.monitor_client_log_file, stderr=log_stderr) - self.logger.info(f"monitor client process id: {self.monitor_client_process.pid}") + self.monitor_client_process = subprocess.Popen( + cmd, stdout=self.monitor_client_log_file, stderr=log_stderr + ) + self.logger.info( + f"monitor client process id: {self.monitor_client_process.pid}" + ) if self.monitor_client_log_file and self.monitor_client_log_file_path: - self.logger.info(f"Monitor log file is {self.monitor_client_log_file_path}") + self.logger.info( + f"Monitor log file is {self.monitor_client_log_file_path}" + ) # wait for monitor notification of the group id self._wait_for_group_id() except Exception: @@ -340,13 +415,17 @@ def _start_monitor_client(self): def _start_discovery_service(self): """Runs either SPDK on CEPH NVMEOF Discovery Service.""" - enable_spdk_discovery_controller = self.config.getboolean_with_default("gateway", "enable_spdk_discovery_controller", False) + enable_spdk_discovery_controller = self.config.getboolean_with_default( + "gateway", "enable_spdk_discovery_controller", False + ) if enable_spdk_discovery_controller: self.logger.info("Using SPDK discovery service") return try: - rpc_nvmf.nvmf_delete_subsystem(self.spdk_rpc_client, GatewayUtils.DISCOVERY_NQN) + rpc_nvmf.nvmf_delete_subsystem( + self.spdk_rpc_client, GatewayUtils.DISCOVERY_NQN + ) except Exception: self.logger.exception(f"Delete Discovery subsystem returned with error") raise @@ -412,8 +491,7 @@ def _grpc_server(self, address): ) # Add secure port using credentials - server.add_secure_port( - address, server_credentials) + server.add_secure_port(address, server_credentials) else: # Authentication is not enabled server.add_insecure_port(address) @@ -433,7 +511,7 @@ def handle_process_output_file(self, log_file_dir, prefix): log_file_dir += "/" log_file_path = f"{log_file_dir}{prefix}-{self.name}.log" - log_files = [f"{log_file_path}{suffix}" for suffix in ['.bak', '.gz.bak']] + log_files = [f"{log_file_path}{suffix}" for suffix in [".bak", ".gz.bak"]] for log_file in log_files: try: os.remove(log_file) @@ -469,12 +547,15 @@ def _start_spdk(self, omap_state): sockdir += "/" sockname = self.config.get_with_default("spdk", "rpc_socket_name", "spdk.sock") if sockname.find("/") >= 0: - self.logger.error(f"Invalid SPDK socket name \"{sockname}\". Name should not contain a \"/\".") - raise(f"Invalid SPDK socket name.") + self.logger.error( + f'Invalid SPDK socket name "{sockname}". Name should not contain a "/".' + ) + raise (f"Invalid SPDK socket name.") self.spdk_rpc_socket_path = sockdir + sockname self.logger.info(f"SPDK Socket: {self.spdk_rpc_socket_path}") spdk_tgt_cmd_extra_args = self.config.get_with_default( - "spdk", "tgt_cmd_extra_args", "") + "spdk", "tgt_cmd_extra_args", "" + ) cmd = [spdk_tgt_path, "-u", "-r", self.spdk_rpc_socket_path] # Add extra args from the conf file @@ -502,17 +583,25 @@ def _start_spdk(self, omap_state): self.logger.info(f"Starting {' '.join(cmd)}") try: # start spdk process - time.sleep(2) # this is a temporary hack, we have a timing issue here. Once we solve it the sleep will ve removed - self.spdk_process = subprocess.Popen(cmd, stdout=self.spdk_log_file, stderr=log_stderr) + time.sleep( + 2 + ) # this is a temporary hack, we have a timing issue here. Once we solve it the sleep will ve removed + self.spdk_process = subprocess.Popen( + cmd, stdout=self.spdk_log_file, stderr=log_stderr + ) except Exception: self.logger.exception(f"Unable to start SPDK") raise # Initialization timeout = self.config.getfloat_with_default("spdk", "timeout", 60.0) - protocol_log_level = self.config.get_with_default("spdk", "protocol_log_level", None) + protocol_log_level = self.config.get_with_default( + "spdk", "protocol_log_level", None + ) if not protocol_log_level or not protocol_log_level.strip(): - protocol_log_level = self.config.get_with_default("spdk", "log_level", "WARNING") + protocol_log_level = self.config.get_with_default( + "spdk", "log_level", "WARNING" + ) if not protocol_log_level or not protocol_log_level.strip(): protocol_log_level = "WARNING" else: @@ -553,8 +642,7 @@ def _start_spdk(self, omap_state): raise # Implicitly create transports - spdk_transports = self.config.get_with_default("spdk", "transports", - "tcp") + spdk_transports = self.config.get_with_default("spdk", "transports", "tcp") for trtype in spdk_transports.split(): self._create_transport(trtype.lower()) @@ -562,7 +650,7 @@ def _start_spdk(self, omap_state): return_version = spdk.rpc.spdk_get_version(self.spdk_rpc_client) try: version_string = return_version["version"] - self.logger.info(f"Started SPDK with version \"{version_string}\"") + self.logger.info(f'Started SPDK with version "{version_string}"') except KeyError: self.logger.error(f"Can't find SPDK version string in {return_version}") except Exception: @@ -571,24 +659,30 @@ def _start_spdk(self, omap_state): def _stop_subprocess(self, proc, timeout): """Stops SPDK process.""" - assert proc is not None # should be verified by the caller + assert proc is not None # should be verified by the caller return_code = proc.returncode # Terminate spdk process if return_code is not None: - self.logger.error(f"{self.name} pid {proc.pid} " - f"already terminated, exit code: {return_code}") + self.logger.error( + f"{self.name} pid {proc.pid} " + f"already terminated, exit code: {return_code}" + ) else: - self.logger.info(f"Terminating sub process of ({self.name}) pid {proc.pid} args {proc.args} ...") + self.logger.info( + f"Terminating sub process of ({self.name}) pid {proc.pid} args {proc.args} ..." + ) proc.terminate() try: proc.communicate(timeout=timeout) except subprocess.TimeoutExpired: - self.logger.exception(f"({self.name}) pid {proc.pid} " - f"timeout occurred while terminating sub process:") - proc.kill() # kill -9, send KILL signal + self.logger.exception( + f"({self.name}) pid {proc.pid} " + f"timeout occurred while terminating sub process:" + ) + proc.kill() # kill -9, send KILL signal def _stop_monitor_client(self): """Stops Monitor client.""" @@ -602,7 +696,10 @@ def _stop_monitor_client(self): pass self.monitor_client_log_file = None if self.monitor_client_log_file_path: - GatewayLogger.compress_file(self.monitor_client_log_file_path, f"{self.monitor_client_log_file_path}.gz") + GatewayLogger.compress_file( + self.monitor_client_log_file_path, + f"{self.monitor_client_log_file_path}.gz", + ) self.monitor_client_log_file_path = None def _stop_spdk(self): @@ -618,7 +715,9 @@ def _stop_spdk(self): pass self.spdk_log_file = None if self.spdk_log_file_path: - GatewayLogger.compress_file(self.spdk_log_file_path, f"{self.spdk_log_file_path}.gz") + GatewayLogger.compress_file( + self.spdk_log_file_path, f"{self.spdk_log_file_path}.gz" + ) self.spdk_log_file_path = None # Clean spdk rpc socket @@ -626,11 +725,13 @@ def _stop_spdk(self): try: os.remove(self.spdk_rpc_socket_path) except Exception: - self.logger.exception(f"An error occurred while removing RPC socket {self.spdk_rpc_socket_path}") + self.logger.exception( + f"An error occurred while removing RPC socket {self.spdk_rpc_socket_path}" + ) def _stop_discovery(self): """Stops Discovery service process.""" - assert self.discovery_pid is not None # should be verified by the caller + assert self.discovery_pid is not None # should be verified by the caller self.logger.info("Terminating discovery service...") # discovery service selector loop should exit due to KeyboardInterrupt exception @@ -638,14 +739,14 @@ def _stop_discovery(self): os.kill(self.discovery_pid, signal.SIGINT) os.waitpid(self.discovery_pid, 0) except (ChildProcessError, ProcessLookupError): - pass # ignore + pass # ignore self.logger.info("Discovery service terminated") self.discovery_pid = None def _create_transport(self, trtype): """Initializes a transport type.""" - args = {'trtype': trtype} + args = {"trtype": trtype} name = "transport_" + trtype + "_options" options = self.config.get_with_default("spdk", name, "") @@ -659,19 +760,23 @@ def _create_transport(self, trtype): raise try: - status = rpc_nvmf.nvmf_create_transport( - self.spdk_rpc_client, **args) + status = rpc_nvmf.nvmf_create_transport(self.spdk_rpc_client, **args) except Exception: self.logger.exception(f"Create Transport {trtype} returned with error") raise def keep_alive(self): """Continuously confirms communication with SPDK process.""" - allowed_consecutive_spdk_ping_failures = self.config.getint_with_default("gateway", - "allowed_consecutive_spdk_ping_failures", 1) - spdk_ping_interval_in_seconds = self.config.getfloat_with_default("gateway", "spdk_ping_interval_in_seconds", 2.0) + allowed_consecutive_spdk_ping_failures = self.config.getint_with_default( + "gateway", "allowed_consecutive_spdk_ping_failures", 1 + ) + spdk_ping_interval_in_seconds = self.config.getfloat_with_default( + "gateway", "spdk_ping_interval_in_seconds", 2.0 + ) if spdk_ping_interval_in_seconds < 0.0: - self.logger.warning(f"Invalid SPDK ping interval {spdk_ping_interval_in_seconds}, will reset to 0") + self.logger.warning( + f"Invalid SPDK ping interval {spdk_ping_interval_in_seconds}, will reset to 0" + ) spdk_ping_interval_in_seconds = 0.0 consecutive_ping_failures = 0 @@ -691,10 +796,14 @@ def keep_alive(self): if not alive: consecutive_ping_failures += 1 if consecutive_ping_failures >= allowed_consecutive_spdk_ping_failures: - self.logger.critical(f"SPDK ping failed {consecutive_ping_failures} times, aborting") + self.logger.critical( + f"SPDK ping failed {consecutive_ping_failures} times, aborting" + ) raise SystemExit(f"SPDK ping failed, quitting gateway") else: - self.logger.warning(f"SPDK ping failed {consecutive_ping_failures} times, will keep trying") + self.logger.warning( + f"SPDK ping failed {consecutive_ping_failures} times, will keep trying" + ) else: consecutive_ping_failures = 0 @@ -712,59 +821,85 @@ def gateway_rpc_caller(self, requests, is_add_req): for key, val in requests.items(): if key.startswith(GatewayState.SUBSYSTEM_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.create_subsystem_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.create_subsystem_req(), ignore_unknown_fields=True + ) self.gateway_rpc.create_subsystem(req) else: - req = json_format.Parse(val, - pb2.delete_subsystem_req(), - ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.delete_subsystem_req(), ignore_unknown_fields=True + ) self.gateway_rpc.delete_subsystem(req) elif key.startswith(GatewayState.NAMESPACE_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.namespace_add_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.namespace_add_req(), ignore_unknown_fields=True + ) self.gateway_rpc.namespace_add(req) else: - req = json_format.Parse(val, - pb2.namespace_delete_req(), - ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.namespace_delete_req(), ignore_unknown_fields=True + ) self.gateway_rpc.namespace_delete(req) elif key.startswith(GatewayState.NAMESPACE_QOS_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.namespace_set_qos_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.namespace_set_qos_req(), ignore_unknown_fields=True + ) self.gateway_rpc.namespace_set_qos_limits(req) else: # Do nothing, this is covered by the delete namespace code pass elif key.startswith(GatewayState.HOST_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.add_host_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.add_host_req(), ignore_unknown_fields=True + ) self.gateway_rpc.add_host(req) else: - req = json_format.Parse(val, pb2.remove_host_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.remove_host_req(), ignore_unknown_fields=True + ) self.gateway_rpc.remove_host(req) elif key.startswith(GatewayState.LISTENER_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.create_listener_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.create_listener_req(), ignore_unknown_fields=True + ) self.gateway_rpc.create_listener(req) else: - req = json_format.Parse(val, pb2.delete_listener_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.delete_listener_req(), ignore_unknown_fields=True + ) self.gateway_rpc.delete_listener(req) elif key.startswith(GatewayState.NAMESPACE_LB_GROUP_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.namespace_change_load_balancing_group_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, + pb2.namespace_change_load_balancing_group_req(), + ignore_unknown_fields=True, + ) self.gateway_rpc.namespace_change_load_balancing_group(req) elif key.startswith(GatewayState.NAMESPACE_HOST_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.namespace_add_host_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.namespace_add_host_req(), ignore_unknown_fields=True + ) self.gateway_rpc.namespace_add_host(req) else: - req = json_format.Parse(val, pb2.namespace_delete_host_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.namespace_delete_host_req(), ignore_unknown_fields=True + ) self.gateway_rpc.namespace_delete_host(req) elif key.startswith(GatewayState.HOST_KEY_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.change_host_key_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.change_host_key_req(), ignore_unknown_fields=True + ) self.gateway_rpc.change_host_key(req) elif key.startswith(GatewayState.SUBSYSTEM_KEY_PREFIX): if is_add_req: - req = json_format.Parse(val, pb2.change_subsystem_key_req(), ignore_unknown_fields=True) + req = json_format.Parse( + val, pb2.change_subsystem_key_req(), ignore_unknown_fields=True + ) self.gateway_rpc.change_subsystem_key(req) diff --git a/control/state.py b/control/state.py index 27e5d523..cbe36b6a 100644 --- a/control/state.py +++ b/control/state.py @@ -20,6 +20,7 @@ from google.protobuf import json_format from .proto import gateway_pb2 as pb2 + class GatewayState(ABC): """Persists gateway NVMeoF target state. @@ -63,7 +64,7 @@ def build_namespace_qos_key(subsystem_nqn: str, nsid) -> str: key += GatewayState.OMAP_KEY_DELIMITER + str(nsid) return key - def build_namespace_host_key(subsystem_nqn: str, nsid, host : str) -> str: + def build_namespace_host_key(subsystem_nqn: str, nsid, host: str) -> str: key = GatewayState.NAMESPACE_HOST_PREFIX + subsystem_nqn if nsid is not None: key += GatewayState.OMAP_KEY_DELIMITER + str(nsid) @@ -94,15 +95,42 @@ def build_partial_listener_key(subsystem_nqn: str, host: str) -> str: key += GatewayState.OMAP_KEY_DELIMITER + host return key - def build_listener_key_suffix(host: str, trtype: str, traddr: str, trsvcid: int) -> str: + def build_listener_key_suffix( + host: str, trtype: str, traddr: str, trsvcid: int + ) -> str: if host: - return GatewayState.OMAP_KEY_DELIMITER + host + GatewayState.OMAP_KEY_DELIMITER + trtype + GatewayState.OMAP_KEY_DELIMITER + traddr + GatewayState.OMAP_KEY_DELIMITER + str(trsvcid) + return ( + GatewayState.OMAP_KEY_DELIMITER + + host + + GatewayState.OMAP_KEY_DELIMITER + + trtype + + GatewayState.OMAP_KEY_DELIMITER + + traddr + + GatewayState.OMAP_KEY_DELIMITER + + str(trsvcid) + ) if trtype: - return GatewayState.OMAP_KEY_DELIMITER + trtype + GatewayState.OMAP_KEY_DELIMITER + traddr + GatewayState.OMAP_KEY_DELIMITER + str(trsvcid) - return GatewayState.OMAP_KEY_DELIMITER + traddr + GatewayState.OMAP_KEY_DELIMITER + str(trsvcid) - - def build_listener_key(subsystem_nqn: str, host: str, trtype: str, traddr: str, trsvcid: int) -> str: - return GatewayState.build_partial_listener_key(subsystem_nqn, host) + GatewayState.build_listener_key_suffix(None, trtype, traddr, str(trsvcid)) + return ( + GatewayState.OMAP_KEY_DELIMITER + + trtype + + GatewayState.OMAP_KEY_DELIMITER + + traddr + + GatewayState.OMAP_KEY_DELIMITER + + str(trsvcid) + ) + return ( + GatewayState.OMAP_KEY_DELIMITER + + traddr + + GatewayState.OMAP_KEY_DELIMITER + + str(trsvcid) + ) + + def build_listener_key( + subsystem_nqn: str, host: str, trtype: str, traddr: str, trsvcid: int + ) -> str: + return GatewayState.build_partial_listener_key( + subsystem_nqn, host + ) + GatewayState.build_listener_key_suffix(None, trtype, traddr, str(trsvcid)) @abstractmethod def get_state(self) -> Dict[str, str]: @@ -132,8 +160,11 @@ def remove_namespace(self, subsystem_nqn: str, nsid: str): # Delete all keys related to the namespace state = self.get_state() for key in state.keys(): - if (key.startswith(GatewayState.build_namespace_qos_key(subsystem_nqn, nsid)) or - key.startswith(GatewayState.build_namespace_host_key(subsystem_nqn, nsid, ""))): + if key.startswith( + GatewayState.build_namespace_qos_key(subsystem_nqn, nsid) + ) or key.startswith( + GatewayState.build_namespace_host_key(subsystem_nqn, nsid, "") + ): self._remove_key(key) def add_namespace_qos(self, subsystem_nqn: str, nsid: str, val: str): @@ -146,12 +177,12 @@ def remove_namespace_qos(self, subsystem_nqn: str, nsid: str): key = GatewayState.build_namespace_qos_key(subsystem_nqn, nsid) self._remove_key(key) - def add_namespace_host(self, subsystem_nqn: str, nsid: str, host : str, val: str): + def add_namespace_host(self, subsystem_nqn: str, nsid: str, host: str, val: str): """Adds namespace's host to the state data store.""" key = GatewayState.build_namespace_host_key(subsystem_nqn, nsid, host) self._add_key(key, val) - def remove_namespace_host(self, subsystem_nqn: str, nsid: str, host : str): + def remove_namespace_host(self, subsystem_nqn: str, nsid: str, host: str): """Removes namespace's host from the state data store.""" key = GatewayState.build_namespace_host_key(subsystem_nqn, nsid, host) self._remove_key(key) @@ -169,11 +200,19 @@ def remove_subsystem(self, subsystem_nqn: str): # Delete all keys related to subsystem state = self.get_state() for key in state.keys(): - if (key.startswith(GatewayState.build_namespace_key(subsystem_nqn, None)) or - key.startswith(GatewayState.build_namespace_qos_key(subsystem_nqn, None)) or - key.startswith(GatewayState.build_namespace_host_key(subsystem_nqn, None, "")) or - key.startswith(GatewayState.build_host_key(subsystem_nqn, None)) or - key.startswith(GatewayState.build_partial_listener_key(subsystem_nqn, None))): + if ( + key.startswith(GatewayState.build_namespace_key(subsystem_nqn, None)) + or key.startswith( + GatewayState.build_namespace_qos_key(subsystem_nqn, None) + ) + or key.startswith( + GatewayState.build_namespace_host_key(subsystem_nqn, None, "") + ) + or key.startswith(GatewayState.build_host_key(subsystem_nqn, None)) + or key.startswith( + GatewayState.build_partial_listener_key(subsystem_nqn, None) + ) + ): self._remove_key(key) def add_host(self, subsystem_nqn: str, host_nqn: str, val: str): @@ -188,15 +227,29 @@ def remove_host(self, subsystem_nqn: str, host_nqn: str): if key in state.keys(): self._remove_key(key) - def add_listener(self, subsystem_nqn: str, gateway: str, trtype: str, traddr: str, trsvcid: int, val: str): + def add_listener( + self, + subsystem_nqn: str, + gateway: str, + trtype: str, + traddr: str, + trsvcid: int, + val: str, + ): """Adds a listener to the state data store.""" - key = GatewayState.build_listener_key(subsystem_nqn, gateway, trtype, traddr, trsvcid) + key = GatewayState.build_listener_key( + subsystem_nqn, gateway, trtype, traddr, trsvcid + ) self._add_key(key, val) - def remove_listener(self, subsystem_nqn: str, gateway: str, trtype: str, traddr: str, trsvcid: int): + def remove_listener( + self, subsystem_nqn: str, gateway: str, trtype: str, traddr: str, trsvcid: int + ): """Removes a listener from the state data store.""" state = self.get_state() - key = GatewayState.build_listener_key(subsystem_nqn, gateway, trtype, traddr, trsvcid) + key = GatewayState.build_listener_key( + subsystem_nqn, gateway, trtype, traddr, trsvcid + ) if key in state.keys(): self._remove_key(key) @@ -236,10 +289,13 @@ def reset(self, omap_state): """Resets dictionary with OMAP state.""" self.state = omap_state + class ReleasedLock: def __init__(self, lock: threading.Lock): self.lock = lock - assert self.lock.locked(), "Lock must be locked when creating ReleasedLock instance" + assert ( + self.lock.locked() + ), "Lock must be locked when creating ReleasedLock instance" def __enter__(self): self.lock.release() @@ -248,6 +304,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_value, traceback): self.lock.acquire() + class OmapLock: OMAP_FILE_LOCK_NAME = "omap_file_lock" OMAP_FILE_LOCK_COOKIE = "omap_file_cookie" @@ -258,14 +315,25 @@ def __init__(self, omap_state, gateway_state, rpc_lock: threading.Lock) -> None: self.gateway_state = gateway_state self.rpc_lock = rpc_lock self.is_locked = False - self.omap_file_lock_duration = self.omap_state.config.getint_with_default("gateway", "omap_file_lock_duration", 20) - self.omap_file_update_reloads = self.omap_state.config.getint_with_default("gateway", "omap_file_update_reloads", 10) - self.omap_file_lock_retries = self.omap_state.config.getint_with_default("gateway", "omap_file_lock_retries", 30) - self.omap_file_lock_retry_sleep_interval = self.omap_state.config.getfloat_with_default("gateway", - "omap_file_lock_retry_sleep_interval", 1.0) + self.omap_file_lock_duration = self.omap_state.config.getint_with_default( + "gateway", "omap_file_lock_duration", 20 + ) + self.omap_file_update_reloads = self.omap_state.config.getint_with_default( + "gateway", "omap_file_update_reloads", 10 + ) + self.omap_file_lock_retries = self.omap_state.config.getint_with_default( + "gateway", "omap_file_lock_retries", 30 + ) + self.omap_file_lock_retry_sleep_interval = ( + self.omap_state.config.getfloat_with_default( + "gateway", "omap_file_lock_retry_sleep_interval", 1.0 + ) + ) self.lock_start_time = 0.0 # This is used for testing purposes only. To allow us testing locking from two gateways at the same time - self.omap_file_disable_unlock = self.omap_state.config.getboolean_with_default("gateway", "omap_file_disable_unlock", False) + self.omap_file_disable_unlock = self.omap_state.config.getboolean_with_default( + "gateway", "omap_file_disable_unlock", False + ) if self.omap_file_disable_unlock: self.logger.warning(f"Will not unlock OMAP file for testing purposes") @@ -290,7 +358,9 @@ def __exit__(self, typ, value, traceback): self.lock_start_time = 0.0 self.unlock_omap() if duration > self.omap_file_lock_duration: - self.logger.error(f"Operation ran for {duration:.2f} seconds, but the OMAP lock expired after {self.omap_file_lock_duration} seconds") + self.logger.error( + f"Operation ran for {duration:.2f} seconds, but the OMAP lock expired after {self.omap_file_lock_duration} seconds" + ) def get_omap_lock_to_use(self, context): if context: @@ -301,7 +371,9 @@ def get_omap_lock_to_use(self, context): # This function accepts a function in which there is Omap locking. It will execute this function # and in case the Omap is not current, will reload it and try again # - def execute_omap_locking_function(self, grpc_func, omap_locking_func, request, context): + def execute_omap_locking_function( + self, grpc_func, omap_locking_func, request, context + ): for i in range(0, self.omap_file_update_reloads + 1): need_to_update = False try: @@ -321,7 +393,9 @@ def execute_omap_locking_function(self, grpc_func, omap_locking_func, request, c time.sleep(1) if need_to_update: - raise Exception(f"Unable to lock OMAP file after reloading {self.omap_file_update_reloads} times, exiting") + raise Exception( + f"Unable to lock OMAP file after reloading {self.omap_file_update_reloads} times, exiting" + ) def lock_omap(self): got_lock = False @@ -329,12 +403,20 @@ def lock_omap(self): if not self.omap_state.ioctx: self.logger.warning(f"Not locking OMAP as Rados connection is closed") - raise Exception("An attempt to lock OMAP file after Rados connection was closed") + raise Exception( + "An attempt to lock OMAP file after Rados connection was closed" + ) for i in range(0, self.omap_file_lock_retries + 1): try: - self.omap_state.ioctx.lock_exclusive(self.omap_state.omap_name, self.OMAP_FILE_LOCK_NAME, - self.OMAP_FILE_LOCK_COOKIE, "OMAP file changes lock", self.omap_file_lock_duration, 0) + self.omap_state.ioctx.lock_exclusive( + self.omap_state.omap_name, + self.OMAP_FILE_LOCK_NAME, + self.OMAP_FILE_LOCK_COOKIE, + "OMAP file changes lock", + self.omap_file_lock_duration, + 0, + ) got_lock = True if i > 0: self.logger.info(f"Succeeded to lock OMAP file after {i} retries") @@ -345,7 +427,8 @@ def lock_omap(self): break except rados.ObjectBusy as ex: self.logger.warning( - f"The OMAP file is locked, will try again in {self.omap_file_lock_retry_sleep_interval} seconds") + f"The OMAP file is locked, will try again in {self.omap_file_lock_retry_sleep_interval} seconds" + ) with ReleasedLock(self.rpc_lock): time.sleep(self.omap_file_lock_retry_sleep_interval) except Exception: @@ -353,7 +436,9 @@ def lock_omap(self): raise if not got_lock: - self.logger.error(f"Unable to lock OMAP file after {self.omap_file_lock_retries} tries. Exiting!") + self.logger.error( + f"Unable to lock OMAP file after {self.omap_file_lock_retries} tries. Exiting!" + ) raise Exception("Unable to lock OMAP file") self.is_locked = True @@ -362,10 +447,15 @@ def lock_omap(self): if omap_version > local_version: self.logger.warning( - f"Local version {local_version} differs from OMAP file version {omap_version}." - f" The file is not current, will reload it and try again") + f"Local version {local_version} differs from OMAP file version {omap_version}." + f" The file is not current, will reload it and try again" + ) self.unlock_omap() - raise OSError(errno.EAGAIN, "Unable to lock OMAP file, file not current", self.omap_state.omap_name) + raise OSError( + errno.EAGAIN, + "Unable to lock OMAP file, file not current", + self.omap_state.omap_name, + ) def unlock_omap(self): if self.omap_file_disable_unlock: @@ -377,10 +467,16 @@ def unlock_omap(self): return try: - self.omap_state.ioctx.unlock(self.omap_state.omap_name, self.OMAP_FILE_LOCK_NAME, self.OMAP_FILE_LOCK_COOKIE) + self.omap_state.ioctx.unlock( + self.omap_state.omap_name, + self.OMAP_FILE_LOCK_NAME, + self.OMAP_FILE_LOCK_COOKIE, + ) except rados.ObjectNotFound as ex: if self.is_locked: - self.logger.warning(f"No such lock, the lock duration might have passed") + self.logger.warning( + f"No such lock, the lock duration might have passed" + ) except Exception: self.logger.exception(f"Unable to unlock OMAP file") pass @@ -389,6 +485,7 @@ def unlock_omap(self): def locked(self): return self.is_locked + class OmapGatewayState(GatewayState): """Persists gateway NVMeoF target state to an OMAP object. @@ -416,8 +513,12 @@ def __init__(self, config, id_text=""): self.ioctx = None self.watch = None gateway_group = self.config.get("gateway", "group") - self.omap_name = f"nvmeof.{gateway_group}.state" if gateway_group else "nvmeof.state" - self.notify_timeout = self.config.getint_with_default("gateway", "state_update_timeout_in_msec", 2000) + self.omap_name = ( + f"nvmeof.{gateway_group}.state" if gateway_group else "nvmeof.state" + ) + self.notify_timeout = self.config.getint_with_default( + "gateway", "state_update_timeout_in_msec", 2000 + ) self.conn = None self.id_text = id_text @@ -427,11 +528,11 @@ def __init__(self, config, id_text=""): with rados.WriteOpCtx() as write_op: # Set exclusive parameter to fail write_op if object exists write_op.new(rados.LIBRADOS_CREATE_EXCLUSIVE) - self.ioctx.set_omap(write_op, (self.OMAP_VERSION_KEY,), - (str(self.version),)) + self.ioctx.set_omap( + write_op, (self.OMAP_VERSION_KEY,), (str(self.version),) + ) self.ioctx.operate_write_op(write_op, self.omap_name) - self.logger.info( - f"First gateway: created object {self.omap_name}") + self.logger.info(f"First gateway: created object {self.omap_name}") except rados.ObjectExists: self.logger.info(f"{self.omap_name} OMAP object already exists.") except Exception: @@ -444,8 +545,10 @@ def __exit__(self, exc_type, exc_value, traceback): def check_for_old_format_omap_files(self): omap_dict = self.get_state() for omap_item_key in omap_dict.keys(): - if omap_item_key.startswith("bdev"): - raise Exception("Old OMAP file format, still contains bdevs, please remove file and try again") + if omap_item_key.startswith("bdev"): + raise Exception( + "Old OMAP file format, still contains bdevs, please remove file and try again" + ) def open_rados_connection(self, config): ceph_pool = config.get("ceph", "pool") @@ -468,12 +571,13 @@ def set_local_version(self, version_update: int): def get_omap_version(self) -> int: """Returns OMAP version.""" if not self.ioctx: - self.logger.warning(f"Trying to get OMAP version when Rados connection is closed") + self.logger.warning( + f"Trying to get OMAP version when Rados connection is closed" + ) return -1 with rados.ReadOpCtx() as read_op: - i, _ = self.ioctx.get_omap_vals_by_keys(read_op, - (self.OMAP_VERSION_KEY,)) + i, _ = self.ioctx.get_omap_vals_by_keys(read_op, (self.OMAP_VERSION_KEY,)) self.ioctx.operate_read_op(read_op, self.omap_name) value_list = list(dict(i).values()) if len(value_list) == 1: @@ -482,15 +586,20 @@ def get_omap_version(self) -> int: else: self.logger.error( f"Read of OMAP version key ({self.OMAP_VERSION_KEY}) returns" - f" invalid number of values ({value_list}).") + f" invalid number of values ({value_list})." + ) raise def get_state(self) -> Dict[str, str]: """Returns dict of all OMAP keys and values.""" - omap_list = [("", 0)] # Dummy, non empty, list value. Just so we would enter the while + omap_list = [ + ("", 0) + ] # Dummy, non empty, list value. Just so we would enter the while omap_dict = {} if not self.ioctx: - self.logger.warning(f"Trying to get OMAP state when Rados connection is closed") + self.logger.warning( + f"Trying to get OMAP state when Rados connection is closed" + ) return omap_dict # The number of items returned is limited by Ceph, so we need to read in a loop until no more items are returned while len(omap_list) > 0: @@ -511,11 +620,15 @@ def _add_key(self, key: str, val: str): version_update = self.version + 1 with rados.WriteOpCtx() as write_op: # Compare operation failure will cause write failure - write_op.omap_cmp(self.OMAP_VERSION_KEY, str(self.version), - rados.LIBRADOS_CMPXATTR_OP_EQ) + write_op.omap_cmp( + self.OMAP_VERSION_KEY, + str(self.version), + rados.LIBRADOS_CMPXATTR_OP_EQ, + ) self.ioctx.set_omap(write_op, (key,), (val,)) - self.ioctx.set_omap(write_op, (self.OMAP_VERSION_KEY,), - (str(version_update),)) + self.ioctx.set_omap( + write_op, (self.OMAP_VERSION_KEY,), (str(version_update),) + ) self.ioctx.operate_write_op(write_op, self.omap_name) self.version = version_update self.logger.debug(f"omap_key generated: {key}") @@ -525,7 +638,7 @@ def _add_key(self, key: str, val: str): # Notify other gateways within the group of change try: - self.ioctx.notify(self.omap_name, timeout_ms = self.notify_timeout) + self.ioctx.notify(self.omap_name, timeout_ms=self.notify_timeout) except Exception as ex: self.logger.warning(f"Failed to notify.") @@ -538,11 +651,15 @@ def _remove_key(self, key: str): version_update = self.version + 1 with rados.WriteOpCtx() as write_op: # Compare operation failure will cause remove failure - write_op.omap_cmp(self.OMAP_VERSION_KEY, str(self.version), - rados.LIBRADOS_CMPXATTR_OP_EQ) + write_op.omap_cmp( + self.OMAP_VERSION_KEY, + str(self.version), + rados.LIBRADOS_CMPXATTR_OP_EQ, + ) self.ioctx.remove_omap_keys(write_op, (key,)) - self.ioctx.set_omap(write_op, (self.OMAP_VERSION_KEY,), - (str(version_update),)) + self.ioctx.set_omap( + write_op, (self.OMAP_VERSION_KEY,), (str(version_update),) + ) self.ioctx.operate_write_op(write_op, self.omap_name) self.version = version_update self.logger.debug(f"omap_key removed: {key}") @@ -552,7 +669,7 @@ def _remove_key(self, key: str): # Notify other gateways within the group of change try: - self.ioctx.notify(self.omap_name, timeout_ms = self.notify_timeout) + self.ioctx.notify(self.omap_name, timeout_ms=self.notify_timeout) except Exception as ex: self.logger.warning(f"Failed to notify.") @@ -565,8 +682,7 @@ def delete_state(self): with rados.WriteOpCtx() as write_op: self.ioctx.clear_omap(write_op) self.ioctx.operate_write_op(write_op, self.omap_name) - self.ioctx.set_omap(write_op, (self.OMAP_VERSION_KEY,), - (str(1),)) + self.ioctx.set_omap(write_op, (self.OMAP_VERSION_KEY,), (str(1),)) self.ioctx.operate_write_op(write_op, self.omap_name) self.logger.info(f"Deleted OMAP contents.") except Exception: @@ -590,7 +706,7 @@ def _watcher_callback(notify_id, notifier_id, watch_id, data): else: self.logger.info(f"Watch already exists.") - def cleanup_omap(self, omap_lock = None): + def cleanup_omap(self, omap_lock=None): self.logger.info(f"Cleanup OMAP on exit ({self.id_text})") if self.watch: try: @@ -615,6 +731,7 @@ def cleanup_omap(self, omap_lock = None): self.conn.shutdown() self.conn = None + class GatewayStateHandler: """Maintains consistency in NVMeoF target state store instances. @@ -636,13 +753,13 @@ def __init__(self, config, local, omap, gateway_rpc_caller, id_text=""): self.gateway_rpc_caller = gateway_rpc_caller self.update_timer = None self.logger = GatewayLogger(self.config).logger - self.update_interval = self.config.getint("gateway", - "state_update_interval_sec") + self.update_interval = self.config.getint( + "gateway", "state_update_interval_sec" + ) if self.update_interval < 1: self.logger.info("Invalid state_update_interval_sec. Setting to 1.") self.update_interval = 1 - self.use_notify = self.config.getboolean("gateway", - "state_update_notify") + self.use_notify = self.config.getboolean("gateway", "state_update_notify") self.update_is_active_lock = threading.Lock() self.id_text = id_text @@ -666,12 +783,12 @@ def remove_namespace_qos(self, subsystem_nqn: str, nsid: str): self.omap.remove_namespace_qos(subsystem_nqn, nsid) self.local.remove_namespace_qos(subsystem_nqn, nsid) - def add_namespace_host(self, subsystem_nqn: str, nsid: str, host : str, val: str): + def add_namespace_host(self, subsystem_nqn: str, nsid: str, host: str, val: str): """Adds namespace's host to the state data store.""" self.omap.add_namespace_host(subsystem_nqn, nsid, host, val) self.local.add_namespace_host(subsystem_nqn, nsid, host, val) - def remove_namespace_host(self, subsystem_nqn: str, nsid: str, host : str): + def remove_namespace_host(self, subsystem_nqn: str, nsid: str, host: str): """Removes namespace's host from the state data store.""" self.omap.remove_namespace_host(subsystem_nqn, nsid, host) self.local.remove_namespace_host(subsystem_nqn, nsid, host) @@ -696,18 +813,25 @@ def remove_host(self, subsystem_nqn: str, host_nqn: str): self.omap.remove_host(subsystem_nqn, host_nqn) self.local.remove_host(subsystem_nqn, host_nqn) - def add_listener(self, subsystem_nqn: str, gateway: str, trtype: str, traddr: str, trsvcid: str, val: str): + def add_listener( + self, + subsystem_nqn: str, + gateway: str, + trtype: str, + traddr: str, + trsvcid: str, + val: str, + ): """Adds a listener to the state data store.""" self.omap.add_listener(subsystem_nqn, gateway, trtype, traddr, trsvcid, val) self.local.add_listener(subsystem_nqn, gateway, trtype, traddr, trsvcid, val) - def remove_listener(self, subsystem_nqn: str, gateway: str, trtype: str, - traddr: str, trsvcid: str): + def remove_listener( + self, subsystem_nqn: str, gateway: str, trtype: str, traddr: str, trsvcid: str + ): """Removes a listener from the state data store.""" - self.omap.remove_listener(subsystem_nqn, gateway, trtype, traddr, - trsvcid) - self.local.remove_listener(subsystem_nqn, gateway, trtype, traddr, - trsvcid) + self.omap.remove_listener(subsystem_nqn, gateway, trtype, traddr, trsvcid) + self.local.remove_listener(subsystem_nqn, gateway, trtype, traddr, trsvcid) def delete_state(self): """Deletes state data stores.""" @@ -723,9 +847,9 @@ def start_update(self): # Start polling for state updates if self.update_timer is None: - self.update_timer = threading.Thread(target=self._update_caller, - daemon=True, - args=(notify_event,)) + self.update_timer = threading.Thread( + target=self._update_caller, daemon=True, args=(notify_event,) + ) self.update_timer.start() else: self.logger.info("Update timer already set.") @@ -743,19 +867,27 @@ def namespace_only_lb_group_id_changed(self, old_val, new_val): old_req = None new_req = None try: - old_req = json_format.Parse(old_val, pb2.namespace_add_req(), ignore_unknown_fields=True) + old_req = json_format.Parse( + old_val, pb2.namespace_add_req(), ignore_unknown_fields=True + ) except json_format.ParseError: self.logger.exception(f"Got exception parsing {old_val}") return (False, None) try: - new_req = json_format.Parse(new_val, pb2.namespace_add_req(), ignore_unknown_fields=True) + new_req = json_format.Parse( + new_val, pb2.namespace_add_req(), ignore_unknown_fields=True + ) except json_format.ParseError: self.logger.exeption(f"Got exception parsing {new_val}") return (False, None) if not old_req or not new_req: - self.logger.debug(f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}") + self.logger.debug( + f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}" + ) return (False, None) - assert old_req != new_req, f"Something was wrong we shouldn't get identical old and new values ({old_req})" + assert ( + old_req != new_req + ), f"Something was wrong we shouldn't get identical old and new values ({old_req})" old_req.anagrpid = new_req.anagrpid if old_req != new_req: # Something besides the group id is different @@ -767,19 +899,27 @@ def host_only_key_changed(self, old_val, new_val): old_req = None new_req = None try: - old_req = json_format.Parse(old_val, pb2.add_host_req(), ignore_unknown_fields=True ) + old_req = json_format.Parse( + old_val, pb2.add_host_req(), ignore_unknown_fields=True + ) except json_format.ParseError: self.logger.exception(f"Got exception parsing {old_val}") return (False, None) try: - new_req = json_format.Parse(new_val, pb2.add_host_req(), ignore_unknown_fields=True) + new_req = json_format.Parse( + new_val, pb2.add_host_req(), ignore_unknown_fields=True + ) except json_format.ParseError: self.logger.exeption(f"Got exception parsing {new_val}") return (False, None) if not old_req or not new_req: - self.logger.debug(f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}") + self.logger.debug( + f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}" + ) return (False, None) - assert old_req != new_req, f"Something was wrong we shouldn't get identical old and new values ({old_req})" + assert ( + old_req != new_req + ), f"Something was wrong we shouldn't get identical old and new values ({old_req})" # Because of Json formatting of empty fields we might get a difference here, so just use the same values for empty if not old_req.dhchap_key: old_req.dhchap_key = "" @@ -796,19 +936,27 @@ def subsystem_only_key_changed(self, old_val, new_val): old_req = None new_req = None try: - old_req = json_format.Parse(old_val, pb2.create_subsystem_req(), ignore_unknown_fields=True ) + old_req = json_format.Parse( + old_val, pb2.create_subsystem_req(), ignore_unknown_fields=True + ) except json_format.ParseError: self.logger.exception(f"Got exception parsing {old_val}") return (False, None) try: - new_req = json_format.Parse(new_val, pb2.create_subsystem_req(), ignore_unknown_fields=True) + new_req = json_format.Parse( + new_val, pb2.create_subsystem_req(), ignore_unknown_fields=True + ) except json_format.ParseError: self.logger.exeption(f"Got exception parsing {new_val}") return (False, None) if not old_req or not new_req: - self.logger.debug(f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}") + self.logger.debug( + f"Failed to parse requests, old: {old_val} -> {old_req}, new: {new_val} -> {new_req}" + ) return (False, None) - assert old_req != new_req, f"Something was wrong we shouldn't get identical old and new values ({old_req})" + assert ( + old_req != new_req + ), f"Something was wrong we shouldn't get identical old and new values ({old_req})" # Because of Json formatting of empty fields we might get a difference here, so just use the same values for empty if not old_req.dhchap_key: old_req.dhchap_key = "" @@ -822,40 +970,52 @@ def subsystem_only_key_changed(self, old_val, new_val): def break_namespace_key(self, ns_key: str): if not ns_key.startswith(GatewayState.NAMESPACE_PREFIX): - self.logger.warning(f"Invalid namespace key \"{ns_key}\", can't find key parts") + self.logger.warning( + f'Invalid namespace key "{ns_key}", can\'t find key parts' + ) return (None, None) - key_end = ns_key[len(GatewayState.NAMESPACE_PREFIX) : ] + key_end = ns_key[len(GatewayState.NAMESPACE_PREFIX) :] key_parts = key_end.split(GatewayState.OMAP_KEY_DELIMITER) if len(key_parts) != 2: - self.logger.warning(f"Invalid namespace key \"{ns_key}\", can't find key parts") + self.logger.warning( + f'Invalid namespace key "{ns_key}", can\'t find key parts' + ) return (None, None) if not GatewayUtils.is_valid_nqn(key_parts[0]): - self.logger.warning(f"Invalid NQN \"{key_parts[0]}\" found for namespace key \"{ns_key}\", can't find key parts") + self.logger.warning( + f'Invalid NQN "{key_parts[0]}" found for namespace key "{ns_key}", can\'t find key parts' + ) return (None, None) nqn = key_parts[0] try: nsid = int(key_parts[1]) except ValueError: - self.logger.exception(f"Invalid NSID \"{key_parts[1]}\" found for namespace key \"{ns_key}\", can't find key parts") + self.logger.exception( + f'Invalid NSID "{key_parts[1]}" found for namespace key "{ns_key}", can\'t find key parts' + ) return (None, None) return (nqn, nsid) def break_host_key(self, host_key: str): if not host_key.startswith(GatewayState.HOST_PREFIX): - self.logger.warning(f"Invalid host key \"{host_key}\", can't find key parts") + self.logger.warning(f'Invalid host key "{host_key}", can\'t find key parts') return (None, None) - key_end = host_key[len(GatewayState.HOST_PREFIX) : ] + key_end = host_key[len(GatewayState.HOST_PREFIX) :] key_parts = key_end.split(GatewayState.OMAP_KEY_DELIMITER) if len(key_parts) != 2: - self.logger.warning(f"Invalid host key \"{host_key}\", can't find key parts") + self.logger.warning(f'Invalid host key "{host_key}", can\'t find key parts') return (None, None) if not GatewayUtils.is_valid_nqn(key_parts[0]): - self.logger.warning(f"Invalid subsystem NQN \"{key_parts[0]}\" found for host key \"{host_key}\", can't find key parts") + self.logger.warning( + f'Invalid subsystem NQN "{key_parts[0]}" found for host key "{host_key}", can\'t find key parts' + ) return (None, None) subsys_nqn = key_parts[0] if key_parts[1] != "*" and not GatewayUtils.is_valid_nqn(key_parts[1]): - self.logger.warning(f"Invalid host NQN \"{key_parts[0]}\" found for host key \"{host_key}\", can't find key parts") + self.logger.warning( + f'Invalid host NQN "{key_parts[0]}" found for host key "{host_key}", can\'t find key parts' + ) return (None, None) host_nqn = key_parts[1] @@ -863,21 +1023,27 @@ def break_host_key(self, host_key: str): def break_subsystem_key(self, subsys_key: str): if not subsys_key.startswith(GatewayState.SUBSYSTEM_PREFIX): - self.logger.warning(f"Invalid subsystem key \"{subsys_key}\", can't find key") + self.logger.warning( + f'Invalid subsystem key "{subsys_key}", can\'t find key' + ) return None key_parts = subsys_key.split(GatewayState.OMAP_KEY_DELIMITER) if len(key_parts) != 2: - self.logger.warning(f"Invalid subsystem key \"{subsys_key}\", can't find key") + self.logger.warning( + f'Invalid subsystem key "{subsys_key}", can\'t find key' + ) return None if not GatewayUtils.is_valid_nqn(key_parts[1]): - self.logger.warning(f"Invalid subsystem NQN \"{key_parts[0]}\" found for subsystem key \"{subsys_key}\", can't find key") + self.logger.warning( + f'Invalid subsystem NQN "{key_parts[0]}" found for subsystem key "{subsys_key}", can\'t find key' + ) return None subsys_nqn = key_parts[1] return subsys_nqn def get_str_from_bytes(val): - val_str = val.decode() if type(val) == type(b'') else val + val_str = val.decode() if type(val) == type(b"") else val return val_str def compare_state_values(val1, val2) -> bool: @@ -913,7 +1079,9 @@ def update(self) -> bool: local_version = self.omap.get_local_version() if local_version < omap_version: - self.logger.debug(f"Start update from {local_version} to {omap_version} ({self.id_text}).") + self.logger.debug( + f"Start update from {local_version} to {omap_version} ({self.id_text})." + ) local_state_dict = self.local.get_state() local_state_keys = local_state_dict.keys() omap_state_keys = omap_state_dict.keys() @@ -927,7 +1095,9 @@ def update(self) -> bool: changed = { key: omap_state_dict[key] for key in same_keys - if not GatewayStateHandler.compare_state_values(local_state_dict[key], omap_state_dict[key]) + if not GatewayStateHandler.compare_state_values( + local_state_dict[key], omap_state_dict[key] + ) } grouped_changed = self._group_by_prefix(changed, prefix_list) @@ -941,34 +1111,61 @@ def update(self) -> bool: for key in changed.keys(): if key.startswith(ns_prefix): try: - (should_process, new_lb_grp_id) = self.namespace_only_lb_group_id_changed(local_state_dict[key], - omap_state_dict[key]) + (should_process, new_lb_grp_id) = ( + self.namespace_only_lb_group_id_changed( + local_state_dict[key], omap_state_dict[key] + ) + ) if should_process: - assert new_lb_grp_id, "Shouldn't get here with an empty lb group id" - self.logger.debug(f"Found {key} where only the load balancing group id has changed. The new group id is {new_lb_grp_id}") + assert ( + new_lb_grp_id + ), "Shouldn't get here with an empty lb group id" + self.logger.debug( + f"Found {key} where only the load balancing group id has changed. The new group id is {new_lb_grp_id}" + ) only_lb_group_changed.append((key, new_lb_grp_id)) except Exception as ex: - self.logger.warning("Got exception checking namespace for load balancing group id change") + self.logger.warning( + "Got exception checking namespace for load balancing group id change" + ) elif key.startswith(host_prefix): try: - (should_process, - new_dhchap_key) = self.host_only_key_changed(local_state_dict[key], omap_state_dict[key]) + (should_process, new_dhchap_key) = ( + self.host_only_key_changed( + local_state_dict[key], omap_state_dict[key] + ) + ) if should_process: - assert new_dhchap_key, "Shouldn't get here with an empty dhchap key" - self.logger.debug(f"Found {key} where only the key has changed. The new DHCHAP key is {new_dhchap_key}") + assert ( + new_dhchap_key + ), "Shouldn't get here with an empty dhchap key" + self.logger.debug( + f"Found {key} where only the key has changed. The new DHCHAP key is {new_dhchap_key}" + ) only_host_key_changed.append((key, new_dhchap_key)) except Exception as ex: - self.logger.warning("Got exception checking host for key change") + self.logger.warning( + "Got exception checking host for key change" + ) elif key.startswith(subsystem_prefix): try: - (should_process, - new_dhchap_key) = self.subsystem_only_key_changed(local_state_dict[key], omap_state_dict[key]) + (should_process, new_dhchap_key) = ( + self.subsystem_only_key_changed( + local_state_dict[key], omap_state_dict[key] + ) + ) if should_process: - assert new_dhchap_key, "Shouldn't get here with an empty dhchap key" - self.logger.debug(f"Found {key} where only the key has changed. The new DHCHAP key is {new_dhchap_key}") + assert ( + new_dhchap_key + ), "Shouldn't get here with an empty dhchap key" + self.logger.debug( + f"Found {key} where only the key has changed. The new DHCHAP key is {new_dhchap_key}" + ) only_subsystem_key_changed.append((key, new_dhchap_key)) except Exception as ex: - self.logger.warning("Got exception checking subsystem for key change") + self.logger.warning( + "Got exception checking subsystem for key change" + ) for ns_key, new_lb_grp in only_lb_group_changed: ns_nqn = None @@ -977,17 +1174,27 @@ def update(self) -> bool: changed.pop(ns_key) (ns_nqn, ns_nsid) = self.break_namespace_key(ns_key) except Exception as ex: - self.logger.error(f"Exception removing {ns_key} from {changed}:\n{ex}") + self.logger.error( + f"Exception removing {ns_key} from {changed}:\n{ex}" + ) if ns_nqn and ns_nsid: try: - lbgroup_key = GatewayState.build_namespace_lbgroup_key(ns_nqn, ns_nsid) - req = pb2.namespace_change_load_balancing_group_req(subsystem_nqn=ns_nqn, nsid=ns_nsid, - anagrpid=new_lb_grp) - json_req = json_format.MessageToJson(req, preserving_proto_field_name=True, - including_default_value_fields=True) + lbgroup_key = GatewayState.build_namespace_lbgroup_key( + ns_nqn, ns_nsid + ) + req = pb2.namespace_change_load_balancing_group_req( + subsystem_nqn=ns_nqn, nsid=ns_nsid, anagrpid=new_lb_grp + ) + json_req = json_format.MessageToJson( + req, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) added[lbgroup_key] = json_req except Exception as ex: - self.logger.error(f"Exception formatting change namespace load balancing group request:\n{ex}") + self.logger.error( + f"Exception formatting change namespace load balancing group request:\n{ex}" + ) for host_key, new_dhchap_key in only_host_key_changed: subsys_nqn = None @@ -996,20 +1203,34 @@ def update(self) -> bool: changed.pop(host_key) (subsys_nqn, host_nqn) = self.break_host_key(host_key) except Exception as ex: - self.logger.error(f"Exception removing {host_key} from {changed}:\n{ex}") + self.logger.error( + f"Exception removing {host_key} from {changed}:\n{ex}" + ) if host_nqn == "*": - self.logger.warning(f"Something went wrong, host \"*\" can't have DH-HMAC-CHAP keys, ignore") + self.logger.warning( + f'Something went wrong, host "*" can\'t have DH-HMAC-CHAP keys, ignore' + ) continue if subsys_nqn and host_nqn: try: - host_key_key = GatewayState.build_host_key_key(subsys_nqn, host_nqn) - req = pb2.change_host_key_req(subsystem_nqn=subsys_nqn, host_nqn=host_nqn, - dhchap_key=new_dhchap_key) - json_req = json_format.MessageToJson(req, preserving_proto_field_name=True, - including_default_value_fields=True) + host_key_key = GatewayState.build_host_key_key( + subsys_nqn, host_nqn + ) + req = pb2.change_host_key_req( + subsystem_nqn=subsys_nqn, + host_nqn=host_nqn, + dhchap_key=new_dhchap_key, + ) + json_req = json_format.MessageToJson( + req, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) added[host_key_key] = json_req except Exception as ex: - self.logger.error(f"Exception formatting change host key request:\n{ex}") + self.logger.error( + f"Exception formatting change host key request:\n{ex}" + ) for subsys_key, new_dhchap_key in only_subsystem_key_changed: subsys_nqn = None @@ -1017,18 +1238,33 @@ def update(self) -> bool: changed.pop(subsys_key) subsys_nqn = self.break_subsystem_key(subsys_key) except Exception as ex: - self.logger.error(f"Exception removing {subsys_key} from {changed}:\n{ex}") + self.logger.error( + f"Exception removing {subsys_key} from {changed}:\n{ex}" + ) if subsys_nqn: try: - subsys_key_key = GatewayState.build_subsystem_key_key(subsys_nqn) - req = pb2.change_subsystem_key_req(subsystem_nqn=subsys_nqn, dhchap_key=new_dhchap_key) - json_req = json_format.MessageToJson(req, preserving_proto_field_name=True, - including_default_value_fields=True) + subsys_key_key = GatewayState.build_subsystem_key_key( + subsys_nqn + ) + req = pb2.change_subsystem_key_req( + subsystem_nqn=subsys_nqn, dhchap_key=new_dhchap_key + ) + json_req = json_format.MessageToJson( + req, + preserving_proto_field_name=True, + including_default_value_fields=True, + ) added[subsys_key_key] = json_req except Exception as ex: - self.logger.error(f"Exception formatting change subsystem key request:\n{ex}") - - if len(only_lb_group_changed) > 0 or len(only_host_key_changed) > 0 or len(only_subsystem_key_changed) > 0: + self.logger.error( + f"Exception formatting change subsystem key request:\n{ex}" + ) + + if ( + len(only_lb_group_changed) > 0 + or len(only_host_key_changed) > 0 + or len(only_subsystem_key_changed) > 0 + ): grouped_changed = self._group_by_prefix(changed, prefix_list) if len(only_subsystem_key_changed) > 0: prefix_list += [GatewayState.SUBSYSTEM_KEY_PREFIX] @@ -1055,7 +1291,9 @@ def update(self) -> bool: # Update local state and version self.local.reset(omap_state_dict) self.omap.set_local_version(omap_version) - self.logger.debug(f"Update complete ({local_version} -> {omap_version}) ({self.id_text}).") + self.logger.debug( + f"Update complete ({local_version} -> {omap_version}) ({self.id_text})." + ) return True diff --git a/control/utils.py b/control/utils.py index 570f5415..2a550046 100644 --- a/control/utils.py +++ b/control/utils.py @@ -21,7 +21,7 @@ class GatewayEnumUtils: - def get_value_from_key(e_type, keyval, ignore_case = False): + def get_value_from_key(e_type, keyval, ignore_case=False): val = None try: key_index = e_type.keys().index(keyval) @@ -49,17 +49,18 @@ def get_key_from_value(e_type, val): pass return keyval + class GatewayUtils: DISCOVERY_NQN = "nqn.2014-08.org.nvmexpress.discovery" # We need to enclose IPv6 addresses in brackets before concatenating a colon and port number to it - def escape_address_if_ipv6(addr : str) -> str: + def escape_address_if_ipv6(addr: str) -> str: ret_addr = addr if ":" in addr and not addr.strip().startswith("["): ret_addr = f"[{addr}]" return ret_addr - def unescape_address_if_ipv6(addr : str, adrfam : str) -> str: + def unescape_address_if_ipv6(addr: str, adrfam: str) -> str: ret_addr = addr.strip() if adrfam.lower() == "ipv6": ret_addr = ret_addr.removeprefix("[").removesuffix("]") @@ -83,9 +84,15 @@ def is_valid_rev_domain(rev_domain): return (errno.EINVAL, f"domain label {lbl} doesn't start with a letter") if lbl.endswith("-"): - return (errno.EINVAL, f"domain label {lbl} doesn't end with an alphanumeric character") + return ( + errno.EINVAL, + f"domain label {lbl} doesn't end with an alphanumeric character", + ) if not lbl.replace("-", "").isalnum(): - return (errno.EINVAL, f"domain label {lbl} contains a character which is not [a-z,A-Z,0-9,'-','.']") + return ( + errno.EINVAL, + f"domain label {lbl} contains a character which is not [a-z,A-Z,0-9,'-','.']", + ) return (0, os.strerror(0)) @@ -131,57 +138,82 @@ def is_valid_nqn(nqn): try: b = nqn.encode(encoding="utf-8") except UnicodeEncodeError: - return (errno.EINVAL, f"Invalid NQN \"{nqn}\", must have an UTF-8 encoding") + return (errno.EINVAL, f'Invalid NQN "{nqn}", must have an UTF-8 encoding') if len(nqn) < NQN_MIN_LENGTH: - return (errno.EINVAL, f"NQN \"{nqn}\" is too short, minimal length is {NQN_MIN_LENGTH}") + return ( + errno.EINVAL, + f'NQN "{nqn}" is too short, minimal length is {NQN_MIN_LENGTH}', + ) if len(nqn) > NQN_MAX_LENGTH: - return (errno.EINVAL, f"NQN \"{nqn}\" is too long, maximal length is {NQN_MAX_LENGTH}") + return ( + errno.EINVAL, + f'NQN "{nqn}" is too long, maximal length is {NQN_MAX_LENGTH}', + ) if GatewayUtils.is_discovery_nqn(nqn): # The NQN is technically valid but we will probably reject it later as being a discovery one return (0, os.strerror(0)) if nqn.startswith(NQN_UUID_PREFIX): if len(nqn) != NQN_UUID_PREFIX_LENGTH + UUID_STRING_LENGTH: - return (errno.EINVAL, f"Invalid NQN \"{nqn}\": UUID is not the correct length") - uuid_part = nqn[NQN_UUID_PREFIX_LENGTH : ] + return ( + errno.EINVAL, + f'Invalid NQN "{nqn}": UUID is not the correct length', + ) + uuid_part = nqn[NQN_UUID_PREFIX_LENGTH:] if not GatewayUtils.is_valid_uuid(uuid_part): - return (errno.EINVAL, f"Invalid NQN \"{nqn}\": UUID is not formatted correctly") + return ( + errno.EINVAL, + f'Invalid NQN "{nqn}": UUID is not formatted correctly', + ) return (0, os.strerror(0)) if not nqn.startswith(NQN_PREFIX): - return (errno.EINVAL, f"Invalid NQN \"{nqn}\", doesn't start with \"{NQN_PREFIX}\"") - - nqn_no_prefix = nqn[len(NQN_PREFIX) : ] - date_part = nqn_no_prefix[ : 8] - rev_domain_part = nqn_no_prefix[8 : ] + return ( + errno.EINVAL, + f'Invalid NQN "{nqn}", doesn\'t start with "{NQN_PREFIX}"', + ) + + nqn_no_prefix = nqn[len(NQN_PREFIX) :] + date_part = nqn_no_prefix[:8] + rev_domain_part = nqn_no_prefix[8:] if not date_part.endswith("."): - return (errno.EINVAL, f"Invalid NQN \"{nqn}\": invalid date code") - date_part = date_part[ : -1] + return (errno.EINVAL, f'Invalid NQN "{nqn}": invalid date code') + date_part = date_part[:-1] try: year_part, month_part = date_part.split("-") if len(year_part) != 4 or len(month_part) != 2: - return (errno.EINVAL, f"Invalid NQN \"{nqn}\": invalid date code") + return (errno.EINVAL, f'Invalid NQN "{nqn}": invalid date code') n = int(year_part) n = int(month_part) except ValueError: - return (errno.EINVAL, f"Invalid NQN \"{nqn}\": invalid date code") + return (errno.EINVAL, f'Invalid NQN "{nqn}": invalid date code') try: rev_domain_part, user_part = rev_domain_part.split(":", 1) except ValueError: - return (errno.EINVAL, f"Invalid NQN \"{nqn}\": must contain a user specified name starting with a \":\"") + return ( + errno.EINVAL, + f'Invalid NQN "{nqn}": must contain a user specified name starting with a ":"', + ) if not user_part: - return (errno.EINVAL, f"Invalid NQN \"{nqn}\": must contain a user specified name starting with a \":\"") + return ( + errno.EINVAL, + f'Invalid NQN "{nqn}": must contain a user specified name starting with a ":"', + ) rc = GatewayUtils.is_valid_rev_domain(rev_domain_part) if rc[0] != 0: - return (errno.EINVAL, f"Invalid NQN \"{nqn}\": reverse domain is not formatted correctly: {rc[1]}") + return ( + errno.EINVAL, + f'Invalid NQN "{nqn}": reverse domain is not formatted correctly: {rc[1]}', + ) return (0, os.strerror(0)) + class GatewayLogger: CEPH_LOG_DIRECTORY = "/var/log/ceph/" MAX_LOG_FILE_SIZE_DEFAULT = 10 @@ -196,7 +228,9 @@ class GatewayLogger: def __init__(self, config=None): if config: - self.log_directory = config.get_with_default("gateway-logs", "log_directory", GatewayLogger.CEPH_LOG_DIRECTORY) + self.log_directory = config.get_with_default( + "gateway-logs", "log_directory", GatewayLogger.CEPH_LOG_DIRECTORY + ) gateway_name = config.get("gateway", "name") else: self.log_directory = GatewayLogger.CEPH_LOG_DIRECTORY @@ -207,7 +241,9 @@ def __init__(self, config=None): if not gateway_name: gateway_name = socket.gethostname() - self.log_directory = self.log_directory + GatewayLogger.NVME_LOG_DIR_PREFIX + gateway_name + self.log_directory = ( + self.log_directory + GatewayLogger.NVME_LOG_DIR_PREFIX + gateway_name + ) if GatewayLogger.logger: assert self.logger == GatewayLogger.logger @@ -220,13 +256,33 @@ def __init__(self, config=None): frmtr = logging.Formatter(fmt=format_string, datefmt=date_fmt_string) if config: - verbose = config.getboolean_with_default("gateway-logs", "verbose_log_messages", True) - log_files_enabled = config.getboolean_with_default("gateway-logs", "log_files_enabled", True) - log_files_rotation_enabled = config.getboolean_with_default("gateway-logs", "log_files_rotation_enabled", True) - max_log_file_size = config.getint_with_default("gateway-logs", "max_log_file_size_in_mb", GatewayLogger.MAX_LOG_FILE_SIZE_DEFAULT) - max_log_files_count = config.getint_with_default("gateway-logs", "max_log_files_count", GatewayLogger.MAX_LOG_FILES_COUNT_DEFAULT) - max_log_directory_backups = config.getint_with_default("gateway-logs", "max_log_directory_backups", GatewayLogger.MAX_LOG_DIRECTORY_BACKUPS_DEFAULT) - log_level = config.get_with_default("gateway-logs", "log_level", "INFO").upper() + verbose = config.getboolean_with_default( + "gateway-logs", "verbose_log_messages", True + ) + log_files_enabled = config.getboolean_with_default( + "gateway-logs", "log_files_enabled", True + ) + log_files_rotation_enabled = config.getboolean_with_default( + "gateway-logs", "log_files_rotation_enabled", True + ) + max_log_file_size = config.getint_with_default( + "gateway-logs", + "max_log_file_size_in_mb", + GatewayLogger.MAX_LOG_FILE_SIZE_DEFAULT, + ) + max_log_files_count = config.getint_with_default( + "gateway-logs", + "max_log_files_count", + GatewayLogger.MAX_LOG_FILES_COUNT_DEFAULT, + ) + max_log_directory_backups = config.getint_with_default( + "gateway-logs", + "max_log_directory_backups", + GatewayLogger.MAX_LOG_DIRECTORY_BACKUPS_DEFAULT, + ) + log_level = config.get_with_default( + "gateway-logs", "log_level", "INFO" + ).upper() else: verbose = True log_files_enabled = False @@ -239,16 +295,20 @@ def __init__(self, config=None): self.handler = None logdir_ok = False if log_files_enabled: - GatewayLogger.rotate_backup_directories(self.log_directory, max_log_directory_backups) + GatewayLogger.rotate_backup_directories( + self.log_directory, max_log_directory_backups + ) if not log_files_rotation_enabled: max_log_file_size = 0 max_log_files_count = 0 try: os.makedirs(self.log_directory, 0o755, True) logdir_ok = True - self.handler = logging.handlers.RotatingFileHandler(self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME, - maxBytes = max_log_file_size * 1024 * 1024, - backupCount = max_log_files_count) + self.handler = logging.handlers.RotatingFileHandler( + self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME, + maxBytes=max_log_file_size * 1024 * 1024, + backupCount=max_log_files_count, + ) self.handler.setFormatter(frmtr) if log_files_rotation_enabled: self.handler.rotator = GatewayLogger.log_file_rotate @@ -257,32 +317,42 @@ def __init__(self, config=None): if not verbose: format_string = None - logging.basicConfig(level=log_level, format=format_string, datefmt=date_fmt_string) + logging.basicConfig( + level=log_level, format=format_string, datefmt=date_fmt_string + ) self.logger = logging.getLogger("nvmeof") if self.handler: self.logger.addHandler(self.handler) self.set_log_level(log_level) - self.logger.info(f"Initialize gateway log level to \"{log_level}\"") + self.logger.info(f'Initialize gateway log level to "{log_level}"') GatewayLogger.logger = self.logger GatewayLogger.handler = self.handler if not GatewayLogger.init_executed: if log_files_enabled: if not logdir_ok: - self.logger.error(f"Failed to create directory {self.log_directory}, the log wouldn't be saved to a file") + self.logger.error( + f"Failed to create directory {self.log_directory}, the log wouldn't be saved to a file" + ) elif not self.handler: - self.logger.error(f"Failed to set up log file handler, the log wouldn't be saved to a file") + self.logger.error( + f"Failed to set up log file handler, the log wouldn't be saved to a file" + ) else: rot_msg = "" if log_files_rotation_enabled: rot_msg = ", using rotation" - self.logger.info(f"Log files will be saved in {self.log_directory}{rot_msg}") + self.logger.info( + f"Log files will be saved in {self.log_directory}{rot_msg}" + ) else: - self.logger.warning(f"Log files are disabled, the log wouldn't be saved to a file") + self.logger.warning( + f"Log files are disabled, the log wouldn't be saved to a file" + ) GatewayLogger.init_executed = True def rotate_backup_directories(dirname, count): try: - shutil.rmtree(dirname + f".bak{count}", ignore_errors = True) + shutil.rmtree(dirname + f".bak{count}", ignore_errors=True) except Exception: pass for i in range(count, 2, -1): @@ -301,7 +371,7 @@ def rotate_backup_directories(dirname, count): # Just to be on the safe side, in case the rename failed try: - shutil.rmtree(dirname, ignore_errors = True) + shutil.rmtree(dirname, ignore_errors=True) except Exception: pass @@ -326,7 +396,7 @@ def log_file_rotate(src, dest): GatewayLogger.logger.info(m) for e in errs: GatewayLogger.logger.error(e) - + else: os.rename(src, dest) @@ -343,8 +413,8 @@ def compress_file(src, dest): pass need_to_remove_dest = False try: - with open(src, 'rb') as f_in: - with gzip.open(dest, 'wb') as f_out: + with open(src, "rb") as f_in: + with gzip.open(dest, "wb") as f_out: shutil.copyfileobj(f_in, f_out) except FileNotFoundError: errs.append(f"Failure compressing file {src}: file not found") @@ -381,7 +451,9 @@ def compress_final_log_file(self, gw_name): return if not self.log_directory.endswith(gw_name): - self.logger.error(f"Log directory {self.log_directory} doesn't belong to gateway {gw_name}, do not compress log file") + self.logger.error( + f"Log directory {self.log_directory} doesn't belong to gateway {gw_name}, do not compress log file" + ) return self.logger.removeHandler(self.handler) @@ -389,12 +461,18 @@ def compress_final_log_file(self, gw_name): GatewayLogger.handler = None dest_name = self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME + ".gz" - if os.access(self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME + ".1", - os.F_OK) and not os.access(self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME + ".0", - os.F_OK): - dest_name = self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME + ".0" - - msgs, errs = GatewayLogger.compress_file(self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME, dest_name) + if os.access( + self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME + ".1", os.F_OK + ) and not os.access( + self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME + ".0", os.F_OK + ): + dest_name = ( + self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME + ".0" + ) + + msgs, errs = GatewayLogger.compress_file( + self.log_directory + "/" + GatewayLogger.NVME_LOG_FILE_NAME, dest_name + ) for m in msgs: self.logger.info(m) for e in errs: @@ -404,7 +482,7 @@ def compress_final_log_file(self, gw_name): class NICS: - ignored_device_prefixes = ('lo') + ignored_device_prefixes = "lo" def __init__(self): self.addresses = {} @@ -412,7 +490,11 @@ def __init__(self): self._build_adapter_info() def _build_adapter_info(self): - for device_name in [nic for nic in netifaces.interfaces() if not nic.startswith(NICS.ignored_device_prefixes)]: + for device_name in [ + nic + for nic in netifaces.interfaces() + if not nic.startswith(NICS.ignored_device_prefixes) + ]: nic = NIC(device_name) for ipv4_addr in nic.ipv4_addresses: self.addresses[ipv4_addr] = device_name @@ -424,12 +506,12 @@ def _build_adapter_info(self): class NIC: - sysfs_root = '/sys/class/net' + sysfs_root = "/sys/class/net" def __init__(self, device_name: str) -> None: self.device_name = device_name - self.mac_list = '' + self.mac_list = "" self.ipv4_list = [] self.ipv6_list = [] @@ -449,14 +531,16 @@ def _read_sysfs(self, file_name: str) -> Tuple[int, str]: except Exception: # log the error and the filename err = 1 - content = '' + content = "" return err, content @property def operstate(self) -> str: - err, content = self._read_sysfs(f"{NIC.sysfs_root}/{self.device_name}/operstate") - return content if not err else '' + err, content = self._read_sysfs( + f"{NIC.sysfs_root}/{self.device_name}/operstate" + ) + return content if not err else "" @property def mtu(self) -> int: @@ -466,7 +550,7 @@ def mtu(self) -> int: @property def duplex(self) -> str: err, content = self._read_sysfs(f"{NIC.sysfs_root}/{self.device_name}/duplex") - return content if not err else '' + return content if not err else "" @property def speed(self) -> int: @@ -476,18 +560,18 @@ def speed(self) -> int: @property def mac_address(self) -> str: if self.mac_list: - return self.mac_list[0].get('addr') + return self.mac_list[0].get("addr") else: - return '' + return "" @property def ipv4_addresses(self) -> List[str]: - return [ipv4_info.get('addr') for ipv4_info in self.ipv4_list] + return [ipv4_info.get("addr") for ipv4_info in self.ipv4_list] @property def ipv6_addresses(self) -> List[str]: # Note. ipv6 addresses are suffixed by the adapter name - return [ipv6_info.get('addr').split('%')[0] for ipv6_info in self.ipv6_list] + return [ipv6_info.get("addr").split("%")[0] for ipv6_info in self.ipv6_list] def __str__(self): return ( diff --git a/tests/conftest.py b/tests/conftest.py index 73f4baa5..21273b1b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,14 +7,15 @@ def pytest_addoption(parser): """Sets command line options for testing.""" # Specify base config file for tests - parser.addoption("--config", - action="store", - help="Path to config file", - default="ceph-nvmeof.conf") - parser.addoption("--image", - action="store", - help="RBD image name", - default="mytestdevimage") + parser.addoption( + "--config", + action="store", + help="Path to config file", + default="ceph-nvmeof.conf", + ) + parser.addoption( + "--image", action="store", help="RBD image name", default="mytestdevimage" + ) @pytest.fixture(scope="session") diff --git a/tests/test_cli.py b/tests/test_cli.py index d41440f0..2d6ce24f 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -52,14 +52,20 @@ listener_list = [["-a", addr, "-s", "5001", "-f", "ipv4"], ["-a", addr, "-s", "5002"]] listener_list_no_port = [["-a", addr]] listener_list_invalid_adrfam = [["-a", addr, "-s", "5013", "--adrfam", "JUNK"]] -listener_list_ipv6 = [["-a", addr_ipv6, "-s", "5003", "--adrfam", "ipv6"], ["-a", addr_ipv6, "-s", "5004", "--adrfam", "IPV6"]] -listener_list_discovery = [["-n", discovery_nqn, "-t", host_name, "-a", addr, "-s", "5012"]] +listener_list_ipv6 = [ + ["-a", addr_ipv6, "-s", "5003", "--adrfam", "ipv6"], + ["-a", addr_ipv6, "-s", "5004", "--adrfam", "IPV6"], +] +listener_list_discovery = [ + ["-n", discovery_nqn, "-t", host_name, "-a", addr, "-s", "5012"] +] listener_list_negative_port = [["-t", host_name, "-a", addr, "-s", "-2000"]] listener_list_big_port = [["-t", host_name, "-a", addr, "-s", "70000"]] listener_list_wrong_host = [["-t", "WRONG", "-a", addr, "-s", "5015", "-f", "ipv4"]] config = "ceph-nvmeof.conf" group_name = "GROUPNAME" + @pytest.fixture(scope="module") def gateway(config): """Sets up and tears down Gateway""" @@ -78,7 +84,11 @@ def gateway(config): # Start gateway gateway.gw_logger_object.set_log_level("debug") - ceph_utils.execute_ceph_monitor_command("{" + f'"prefix":"nvme-gw create", "id": "{gateway.name}", "pool": "{pool}", "group": "{group_name}"' + "}") + ceph_utils.execute_ceph_monitor_command( + "{" + + f'"prefix":"nvme-gw create", "id": "{gateway.name}", "pool": "{pool}", "group": "{group_name}"' + + "}" + ) gateway.serve() # Bind the client and Gateway @@ -90,6 +100,7 @@ def gateway(config): gateway.server.stop(grace=1) gateway.gateway_rpc.gateway_state.delete_state() + class TestGet: def test_get_subsystems(self, caplog, gateway): caplog.clear() @@ -159,56 +170,159 @@ def test_get_gateway_info(self, caplog, gateway): assert gw_info.status == 0 assert gw_info.bool_status == True + class TestCreate: def test_create_subsystem(self, caplog, gateway): caplog.clear() cli(["subsystem", "add", "--subsystem", "nqn.2016", "--no-group-append"]) assert f'NQN "nqn.2016" is too short, minimal length is 11' in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", -"nqn.2016-06XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.2016-06XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX", + "--no-group-append", + ] + ) assert f"is too long, maximal length is 223" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "nqn.2014-08.org.nvmexpress:uuid:0", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.2014-08.org.nvmexpress:uuid:0", + "--no-group-append", + ] + ) assert f"UUID is not the correct length" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "nqn.2014-08.org.nvmexpress:uuid:9e9134-3cb431-4f3e-91eb-a13cefaabebf", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.2014-08.org.nvmexpress:uuid:9e9134-3cb431-4f3e-91eb-a13cefaabebf", + "--no-group-append", + ] + ) assert f"UUID is not formatted correctly" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "qqn.2016-06.io.spdk:cnode1", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "qqn.2016-06.io.spdk:cnode1", + "--no-group-append", + ] + ) assert f"doesn't start with" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "nqn.016-206.io.spdk:cnode1", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.016-206.io.spdk:cnode1", + "--no-group-append", + ] + ) assert f"invalid date code" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "nqn.2X16-06.io.spdk:cnode1", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.2X16-06.io.spdk:cnode1", + "--no-group-append", + ] + ) assert f"invalid date code" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "nqn.2016-06.io.spdk:", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.2016-06.io.spdk:", + "--no-group-append", + ] + ) assert f"must contain a user specified name starting with" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "nqn.2016-06.io..spdk:cnode1", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.2016-06.io..spdk:cnode1", + "--no-group-append", + ] + ) assert f"reverse domain is not formatted correctly" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "nqn.2016-06.io.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX.spdk:cnode1", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.2016-06.io.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX.spdk:cnode1", + "--no-group-append", + ] + ) assert f"reverse domain is not formatted correctly" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", "nqn.2016-06.io.-spdk:cnode1", "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + "nqn.2016-06.io.-spdk:cnode1", + "--no-group-append", + ] + ) assert f"reverse domain is not formatted correctly" in caplog.text caplog.clear() cli(["subsystem", "add", "--subsystem", f"{subsystem}_X", "--no-group-append"]) assert f"Invalid NQN" in caplog.text assert f"contains invalid characters" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", subsystem, "--max-namespaces", "2049", "--no-group-append"]) - assert f"The requested max number of namespaces for subsystem {subsystem} (2049) is greater than the global limit on the number of namespaces (11), will continue" in caplog.text + cli( + [ + "subsystem", + "add", + "--subsystem", + subsystem, + "--max-namespaces", + "2049", + "--no-group-append", + ] + ) + assert ( + f"The requested max number of namespaces for subsystem {subsystem} (2049) is greater than the global limit on the number of namespaces (11), will continue" + in caplog.text + ) assert f"Adding subsystem {subsystem}: Successful" in caplog.text cli(["--format", "json", "subsystem", "list"]) assert f'"serial_number": "{serial}"' not in caplog.text assert f'"nqn": "{subsystem}"' in caplog.text assert f'"max_namespaces": 2049' in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", subsystem2, "--serial-number", serial, "--no-group-append"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + subsystem2, + "--serial-number", + serial, + "--no-group-append", + ] + ) assert f"Adding subsystem {subsystem2}: Successful" in caplog.text caplog.clear() cli(["--format", "json", "subsystem", "list"]) @@ -225,14 +339,14 @@ def test_create_subsystem(self, caplog, gateway): assert f'"nqn": "{subsystem2}"' in caplog.text caplog.clear() cli(["subsystem", "list"]) - assert f'{serial}' in caplog.text - assert f'{subsystem}' in caplog.text - assert f'{subsystem2}' in caplog.text + assert f"{serial}" in caplog.text + assert f"{subsystem}" in caplog.text + assert f"{subsystem2}" in caplog.text caplog.clear() cli(["--format", "plain", "subsystem", "list"]) - assert f'{serial}' in caplog.text - assert f'{subsystem}' in caplog.text - assert f'{subsystem2}' in caplog.text + assert f"{serial}" in caplog.text + assert f"{subsystem}" in caplog.text + assert f"{subsystem2}" in caplog.text caplog.clear() cli(["subsystem", "list", "--serial-number", "JUNK"]) assert f"No subsystem with serial number JUNK" in caplog.text @@ -275,7 +389,24 @@ def test_create_subsystem_with_discovery_nqn(self, caplog, gateway): def test_add_namespace_wrong_balancing_group(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image4, "--size", "16MB", "--rbd-create-image", "--load-balancing-group", "100", "--force"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image4, + "--size", + "16MB", + "--rbd-create-image", + "--load-balancing-group", + "100", + "--force", + ] + ) assert f"Failure adding namespace to {subsystem}:" in caplog.text assert f"Load balancing group 100 doesn't exist" in caplog.text @@ -283,7 +414,21 @@ def test_add_namespace_wrong_size(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", "junkimage", "--size", "0", "--rbd-create-image"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + "junkimage", + "--size", + "0", + "--rbd-create-image", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -292,7 +437,21 @@ def test_add_namespace_wrong_size(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", "junkimage", "--size", "1026KB", "--rbd-create-image"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + "junkimage", + "--size", + "1026KB", + "--rbd-create-image", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -302,8 +461,14 @@ def test_add_namespace_wrong_size(self, caplog, gateway): def test_add_namespace_wrong_size_grpc(self, caplog, gateway): gw, stub = gateway caplog.clear() - add_namespace_req = pb2.namespace_add_req(subsystem_nqn=subsystem, rbd_pool_name=pool, rbd_image_name="junkimage", - block_size=512, create_image=True, size=16*1024*1024+20) + add_namespace_req = pb2.namespace_add_req( + subsystem_nqn=subsystem, + rbd_pool_name=pool, + rbd_image_name="junkimage", + block_size=512, + create_image=True, + size=16 * 1024 * 1024 + 20, + ) ret = stub.namespace_add(add_namespace_req) assert ret.status != 0 assert f"Failure adding namespace" in caplog.text @@ -312,8 +477,14 @@ def test_add_namespace_wrong_size_grpc(self, caplog, gateway): def test_add_namespace_wrong_block_size(self, caplog, gateway): gw, stub = gateway caplog.clear() - add_namespace_req = pb2.namespace_add_req(subsystem_nqn=subsystem, rbd_pool_name=pool, rbd_image_name="junkimage", - create_image=True, size=16*1024*1024, force=True) + add_namespace_req = pb2.namespace_add_req( + subsystem_nqn=subsystem, + rbd_pool_name=pool, + rbd_image_name="junkimage", + create_image=True, + size=16 * 1024 * 1024, + force=True, + ) ret = stub.namespace_add(add_namespace_req) assert ret.status != 0 assert f"Failure adding namespace" in caplog.text @@ -321,10 +492,44 @@ def test_add_namespace_wrong_block_size(self, caplog, gateway): def test_add_namespace_double_uuid(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--uuid", uuid, "--size", "16MB", "--rbd-create-image", "--force"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--uuid", + uuid, + "--size", + "16MB", + "--rbd-create-image", + "--force", + ] + ) assert f"Adding namespace 1 to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image3, "--uuid", uuid, "--size", "16MB", "--rbd-create-image", "--force"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image3, + "--uuid", + uuid, + "--size", + "16MB", + "--rbd-create-image", + "--force", + ] + ) assert f"Failure adding namespace, UUID {uuid} is already in use" in caplog.text caplog.clear() cli(["namespace", "del", "--subsystem", subsystem, "--nsid", "1"]) @@ -332,10 +537,42 @@ def test_add_namespace_double_uuid(self, caplog, gateway): def test_add_namespace_double_nsid(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--size", "16MB", "--rbd-create-image", "--force"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--size", + "16MB", + "--rbd-create-image", + "--force", + ] + ) assert f"Adding namespace 1 to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image3, "--nsid", "1", "--size", "16MB", "--rbd-create-image", "--force"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image3, + "--nsid", + "1", + "--size", + "16MB", + "--rbd-create-image", + "--force", + ] + ) assert f"Failure adding namespace, NSID 1 is already in use" in caplog.text caplog.clear() cli(["namespace", "del", "--subsystem", subsystem, "--nsid", "1"]) @@ -343,31 +580,125 @@ def test_add_namespace_double_nsid(self, caplog, gateway): def test_add_namespace(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", "junk", "--rbd-image", image2, "--uuid", uuid, "--size", "16MB", "--rbd-create-image", "--load-balancing-group", anagrpid]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + "junk", + "--rbd-image", + image2, + "--uuid", + uuid, + "--size", + "16MB", + "--rbd-create-image", + "--load-balancing-group", + anagrpid, + ] + ) assert f"RBD pool junk doesn't exist" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--uuid", uuid, "--size", "16MB", "--rbd-create-image", "--load-balancing-group", anagrpid, "--force"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--uuid", + uuid, + "--size", + "16MB", + "--rbd-create-image", + "--load-balancing-group", + anagrpid, + "--force", + ] + ) assert f"Adding namespace 1 to {subsystem}: Successful" in caplog.text assert f"Allocated cluster name='cluster_context_{anagrpid}_0'" in caplog.text assert f"get_cluster cluster_name='cluster_context_{anagrpid}_0'" in caplog.text assert f"no_auto_visible: False" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--size", "36", "--rbd-create-image", "--load-balancing-group", anagrpid, "--force"]) - assert f"Image {pool}/{image2} already exists with a size of 16777216 bytes which differs from the requested size of 37748736 bytes" in caplog.text + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--size", + "36", + "--rbd-create-image", + "--load-balancing-group", + anagrpid, + "--force", + ] + ) + assert ( + f"Image {pool}/{image2} already exists with a size of 16777216 bytes which differs from the requested size of 37748736 bytes" + in caplog.text + ) assert f"Can't create RBD image {pool}/{image2}" in caplog.text caplog.clear() rc = 0 try: - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--block-size", "1024", "--size", "16MB", "--load-balancing-group", anagrpid]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--block-size", + "1024", + "--size", + "16MB", + "--load-balancing-group", + anagrpid, + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass - assert "size argument is not allowed for add command when RBD image creation is disabled" in caplog.text + assert ( + "size argument is not allowed for add command when RBD image creation is disabled" + in caplog.text + ) assert rc == 2 caplog.clear() rc = 0 try: - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--block-size", "1024", "--size=-16MB", "--rbd-create-image", "--load-balancing-group", anagrpid]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--block-size", + "1024", + "--size=-16MB", + "--rbd-create-image", + "--load-balancing-group", + anagrpid, + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -376,7 +707,25 @@ def test_add_namespace(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--block-size", "1024", "--size", "1x6MB", "--load-balancing-group", anagrpid, "--rbd-create-image"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--block-size", + "1024", + "--size", + "1x6MB", + "--load-balancing-group", + anagrpid, + "--rbd-create-image", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -385,7 +734,25 @@ def test_add_namespace(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--block-size", "1024", "--size", "16MiB", "--load-balancing-group", anagrpid, "--rbd-create-image"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--block-size", + "1024", + "--size", + "16MiB", + "--load-balancing-group", + anagrpid, + "--rbd-create-image", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -394,116 +761,485 @@ def test_add_namespace(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image2, "--block-size", "1024", "--size", "16mB", "--load-balancing-group", anagrpid, "--rbd-create-image"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image2, + "--block-size", + "1024", + "--size", + "16mB", + "--load-balancing-group", + anagrpid, + "--rbd-create-image", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass assert "must be numeric" in caplog.text assert rc == 2 caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image, "--block-size", "1024", "--load-balancing-group", anagrpid, "--force"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image, + "--block-size", + "1024", + "--load-balancing-group", + anagrpid, + "--force", + ] + ) assert f"Adding namespace 2 to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", nsid]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + nsid, + ] + ) assert f'"load_balancing_group": {anagrpid}' in caplog.text assert '"block_size": 512' in caplog.text assert f'"uuid": "{uuid}"' in caplog.text assert '"rw_ios_per_second": "0"' in caplog.text assert '"rw_mbytes_per_second": "0"' in caplog.text caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "2"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "2", + ] + ) assert f'"load_balancing_group": {anagrpid}' in caplog.text assert '"block_size": 1024' in caplog.text assert f'"uuid": "{uuid}"' not in caplog.text assert '"rw_ios_per_second": "0"' in caplog.text assert '"rw_mbytes_per_second": "0"' in caplog.text caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--uuid", uuid]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--uuid", + uuid, + ] + ) assert f'"uuid": "{uuid}"' in caplog.text caplog.clear() - cli(["namespace", "change_load_balancing_group", "--subsystem", subsystem, "--nsid", nsid, "--load-balancing-group", "10"]) - assert f"Failure changing load balancing group for namespace with NSID {nsid} in {subsystem}" in caplog.text + cli( + [ + "namespace", + "change_load_balancing_group", + "--subsystem", + subsystem, + "--nsid", + nsid, + "--load-balancing-group", + "10", + ] + ) + assert ( + f"Failure changing load balancing group for namespace with NSID {nsid} in {subsystem}" + in caplog.text + ) assert f"Load balancing group 10 doesn't exist" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image3, "--size", "4GB", "--rbd-create-image"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image3, + "--size", + "4GB", + "--rbd-create-image", + ] + ) assert f"Adding namespace 3 to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "3"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "3", + ] + ) assert '"rbd_image_size": "4294967296"' in caplog.text assert f'"load_balancing_group": {anagrpid}' in caplog.text def test_add_namespace_ipv6(self, caplog, gateway): caplog.clear() - cli(["--server-address", server_addr_ipv6, "namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image, "--load-balancing-group", anagrpid, "--nsid", "4", "--force"]) + cli( + [ + "--server-address", + server_addr_ipv6, + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image, + "--load-balancing-group", + anagrpid, + "--nsid", + "4", + "--force", + ] + ) assert f"Adding namespace 4 to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "4"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "4", + ] + ) assert f'"load_balancing_group": {anagrpid}' in caplog.text - cli(["--server-address", server_addr_ipv6, "namespace", "add", "--subsystem", subsystem, "--nsid", "5", "--rbd-pool", pool, "--rbd-image", image, "--load-balancing-group", anagrpid, "--force"]) + cli( + [ + "--server-address", + server_addr_ipv6, + "namespace", + "add", + "--subsystem", + subsystem, + "--nsid", + "5", + "--rbd-pool", + pool, + "--rbd-image", + image, + "--load-balancing-group", + anagrpid, + "--force", + ] + ) assert f"Adding namespace 5 to {subsystem}: Successful" in caplog.text assert f'will continue as the "force" argument was used' in caplog.text caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "5"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "5", + ] + ) assert f'"load_balancing_group": {anagrpid}' in caplog.text def test_add_namespace_same_image(self, caplog, gateway): caplog.clear() img_name = f"{image}_test" - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", img_name, "--size", "16MB", "--load-balancing-group", anagrpid, "--rbd-create-image", "--nsid", "6", "--uuid", uuid2]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + img_name, + "--size", + "16MB", + "--load-balancing-group", + anagrpid, + "--rbd-create-image", + "--nsid", + "6", + "--uuid", + uuid2, + ] + ) assert f"Adding namespace 6 to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", img_name, "--size", "16MB", "--load-balancing-group", anagrpid, "--rbd-create-image", "--nsid", "7"]) - assert f"RBD image {pool}/{img_name} is already used by a namespace" in caplog.text + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + img_name, + "--size", + "16MB", + "--load-balancing-group", + anagrpid, + "--rbd-create-image", + "--nsid", + "7", + ] + ) + assert ( + f"RBD image {pool}/{img_name} is already used by a namespace" in caplog.text + ) assert f"you can find the offending namespace by using" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", img_name, "--load-balancing-group", anagrpid, "--force", "--nsid", "7"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + img_name, + "--load-balancing-group", + anagrpid, + "--force", + "--nsid", + "7", + ] + ) assert f"Adding namespace 7 to {subsystem}: Successful" in caplog.text - assert f"RBD image {pool}/{img_name} is already used by a namespace" in caplog.text + assert ( + f"RBD image {pool}/{img_name} is already used by a namespace" in caplog.text + ) assert f'will continue as the "force" argument was used' in caplog.text def test_add_namespace_no_auto_visible(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image5, "--size", "16MB", "--rbd-create-image", "--no-auto-visible"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image5, + "--size", + "16MB", + "--rbd-create-image", + "--no-auto-visible", + ] + ) assert f"Adding namespace 8 to {subsystem}: Successful" in caplog.text assert f"no_auto_visible: True" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image6, "--size", "16MB", "--rbd-create-image", "--no-auto-visible"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image6, + "--size", + "16MB", + "--rbd-create-image", + "--no-auto-visible", + ] + ) assert f"Adding namespace 9 to {subsystem}: Successful" in caplog.text assert f"no_auto_visible: True" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image7, "--size", "16MB", "--rbd-create-image", "--no-auto-visible"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image7, + "--size", + "16MB", + "--rbd-create-image", + "--no-auto-visible", + ] + ) assert f"Adding namespace 10 to {subsystem}: Successful" in caplog.text assert f"no_auto_visible: True" in caplog.text def test_add_host_to_namespace(self, caplog, gateway): caplog.clear() - cli(["namespace", "add_host", "--subsystem", subsystem, "--nsid", "8", "--host-nqn", "nqn.2016-06.io.spdk:host8"]) - assert f"Adding host nqn.2016-06.io.spdk:host8 to namespace 8 on {subsystem}: Successful" in caplog.text + cli( + [ + "namespace", + "add_host", + "--subsystem", + subsystem, + "--nsid", + "8", + "--host-nqn", + "nqn.2016-06.io.spdk:host8", + ] + ) + assert ( + f"Adding host nqn.2016-06.io.spdk:host8 to namespace 8 on {subsystem}: Successful" + in caplog.text + ) def test_add_too_many_hosts_to_namespace(self, caplog, gateway): caplog.clear() - cli(["namespace", "add_host", "--subsystem", subsystem, "--nsid", "8", "--host-nqn", "nqn.2016-06.io.spdk:host9"]) - assert f"Failure adding host nqn.2016-06.io.spdk:host9 to namespace 8 on {subsystem}, maximal host count for namespace (1) was already reached" in caplog.text + cli( + [ + "namespace", + "add_host", + "--subsystem", + subsystem, + "--nsid", + "8", + "--host-nqn", + "nqn.2016-06.io.spdk:host9", + ] + ) + assert ( + f"Failure adding host nqn.2016-06.io.spdk:host9 to namespace 8 on {subsystem}, maximal host count for namespace (1) was already reached" + in caplog.text + ) def test_add_all_hosts_to_namespace(self, caplog, gateway): caplog.clear() - cli(["namespace", "add_host", "--subsystem", subsystem, "--nsid", "8", "--host-nqn", "*"]) - assert f"Failure adding host to namespace 8 on {subsystem}, host can't be \"*\"" in caplog.text + cli( + [ + "namespace", + "add_host", + "--subsystem", + subsystem, + "--nsid", + "8", + "--host-nqn", + "*", + ] + ) + assert ( + f'Failure adding host to namespace 8 on {subsystem}, host can\'t be "*"' + in caplog.text + ) def test_add_too_many_namespaces_to_a_subsystem(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image9, "--nsid", "3000", "--size", "16MB", "--rbd-create-image"]) - assert f"Failure adding namespace to {subsystem}: Requested NSID 3000 is bigger than the maximal one" in caplog.text + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image9, + "--nsid", + "3000", + "--size", + "16MB", + "--rbd-create-image", + ] + ) + assert ( + f"Failure adding namespace to {subsystem}: Requested NSID 3000 is bigger than the maximal one" + in caplog.text + ) assert f"Received request to delete bdev" in caplog.text caplog.clear() - cli(["subsystem", "add", "--subsystem", subsystem5, "--no-group-append", "--max-namespaces", "1"]) + cli( + [ + "subsystem", + "add", + "--subsystem", + subsystem5, + "--no-group-append", + "--max-namespaces", + "1", + ] + ) assert f"Adding subsystem {subsystem5}: Successful" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem5, "--rbd-pool", pool, "--rbd-image", image9, "--size", "16MB", "--rbd-create-image"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem5, + "--rbd-pool", + pool, + "--rbd-image", + image9, + "--size", + "16MB", + "--rbd-create-image", + ] + ) assert f"Adding namespace 1 to {subsystem5}: Successful" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem5, "--rbd-pool", pool, "--rbd-image", image10, "--size", "16MB", "--rbd-create-image"]) - assert f"Failure adding namespace to {subsystem5}: Subsystem's maximal number of namespaces (1) has already been reached" in caplog.text + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem5, + "--rbd-pool", + pool, + "--rbd-image", + image10, + "--size", + "16MB", + "--rbd-create-image", + ] + ) + assert ( + f"Failure adding namespace to {subsystem5}: Subsystem's maximal number of namespaces (1) has already been reached" + in caplog.text + ) assert f"Received request to delete bdev" in caplog.text caplog.clear() cli(["subsystem", "del", "--subsystem", subsystem5, "--force"]) @@ -511,75 +1247,279 @@ def test_add_too_many_namespaces_to_a_subsystem(self, caplog, gateway): def test_add_discovery_to_namespace(self, caplog, gateway): caplog.clear() - cli(["namespace", "add_host", "--subsystem", subsystem, "--nsid", "8", "--host-nqn", discovery_nqn]) - assert f"Failure adding host to namespace 8 on {subsystem}, host NQN can't be a discovery NQN" in caplog.text + cli( + [ + "namespace", + "add_host", + "--subsystem", + subsystem, + "--nsid", + "8", + "--host-nqn", + discovery_nqn, + ] + ) + assert ( + f"Failure adding host to namespace 8 on {subsystem}, host NQN can't be a discovery NQN" + in caplog.text + ) def test_add_junk_host_to_namespace(self, caplog, gateway): caplog.clear() - cli(["namespace", "add_host", "--subsystem", subsystem, "--nsid", "8", "--host-nqn", "junk"]) - assert f"Failure adding host junk to namespace 8 on {subsystem}, invalid host NQN" in caplog.text + cli( + [ + "namespace", + "add_host", + "--subsystem", + subsystem, + "--nsid", + "8", + "--host-nqn", + "junk", + ] + ) + assert ( + f"Failure adding host junk to namespace 8 on {subsystem}, invalid host NQN" + in caplog.text + ) def test_add_host_to_namespace_junk_subsystem(self, caplog, gateway): caplog.clear() - cli(["namespace", "add_host", "--subsystem", "junk", "--nsid", "8", "--host-nqn", "nqn.2016-06.io.spdk:hostXX"]) - assert f"Failure adding host nqn.2016-06.io.spdk:hostXX to namespace 8 on junk, invalid subsystem NQN" in caplog.text + cli( + [ + "namespace", + "add_host", + "--subsystem", + "junk", + "--nsid", + "8", + "--host-nqn", + "nqn.2016-06.io.spdk:hostXX", + ] + ) + assert ( + f"Failure adding host nqn.2016-06.io.spdk:hostXX to namespace 8 on junk, invalid subsystem NQN" + in caplog.text + ) def test_add_host_to_wrong_namespace(self, caplog, gateway): caplog.clear() - cli(["namespace", "add_host", "--subsystem", subsystem, "--nsid", "1", "--host-nqn", "nqn.2016-06.io.spdk:host10"]) - assert f"Failure adding host nqn.2016-06.io.spdk:host10 to namespace 1 on {subsystem}, namespace is visible to all hosts" in caplog.text + cli( + [ + "namespace", + "add_host", + "--subsystem", + subsystem, + "--nsid", + "1", + "--host-nqn", + "nqn.2016-06.io.spdk:host10", + ] + ) + assert ( + f"Failure adding host nqn.2016-06.io.spdk:host10 to namespace 1 on {subsystem}, namespace is visible to all hosts" + in caplog.text + ) def test_add_too_many_namespaces_with_hosts(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image8, "--size", "16MB", "--rbd-create-image", "--no-auto-visible"]) - assert f"Failure adding namespace to {subsystem}: Maximal number of namespaces which are not auto visible (3) has already been reached" in caplog.text - caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image8, "--size", "16MB", "--rbd-create-image"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image8, + "--size", + "16MB", + "--rbd-create-image", + "--no-auto-visible", + ] + ) + assert ( + f"Failure adding namespace to {subsystem}: Maximal number of namespaces which are not auto visible (3) has already been reached" + in caplog.text + ) + caplog.clear() + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image8, + "--size", + "16MB", + "--rbd-create-image", + ] + ) assert f"Adding namespace 11 to {subsystem}: Successful" in caplog.text def test_list_namespace_with_hosts(self, caplog, gateway): caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "8"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "8", + ] + ) assert f'"nsid": 8' in caplog.text assert f'"no_auto_visible": true' in caplog.text assert f'"nqn.2016-06.io.spdk:host8"' in caplog.text def test_list_namespace_with_no_hosts(self, caplog, gateway): caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "10"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "10", + ] + ) assert f'"nsid": 10' in caplog.text assert f'"no_auto_visible": true' in caplog.text assert f'"hosts": []' in caplog.text def test_add_too_many_namespaces(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image11, "--size", "16MB", "--rbd-create-image"]) - assert f"Failure adding namespace to {subsystem}: Maximal number of namespaces (11) has already been reached" in caplog.text + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image11, + "--size", + "16MB", + "--rbd-create-image", + ] + ) + assert ( + f"Failure adding namespace to {subsystem}: Maximal number of namespaces (11) has already been reached" + in caplog.text + ) def test_resize_namespace(self, caplog, gateway): gw, stub = gateway caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "6"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "6", + ] + ) assert f'"nsid": 6' in caplog.text assert '"block_size": 512' in caplog.text assert '"rbd_image_size": "16777216"' in caplog.text assert f'"uuid": "{uuid2}"' in caplog.text caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "2MB"]) - assert f"new size 2097152 bytes is smaller than current size 16777216 bytes" in caplog.text - caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "2"]) - assert f"new size 2097152 bytes is smaller than current size 16777216 bytes" in caplog.text - caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "3145728B"]) - assert f"new size 3145728 bytes is smaller than current size 16777216 bytes" in caplog.text - caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "32MB"]) - assert f"Resizing namespace 6 in {subsystem} to 32 MiB: Successful" in caplog.text + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size", + "2MB", + ] + ) + assert ( + f"new size 2097152 bytes is smaller than current size 16777216 bytes" + in caplog.text + ) + caplog.clear() + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size", + "2", + ] + ) + assert ( + f"new size 2097152 bytes is smaller than current size 16777216 bytes" + in caplog.text + ) + caplog.clear() + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size", + "3145728B", + ] + ) + assert ( + f"new size 3145728 bytes is smaller than current size 16777216 bytes" + in caplog.text + ) + caplog.clear() + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size", + "32MB", + ] + ) + assert ( + f"Resizing namespace 6 in {subsystem} to 32 MiB: Successful" in caplog.text + ) caplog.clear() rc = 0 try: - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "32mB"]) + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size", + "32mB", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -588,7 +1528,17 @@ def test_resize_namespace(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size=-32MB"]) + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size=-32MB", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -597,14 +1547,36 @@ def test_resize_namespace(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "3x2GB"]) + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size", + "3x2GB", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass assert "must be numeric" in caplog.text assert rc == 2 caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "6"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "6", + ] + ) assert f'"nsid": 6' in caplog.text assert '"block_size": 512' in caplog.text assert '"rbd_image_size": "33554432"' in caplog.text @@ -617,7 +1589,18 @@ def test_resize_namespace(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "resize", "--subsystem", subsystem, "--uuid", uuid2, "--size", "64MB"]) + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--uuid", + uuid2, + "--size", + "64MB", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -626,17 +1609,54 @@ def test_resize_namespace(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--uuid", uuid2, "--size", "64MB"]) + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--uuid", + uuid2, + "--size", + "64MB", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass assert "error: unrecognized arguments: --uuid" in caplog.text assert rc == 2 caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "64MB"]) - assert f"Resizing namespace 6 in {subsystem} to 64 MiB: Successful" in caplog.text - caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--uuid", uuid2]) + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size", + "64MB", + ] + ) + assert ( + f"Resizing namespace 6 in {subsystem} to 64 MiB: Successful" in caplog.text + ) + caplog.clear() + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--uuid", + uuid2, + ] + ) assert f'"nsid": 6' in caplog.text assert '"block_size": 512' in caplog.text assert '"rbd_image_size": "67108864"' in caplog.text @@ -647,37 +1667,130 @@ def test_resize_namespace(self, caplog, gateway): assert '"nsid": 4' not in caplog.text assert '"nsid": 5' not in caplog.text caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "12", "--size", "128MB"]) - assert f"Failure resizing namespace 12 on {subsystem}: Can't find namespace" in caplog.text - caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "6", "--size", "32MB"]) - assert f"Failure resizing namespace 6 on {subsystem}: new size 33554432 bytes is smaller than current size 67108864 bytes" in caplog.text + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "12", + "--size", + "128MB", + ] + ) + assert ( + f"Failure resizing namespace 12 on {subsystem}: Can't find namespace" + in caplog.text + ) + caplog.clear() + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "6", + "--size", + "32MB", + ] + ) + assert ( + f"Failure resizing namespace 6 on {subsystem}: new size 33554432 bytes is smaller than current size 67108864 bytes" + in caplog.text + ) ns = cli_test(["namespace", "list", "--subsystem", subsystem, "--nsid", "6"]) assert ns != None assert ns.status == 0 assert len(ns.namespaces) == 1 assert ns.namespaces[0].rbd_image_size == 67108864 caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "4", "--size", "6GB"]) - assert f"Resizing namespace 4 in {subsystem} to 6 GiB: Successful" in caplog.text - caplog.clear() - cli(["namespace", "resize", "--subsystem", subsystem, "--nsid", "4", "--size", "8192"]) - assert f"Resizing namespace 4 in {subsystem} to 8 GiB: Successful" in caplog.text + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "4", + "--size", + "6GB", + ] + ) + assert ( + f"Resizing namespace 4 in {subsystem} to 6 GiB: Successful" in caplog.text + ) + caplog.clear() + cli( + [ + "namespace", + "resize", + "--subsystem", + subsystem, + "--nsid", + "4", + "--size", + "8192", + ] + ) + assert ( + f"Resizing namespace 4 in {subsystem} to 8 GiB: Successful" in caplog.text + ) def test_set_namespace_qos_limits(self, caplog, gateway): caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "6"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "6", + ] + ) assert f'"nsid": 6' in caplog.text assert '"rw_ios_per_second": "0"' in caplog.text assert '"rw_mbytes_per_second": "0"' in caplog.text assert '"r_mbytes_per_second": "0"' in caplog.text assert '"w_mbytes_per_second": "0"' in caplog.text caplog.clear() - cli(["namespace", "set_qos", "--subsystem", subsystem, "--nsid", "6", "--rw-ios-per-second", "2000"]) - assert f"Setting QOS limits of namespace 6 in {subsystem}: Successful" in caplog.text - assert f"No previous QOS limits found, this is the first time the limits are set for namespace 6 on {subsystem}" in caplog.text - caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "6"]) + cli( + [ + "namespace", + "set_qos", + "--subsystem", + subsystem, + "--nsid", + "6", + "--rw-ios-per-second", + "2000", + ] + ) + assert ( + f"Setting QOS limits of namespace 6 in {subsystem}: Successful" + in caplog.text + ) + assert ( + f"No previous QOS limits found, this is the first time the limits are set for namespace 6 on {subsystem}" + in caplog.text + ) + caplog.clear() + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "6", + ] + ) assert f'"nsid": 6' in caplog.text assert f'"uuid": "{uuid2}"' in caplog.text assert '"rw_ios_per_second": "2000"' in caplog.text @@ -685,11 +1798,39 @@ def test_set_namespace_qos_limits(self, caplog, gateway): assert '"r_mbytes_per_second": "0"' in caplog.text assert '"w_mbytes_per_second": "0"' in caplog.text caplog.clear() - cli(["namespace", "set_qos", "--subsystem", subsystem, "--nsid", "6", "--rw-megabytes-per-second", "30"]) - assert f"Setting QOS limits of namespace 6 in {subsystem}: Successful" in caplog.text - assert f"No previous QOS limits found, this is the first time the limits are set for namespace 6 on {subsystem}" not in caplog.text - caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--uuid", uuid2]) + cli( + [ + "namespace", + "set_qos", + "--subsystem", + subsystem, + "--nsid", + "6", + "--rw-megabytes-per-second", + "30", + ] + ) + assert ( + f"Setting QOS limits of namespace 6 in {subsystem}: Successful" + in caplog.text + ) + assert ( + f"No previous QOS limits found, this is the first time the limits are set for namespace 6 on {subsystem}" + not in caplog.text + ) + caplog.clear() + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--uuid", + uuid2, + ] + ) assert f'"uuid": "{uuid2}"' in caplog.text assert f'"nsid": 6' in caplog.text assert '"rw_ios_per_second": "2000"' in caplog.text @@ -697,12 +1838,41 @@ def test_set_namespace_qos_limits(self, caplog, gateway): assert '"r_mbytes_per_second": "0"' in caplog.text assert '"w_mbytes_per_second": "0"' in caplog.text caplog.clear() - cli(["namespace", "set_qos", "--subsystem", subsystem, "--nsid", "6", - "--r-megabytes-per-second", "15", "--w-megabytes-per-second", "25"]) - assert f"Setting QOS limits of namespace 6 in {subsystem}: Successful" in caplog.text - assert f"No previous QOS limits found, this is the first time the limits are set for namespace 6 on {subsystem}" not in caplog.text - caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "6"]) + cli( + [ + "namespace", + "set_qos", + "--subsystem", + subsystem, + "--nsid", + "6", + "--r-megabytes-per-second", + "15", + "--w-megabytes-per-second", + "25", + ] + ) + assert ( + f"Setting QOS limits of namespace 6 in {subsystem}: Successful" + in caplog.text + ) + assert ( + f"No previous QOS limits found, this is the first time the limits are set for namespace 6 on {subsystem}" + not in caplog.text + ) + caplog.clear() + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "6", + ] + ) assert f'"nsid": 6' in caplog.text assert '"rw_ios_per_second": "2000"' in caplog.text assert '"rw_mbytes_per_second": "30"' in caplog.text @@ -720,19 +1890,44 @@ def test_set_namespace_qos_limits(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "set_qos", "--subsystem", subsystem, "--nsid", "6", "--w-megabytes-per-second", "JUNK"]) + cli( + [ + "namespace", + "set_qos", + "--subsystem", + subsystem, + "--nsid", + "6", + "--w-megabytes-per-second", + "JUNK", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass - assert "error: argument --w-megabytes-per-second: invalid int value: 'JUNK'" in caplog.text + assert ( + "error: argument --w-megabytes-per-second: invalid int value: 'JUNK'" + in caplog.text + ) assert rc == 2 def test_namespace_io_stats(self, caplog, gateway): caplog.clear() cli(["namespace", "get_io_stats", "--subsystem", subsystem, "--nsid", "6"]) - assert f'IO statistics for namespace 6 in {subsystem}' in caplog.text - caplog.clear() - cli(["--format", "json", "namespace", "get_io_stats", "--subsystem", subsystem, "--nsid", "6"]) + assert f"IO statistics for namespace 6 in {subsystem}" in caplog.text + caplog.clear() + cli( + [ + "--format", + "json", + "namespace", + "get_io_stats", + "--subsystem", + subsystem, + "--nsid", + "6", + ] + ) assert f'"status": 0' in caplog.text assert f'"subsystem_nqn": "{subsystem}"' in caplog.text assert f'"nsid": 6' in caplog.text @@ -745,7 +1940,18 @@ def test_namespace_io_stats(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["namespace", "get_io_stats", "--subsystem", subsystem, "--uuid", uuid2, "--nsid", "1"]) + cli( + [ + "namespace", + "get_io_stats", + "--subsystem", + subsystem, + "--uuid", + uuid2, + "--nsid", + "1", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -754,7 +1960,16 @@ def test_namespace_io_stats(self, caplog, gateway): caplog.clear() rc = 0 try: - cli(["--format", "json", "namespace", "get_io_stats", "--subsystem", subsystem]) + cli( + [ + "--format", + "json", + "namespace", + "get_io_stats", + "--subsystem", + subsystem, + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -770,12 +1985,16 @@ def test_add_host(self, caplog, host): except SystemExit as sysex: rc = int(str(sysex)) pass - assert "error: the following arguments are required: --host-nqn/-t" in caplog.text + assert ( + "error: the following arguments are required: --host-nqn/-t" in caplog.text + ) assert rc == 2 caplog.clear() cli(["host", "add", "--subsystem", subsystem, "--host-nqn", host]) if host == "*": - assert f"Allowing open host access to {subsystem}: Successful" in caplog.text + assert ( + f"Allowing open host access to {subsystem}: Successful" in caplog.text + ) else: assert f"Adding host {host} to {subsystem}: Successful" in caplog.text @@ -784,20 +2003,49 @@ def test_add_host_invalid_nqn(self, caplog): cli(["host", "add", "--subsystem", subsystem, "--host-nqn", "nqn.2016"]) assert f'NQN "nqn.2016" is too short, minimal length is 11' in caplog.text caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", "nqn.2X16-06.io.spdk:host1"]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + "nqn.2X16-06.io.spdk:host1", + ] + ) assert f"invalid date code" in caplog.text caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", "nqn.2016-06.io.spdk:host1_X"]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + "nqn.2016-06.io.spdk:host1_X", + ] + ) assert f"Invalid host NQN" in caplog.text assert f"contains invalid characters" in caplog.text caplog.clear() - cli(["host", "add", "--subsystem", f"{subsystem}_X", "--host-nqn", "nqn.2016-06.io.spdk:host2"]) + cli( + [ + "host", + "add", + "--subsystem", + f"{subsystem}_X", + "--host-nqn", + "nqn.2016-06.io.spdk:host2", + ] + ) assert f"Invalid subsystem NQN" in caplog.text assert f"contains invalid characters" in caplog.text def test_host_list(self, caplog): caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", host5, host6, host7]) + cli( + ["host", "add", "--subsystem", subsystem, "--host-nqn", host5, host6, host7] + ) assert f"Adding host {host5} to {subsystem}: Successful" in caplog.text assert f"Adding host {host6} to {subsystem}: Successful" in caplog.text assert f"Adding host {host7} to {subsystem}: Successful" in caplog.text @@ -805,24 +2053,50 @@ def test_host_list(self, caplog): @pytest.mark.parametrize("listener", listener_list) def test_create_listener(self, caplog, listener, gateway): caplog.clear() - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + listener) + cli( + ["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) assert "ipv4" in caplog.text.lower() - assert f"Adding {subsystem} listener at {listener[1]}:{listener[3]}: Successful" in caplog.text - + assert ( + f"Adding {subsystem} listener at {listener[1]}:{listener[3]}: Successful" + in caplog.text + ) @pytest.mark.parametrize("listener_ipv6", listener_list_ipv6) def test_create_listener_ipv6(self, caplog, listener_ipv6, gateway): caplog.clear() - cli(["--server-address", server_addr_ipv6, "listener", "add", "--subsystem", subsystem, "--host-name", host_name] + listener_ipv6) + cli( + [ + "--server-address", + server_addr_ipv6, + "listener", + "add", + "--subsystem", + subsystem, + "--host-name", + host_name, + ] + + listener_ipv6 + ) assert "ipv6" in caplog.text.lower() - assert f"Adding {subsystem} listener at [{listener_ipv6[1]}]:{listener_ipv6[3]}: Successful" in caplog.text + assert ( + f"Adding {subsystem} listener at [{listener_ipv6[1]}]:{listener_ipv6[3]}: Successful" + in caplog.text + ) @pytest.mark.parametrize("listener", listener_list_no_port) def test_create_listener_no_port(self, caplog, listener, gateway): caplog.clear() - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + listener) + cli( + ["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) assert "ipv4" in caplog.text.lower() - assert f"Adding {subsystem} listener at {listener[1]}:4420: Successful" in caplog.text + assert ( + f"Adding {subsystem} listener at {listener[1]}:4420: Successful" + in caplog.text + ) @pytest.mark.parametrize("listener", listener_list) @pytest.mark.parametrize("listener_ipv6", listener_list_ipv6) @@ -842,7 +2116,10 @@ def test_create_listener_negative_port(self, caplog, listener, gateway): caplog.clear() rc = 0 try: - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + listener) + cli( + ["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -854,7 +2131,10 @@ def test_create_listener_port_too_big(self, caplog, listener, gateway): caplog.clear() rc = 0 try: - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + listener) + cli( + ["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -865,14 +2145,19 @@ def test_create_listener_port_too_big(self, caplog, listener, gateway): def test_create_listener_wrong_hostname(self, caplog, listener, gateway): caplog.clear() cli(["listener", "add", "--subsystem", subsystem] + listener) - assert f"Gateway's host name must match current host ({host_name})" in caplog.text + assert ( + f"Gateway's host name must match current host ({host_name})" in caplog.text + ) @pytest.mark.parametrize("listener", listener_list_invalid_adrfam) def test_create_listener_invalid_adrfam(self, caplog, listener, gateway): caplog.clear() rc = 0 try: - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + listener) + cli( + ["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -885,6 +2170,7 @@ def test_create_listener_on_discovery(self, caplog, listener, gateway): cli(["listener", "add", "--host-name", host_name] + listener) assert "Can't create a listener for a discovery subsystem" in caplog.text + class TestDelete: @pytest.mark.parametrize("host", host_list) def test_remove_host(self, caplog, host, gateway): @@ -895,28 +2181,60 @@ def test_remove_host(self, caplog, host, gateway): except SystemExit as sysex: rc = int(str(sysex)) pass - assert "error: the following arguments are required: --host-nqn/-t" in caplog.text + assert ( + "error: the following arguments are required: --host-nqn/-t" in caplog.text + ) assert rc == 2 caplog.clear() cli(["host", "del", "--subsystem", subsystem, "--host-nqn", host]) if host == "*": - assert f"Disabling open host access to {subsystem}: Successful" in caplog.text + assert ( + f"Disabling open host access to {subsystem}: Successful" in caplog.text + ) else: - assert f"Removing host {host} access from {subsystem}: Successful" in caplog.text + assert ( + f"Removing host {host} access from {subsystem}: Successful" + in caplog.text + ) def remove_host_list(self, caplog): caplog.clear() - cli(["host", "del", "--subsystem", subsystem, "--host-nqn", "nqn.2016-06.io.spdk:host5", "nqn.2016-06.io.spdk:host6", "nqn.2016-06.io.spdk:host7"]) - assert f"Removing host nqn.2016-06.io.spdk:host5 access from {subsystem}: Successful" in caplog.text - assert f"Removing host nqn.2016-06.io.spdk:host6 access from {subsystem}: Successful" in caplog.text - assert f"Removing host nqn.2016-06.io.spdk:host7 access from {subsystem}: Successful" in caplog.text + cli( + [ + "host", + "del", + "--subsystem", + subsystem, + "--host-nqn", + "nqn.2016-06.io.spdk:host5", + "nqn.2016-06.io.spdk:host6", + "nqn.2016-06.io.spdk:host7", + ] + ) + assert ( + f"Removing host nqn.2016-06.io.spdk:host5 access from {subsystem}: Successful" + in caplog.text + ) + assert ( + f"Removing host nqn.2016-06.io.spdk:host6 access from {subsystem}: Successful" + in caplog.text + ) + assert ( + f"Removing host nqn.2016-06.io.spdk:host7 access from {subsystem}: Successful" + in caplog.text + ) @pytest.mark.parametrize("listener", listener_list) - def test_delete_listener_using_wild_hostname_no_force(self, caplog, listener, gateway): + def test_delete_listener_using_wild_hostname_no_force( + self, caplog, listener, gateway + ): caplog.clear() rc = 0 try: - cli(["listener", "del", "--subsystem", subsystem, "--host-name", "*"] + listener) + cli( + ["listener", "del", "--subsystem", subsystem, "--host-name", "*"] + + listener + ) except SystemExit as sysex: rc = int(str(sysex)) pass @@ -926,43 +2244,104 @@ def test_delete_listener_using_wild_hostname_no_force(self, caplog, listener, ga @pytest.mark.parametrize("listener", listener_list) def test_delete_listener(self, caplog, listener, gateway): caplog.clear() - cli(["listener", "del", "--force", "--subsystem", subsystem, "--host-name", host_name] + listener) - assert f"Deleting listener {listener[1]}:{listener[3]} from {subsystem} for host {host_name}: Successful" in caplog.text + cli( + [ + "listener", + "del", + "--force", + "--subsystem", + subsystem, + "--host-name", + host_name, + ] + + listener + ) + assert ( + f"Deleting listener {listener[1]}:{listener[3]} from {subsystem} for host {host_name}: Successful" + in caplog.text + ) @pytest.mark.parametrize("listener_ipv6", listener_list_ipv6) def test_delete_listener_ipv6(self, caplog, listener_ipv6, gateway): caplog.clear() - cli(["--server-address", server_addr_ipv6, "listener", "del", "--subsystem", subsystem, "--host-name", host_name] + listener_ipv6) - assert f"Deleting listener [{listener_ipv6[1]}]:{listener_ipv6[3]} from {subsystem} for host {host_name}: Successful" in caplog.text + cli( + [ + "--server-address", + server_addr_ipv6, + "listener", + "del", + "--subsystem", + subsystem, + "--host-name", + host_name, + ] + + listener_ipv6 + ) + assert ( + f"Deleting listener [{listener_ipv6[1]}]:{listener_ipv6[3]} from {subsystem} for host {host_name}: Successful" + in caplog.text + ) @pytest.mark.parametrize("listener", listener_list_no_port) def test_delete_listener_no_port(self, caplog, listener, gateway): caplog.clear() rc = 0 try: - cli(["listener", "del", "--subsystem", subsystem, "--host-name", host_name] + listener) + cli( + ["listener", "del", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) except SystemExit as sysex: rc = int(str(sysex)) pass - assert "error: the following arguments are required: --trsvcid/-s" in caplog.text + assert ( + "error: the following arguments are required: --trsvcid/-s" in caplog.text + ) assert rc == 2 caplog.clear() - cli(["listener", "del", "--trsvcid", "4420", "--subsystem", subsystem, "--host-name", host_name] + listener) - assert f"Deleting listener {listener[1]}:4420 from {subsystem} for host {host_name}: Successful" in caplog.text + cli( + [ + "listener", + "del", + "--trsvcid", + "4420", + "--subsystem", + subsystem, + "--host-name", + host_name, + ] + + listener + ) + assert ( + f"Deleting listener {listener[1]}:4420 from {subsystem} for host {host_name}: Successful" + in caplog.text + ) @pytest.mark.parametrize("listener", listener_list) def test_delete_listener_using_wild_hostname(self, caplog, listener, gateway): caplog.clear() - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + listener) + cli( + ["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) assert "ipv4" in caplog.text.lower() - assert f"Adding {subsystem} listener at {listener[1]}:{listener[3]}: Successful" in caplog.text + assert ( + f"Adding {subsystem} listener at {listener[1]}:{listener[3]}: Successful" + in caplog.text + ) cli(["--format", "json", "listener", "list", "--subsystem", subsystem]) assert f'"host_name": "{host_name}"' in caplog.text assert f'"traddr": "{listener[1]}"' in caplog.text assert f'"trsvcid": {listener[3]}' in caplog.text caplog.clear() - cli(["listener", "del", "--force", "--subsystem", subsystem, "--host-name", "*"] + listener) - assert f"Deleting listener {listener[1]}:{listener[3]} from {subsystem} for all hosts: Successful" in caplog.text + cli( + ["listener", "del", "--force", "--subsystem", subsystem, "--host-name", "*"] + + listener + ) + assert ( + f"Deleting listener {listener[1]}:{listener[3]} from {subsystem} for all hosts: Successful" + in caplog.text + ) caplog.clear() cli(["--format", "json", "listener", "list", "--subsystem", subsystem]) assert f'"trsvcid": {listener[3]}' not in caplog.text @@ -970,7 +2349,9 @@ def test_delete_listener_using_wild_hostname(self, caplog, listener, gateway): def test_remove_namespace(self, caplog, gateway): gw, stub = gateway caplog.clear() - ns_list = cli_test(["namespace", "list", "--subsystem", subsystem, "--nsid", "6"]) + ns_list = cli_test( + ["namespace", "list", "--subsystem", subsystem, "--nsid", "6"] + ) assert ns_list != None assert ns_list.status == 0 assert len(ns_list.namespaces) == 1 @@ -1039,6 +2420,7 @@ def test_delete_subsystem_with_discovery_nqn(self, caplog, gateway): assert "Can't delete a discovery subsystem" in caplog.text assert rc == 2 + class TestCreateWithAna: def test_create_subsystem_ana(self, caplog, gateway): caplog.clear() @@ -1054,27 +2436,67 @@ def test_create_subsystem_ana(self, caplog, gateway): def test_add_namespace_ana(self, caplog, gateway): caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image, "--load-balancing-group", anagrpid, "--force", "--nsid", "10"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image, + "--load-balancing-group", + anagrpid, + "--force", + "--nsid", + "10", + ] + ) assert f"Adding namespace 10 to {subsystem}: Successful" in caplog.text assert f"get_cluster cluster_name='cluster_context_{anagrpid}_0'" in caplog.text caplog.clear() - cli(["--format", "json", "namespace", "list", "--subsystem", subsystem, "--nsid", "10"]) + cli( + [ + "--format", + "json", + "namespace", + "list", + "--subsystem", + subsystem, + "--nsid", + "10", + ] + ) assert f'"load_balancing_group": {anagrpid}' in caplog.text @pytest.mark.parametrize("listener", listener_list) def test_create_listener_ana(self, caplog, listener, gateway): caplog.clear() - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + listener) + cli( + ["listener", "add", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) assert "ipv4" in caplog.text.lower() - assert f"Adding {subsystem} listener at {listener[1]}:{listener[3]}: Successful" in caplog.text + assert ( + f"Adding {subsystem} listener at {listener[1]}:{listener[3]}: Successful" + in caplog.text + ) + class TestDeleteAna: @pytest.mark.parametrize("listener", listener_list) def test_delete_listener_ana(self, caplog, listener, gateway): caplog.clear() - cli(["listener", "del", "--subsystem", subsystem, "--host-name", host_name] + listener) - assert f"Deleting listener {listener[1]}:{listener[3]} from {subsystem} for host {host_name}: Successful" in caplog.text + cli( + ["listener", "del", "--subsystem", subsystem, "--host-name", host_name] + + listener + ) + assert ( + f"Deleting listener {listener[1]}:{listener[3]} from {subsystem} for host {host_name}: Successful" + in caplog.text + ) def test_remove_namespace_ana(self, caplog, gateway): caplog.clear() @@ -1089,12 +2511,16 @@ def test_delete_subsystem_ana(self, caplog, gateway): cli(["subsystem", "list"]) assert "No subsystems" in caplog.text + class TestSubsysWithGroupName: def test_create_subsys_group_name(self, caplog, gateway): caplog.clear() cli(["subsystem", "add", "--subsystem", subsystem3]) assert f"Adding subsystem {subsystem3}.{group_name}: Successful" in caplog.text - assert f"Subsystem NQN was changed to {subsystem3}.{group_name}, adding the group name" in caplog.text + assert ( + f"Subsystem NQN was changed to {subsystem3}.{group_name}, adding the group name" + in caplog.text + ) assert f"Adding subsystem {subsystem3}: Successful" not in caplog.text cli(["--format", "json", "subsystem", "list"]) assert f'"nqn": "{subsystem3}.{group_name}"' in caplog.text @@ -1103,11 +2529,14 @@ def test_create_subsys_group_name(self, caplog, gateway): cli(["subsystem", "add", "--subsystem", subsystem4, "--no-group-append"]) assert f"Adding subsystem {subsystem4}: Successful" in caplog.text assert f"Subsystem NQN will not be changed" in caplog.text - assert f"Adding subsystem {subsystem4}.{group_name}: Successful" not in caplog.text + assert ( + f"Adding subsystem {subsystem4}.{group_name}: Successful" not in caplog.text + ) cli(["--format", "json", "subsystem", "list"]) assert f'"nqn": "{subsystem4}.{group_name}"' not in caplog.text assert f'"nqn": "{subsystem4}"' in caplog.text + class TestTooManySubsystemsAndHosts: def test_add_too_many_subsystem(self, caplog, gateway): caplog.clear() @@ -1115,7 +2544,10 @@ def test_add_too_many_subsystem(self, caplog, gateway): assert f"Adding subsystem {subsystem6}: Successful" in caplog.text caplog.clear() cli(["subsystem", "add", "--subsystem", subsystem7, "--no-group-append"]) - assert f"Failure creating subsystem {subsystem7}: Maximal number of subsystems (3) has already been reached" in caplog.text + assert ( + f"Failure creating subsystem {subsystem7}: Maximal number of subsystems (3) has already been reached" + in caplog.text + ) def test_too_many_hosts(self, caplog, gateway): caplog.clear() @@ -1132,7 +2564,11 @@ def test_too_many_hosts(self, caplog, gateway): assert f"Adding host {host4} to {subsystem6}: Successful" in caplog.text caplog.clear() cli(["host", "add", "--subsystem", subsystem6, "--host-nqn", host5]) - assert f"Failure adding host {host5} to {subsystem6}: Maximal number of hosts for subsystem (4) has already been reached" in caplog.text + assert ( + f"Failure adding host {host5} to {subsystem6}: Maximal number of hosts for subsystem (4) has already been reached" + in caplog.text + ) + class TestGwLogLevel: def test_gw_log_level(self, caplog, gateway): @@ -1170,14 +2606,15 @@ def test_gw_log_level(self, caplog, gateway): cli(["gw", "get_log_level"]) assert 'Gateway log level is "debug"' in caplog.text + class TestSPDKLOg: def test_log_flags(self, caplog, gateway): caplog.clear() cli(["spdk_log_level", "get"]) assert 'SPDK nvmf log flag "nvmf" is disabled' in caplog.text assert 'SPDK nvmf log flag "nvmf_tcp" is disabled' in caplog.text - assert 'SPDK log level is NOTICE' in caplog.text - assert 'SPDK log print level is INFO' in caplog.text + assert "SPDK log level is NOTICE" in caplog.text + assert "SPDK log print level is INFO" in caplog.text caplog.clear() cli(["spdk_log_level", "set"]) assert "Set SPDK log levels and nvmf log flags: Successful" in caplog.text @@ -1185,8 +2622,8 @@ def test_log_flags(self, caplog, gateway): cli(["spdk_log_level", "get"]) assert 'SPDK nvmf log flag "nvmf" is enabled' in caplog.text assert 'SPDK nvmf log flag "nvmf_tcp" is enabled' in caplog.text - assert 'SPDK log level is NOTICE' in caplog.text - assert 'SPDK log print level is INFO' in caplog.text + assert "SPDK log level is NOTICE" in caplog.text + assert "SPDK log print level is INFO" in caplog.text caplog.clear() cli(["spdk_log_level", "set", "--level", "DEBUG"]) assert "Set SPDK log levels and nvmf log flags: Successful" in caplog.text @@ -1194,8 +2631,8 @@ def test_log_flags(self, caplog, gateway): cli(["spdk_log_level", "get"]) assert 'SPDK nvmf log flag "nvmf" is enabled' in caplog.text assert 'SPDK nvmf log flag "nvmf_tcp" is enabled' in caplog.text - assert 'SPDK log level is DEBUG' in caplog.text - assert 'SPDK log print level is INFO' in caplog.text + assert "SPDK log level is DEBUG" in caplog.text + assert "SPDK log print level is INFO" in caplog.text caplog.clear() cli(["spdk_log_level", "set", "--print", "error"]) assert "Set SPDK log levels and nvmf log flags: Successful" in caplog.text @@ -1203,8 +2640,8 @@ def test_log_flags(self, caplog, gateway): cli(["spdk_log_level", "get"]) assert 'SPDK nvmf log flag "nvmf" is enabled' in caplog.text assert 'SPDK nvmf log flag "nvmf_tcp" is enabled' in caplog.text - assert 'SPDK log level is DEBUG' in caplog.text - assert 'SPDK log print level is ERROR' in caplog.text + assert "SPDK log level is DEBUG" in caplog.text + assert "SPDK log print level is ERROR" in caplog.text caplog.clear() cli(["spdk_log_level", "disable"]) assert "Disable SPDK nvmf log flags: Successful" in caplog.text @@ -1212,8 +2649,8 @@ def test_log_flags(self, caplog, gateway): cli(["spdk_log_level", "get"]) assert 'SPDK nvmf log flag "nvmf" is disabled' in caplog.text assert 'SPDK nvmf log flag "nvmf_tcp" is disabled' in caplog.text - assert 'SPDK log level is NOTICE' in caplog.text - assert 'SPDK log print level is INFO' in caplog.text + assert "SPDK log level is NOTICE" in caplog.text + assert "SPDK log print level is INFO" in caplog.text caplog.clear() rc = 0 try: diff --git a/tests/test_cli_change_keys.py b/tests/test_cli_change_keys.py index e053c1a5..2b976abb 100644 --- a/tests/test_cli_change_keys.py +++ b/tests/test_cli_change_keys.py @@ -26,6 +26,7 @@ hostpsk1 = "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" config = "ceph-nvmeof.conf" + @pytest.fixture(scope="module") def two_gateways(config): """Sets up and tears down two Gateways""" @@ -56,9 +57,17 @@ def two_gateways(config): configB.config["spdk"]["tgt_cmd_extra_args"] = "-m 0x0C" ceph_utils = CephUtils(config) - with (GatewayServer(configA) as gatewayA, GatewayServer(configB) as gatewayB): - ceph_utils.execute_ceph_monitor_command("{" + f'"prefix":"nvme-gw create", "id": "{nameA}", "pool": "{pool}", "group": ""' + "}") - ceph_utils.execute_ceph_monitor_command("{" + f'"prefix":"nvme-gw create", "id": "{nameB}", "pool": "{pool}", "group": ""' + "}") + with GatewayServer(configA) as gatewayA, GatewayServer(configB) as gatewayB: + ceph_utils.execute_ceph_monitor_command( + "{" + + f'"prefix":"nvme-gw create", "id": "{nameA}", "pool": "{pool}", "group": ""' + + "}" + ) + ceph_utils.execute_ceph_monitor_command( + "{" + + f'"prefix":"nvme-gw create", "id": "{nameB}", "pool": "{pool}", "group": ""' + + "}" + ) gatewayA.serve() gatewayB.serve() @@ -73,6 +82,7 @@ def two_gateways(config): gatewayA.server.stop(grace=1) gatewayB.server.stop(grace=1) + def test_change_host_key(caplog, two_gateways): gatewayA, stubA, gatewayB, stubB = two_gateways gwA = gatewayA.gateway_rpc @@ -81,25 +91,102 @@ def test_change_host_key(caplog, two_gateways): cli(["--server-port", "5501", "subsystem", "add", "--subsystem", subsystem]) assert f"create_subsystem {subsystem}: True" in caplog.text caplog.clear() - cli(["--server-port", "5501", "host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn1]) + cli( + [ + "--server-port", + "5501", + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn1, + ] + ) assert f"Adding host {hostnqn1} to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["--server-port", "5501", "host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn2, "--dhchap-key", key1]) + cli( + [ + "--server-port", + "5501", + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn2, + "--dhchap-key", + key1, + ] + ) assert f"Adding host {hostnqn2} to {subsystem}: Successful" in caplog.text - assert f"Host {hostnqn2} has a DH-HMAC-CHAP key but subsystem {subsystem} has no key, a unidirectional authentication will be used" in caplog.text + assert ( + f"Host {hostnqn2} has a DH-HMAC-CHAP key but subsystem {subsystem} has no key, a unidirectional authentication will be used" + in caplog.text + ) caplog.clear() - cli(["--server-port", "5501", "host", "change_key", "--subsystem", subsystem, "--host-nqn", hostnqn1, "--dhchap-key", key2]) - assert f"Changing key for host {hostnqn1} on subsystem {subsystem}: Successful" in caplog.text - assert f"Host {hostnqn1} has a DH-HMAC-CHAP key but subsystem {subsystem} has no key, a unidirectional authentication will be used" in caplog.text + cli( + [ + "--server-port", + "5501", + "host", + "change_key", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn1, + "--dhchap-key", + key2, + ] + ) + assert ( + f"Changing key for host {hostnqn1} on subsystem {subsystem}: Successful" + in caplog.text + ) + assert ( + f"Host {hostnqn1} has a DH-HMAC-CHAP key but subsystem {subsystem} has no key, a unidirectional authentication will be used" + in caplog.text + ) time.sleep(15) - assert f"Received request to change inband authentication key for host {hostnqn1} on subsystem {subsystem}, dhchap: {key2}, context: 6}" return hostnqn + def create_resource_by_index(stub, i, caplog): subsystem = f"{subsystem_prefix}{i}" - subsystem_req = pb2.create_subsystem_req(subsystem_nqn=subsystem, max_namespaces=256, enable_ha=True, no_group_append=True) + subsystem_req = pb2.create_subsystem_req( + subsystem_nqn=subsystem, + max_namespaces=256, + enable_ha=True, + no_group_append=True, + ) ret_subsystem = stub.create_subsystem(subsystem_req) assert ret_subsystem.status == 0 if caplog != None: assert f"create_subsystem {subsystem}: True" in caplog.text assert f"Failure creating subsystem {subsystem}" not in caplog.text - namespace_req = pb2.namespace_add_req(subsystem_nqn=subsystem, - rbd_pool_name=pool, rbd_image_name=image, block_size=4096, - create_image=True, size=16*1024*1024, force=True) + namespace_req = pb2.namespace_add_req( + subsystem_nqn=subsystem, + rbd_pool_name=pool, + rbd_image_name=image, + block_size=4096, + create_image=True, + size=16 * 1024 * 1024, + force=True, + ) ret_namespace = stub.namespace_add(namespace_req) assert ret_namespace.status == 0 hostnqn = build_host_nqn(i) @@ -164,6 +241,7 @@ def create_resource_by_index(stub, i, caplog): assert f"Failure allowing open host access to {subsystem}" not in caplog.text assert f"Failure adding host {hostnqn} to {subsystem}" not in caplog.text + def check_resource_by_index(i, subsys_list, hosts_info): subsystem = f"{subsystem_prefix}{i}" hostnqn = build_host_nqn(i) @@ -182,9 +260,9 @@ def check_resource_by_index(i, subsys_list, hosts_info): pass assert found_host + def test_multi_gateway_omap_reread(config, conn_omap_reread, caplog): - """Tests reading out of date OMAP file - """ + """Tests reading out of date OMAP file""" stubA, stubB, gatewayA, gatewayB = conn_omap_reread nqn = subsystem_prefix + "X1" serial = "Ceph00000000000001" @@ -192,10 +270,23 @@ def test_multi_gateway_omap_reread(config, conn_omap_reread, caplog): num_subsystems = 2 # Send requests to create a subsystem with one namespace to GatewayA - subsystem_req = pb2.create_subsystem_req(subsystem_nqn=nqn, serial_number=serial, max_namespaces=256, enable_ha=True, no_group_append=True) - namespace_req = pb2.namespace_add_req(subsystem_nqn=nqn, nsid=nsid, - rbd_pool_name=pool, rbd_image_name=image, block_size=4096, - create_image=True, size=16*1024*1024, force=True) + subsystem_req = pb2.create_subsystem_req( + subsystem_nqn=nqn, + serial_number=serial, + max_namespaces=256, + enable_ha=True, + no_group_append=True, + ) + namespace_req = pb2.namespace_add_req( + subsystem_nqn=nqn, + nsid=nsid, + rbd_pool_name=pool, + rbd_image_name=image, + block_size=4096, + create_image=True, + size=16 * 1024 * 1024, + force=True, + ) subsystem_list_req = pb2.list_subsystems_req() ret_subsystem = stubA.create_subsystem(subsystem_req) @@ -204,38 +295,62 @@ def test_multi_gateway_omap_reread(config, conn_omap_reread, caplog): assert ret_namespace.status == 0 # Until we create some resource on GW-B it shouldn't still have the resrouces created on GW-A, only the discovery subsystem - listB = json.loads(json_format.MessageToJson( - stubB.list_subsystems(subsystem_list_req), - preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] + listB = json.loads( + json_format.MessageToJson( + stubB.list_subsystems(subsystem_list_req), + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + )["subsystems"] assert len(listB) == 1 - listA = json.loads(json_format.MessageToJson( - stubA.list_subsystems(subsystem_list_req), - preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] + listA = json.loads( + json_format.MessageToJson( + stubA.list_subsystems(subsystem_list_req), + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + )["subsystems"] assert len(listA) == num_subsystems - ns2_req = pb2.namespace_add_req(subsystem_nqn=nqn, - rbd_pool_name=pool, - rbd_image_name=image, - block_size=4096, create_image=True, size=16*1024*1024, force=True) + ns2_req = pb2.namespace_add_req( + subsystem_nqn=nqn, + rbd_pool_name=pool, + rbd_image_name=image, + block_size=4096, + create_image=True, + size=16 * 1024 * 1024, + force=True, + ) ret_ns2 = stubB.namespace_add(ns2_req) assert ret_ns2.status == 0 assert "The file is not current, will reload it and try again" in caplog.text # Make sure that after reading the OMAP file GW-B has the subsystem and namespace created on GW-A - listB = json.loads(json_format.MessageToJson( - stubB.list_subsystems(subsystem_list_req), - preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] + listB = json.loads( + json_format.MessageToJson( + stubB.list_subsystems(subsystem_list_req), + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + )["subsystems"] assert len(listB) == num_subsystems - assert listB[num_subsystems-1]["nqn"] == nqn - assert listB[num_subsystems-1]["serial_number"] == serial - assert listB[num_subsystems-1]["namespace_count"] == num_subsystems # We created one namespace on each subsystem + assert listB[num_subsystems - 1]["nqn"] == nqn + assert listB[num_subsystems - 1]["serial_number"] == serial + assert ( + listB[num_subsystems - 1]["namespace_count"] == num_subsystems + ) # We created one namespace on each subsystem caplog.clear() - ns3_req = pb2.namespace_add_req(subsystem_nqn=nqn, - rbd_pool_name=pool, - rbd_image_name=image, - block_size=4096, create_image=True, size=16*1024*1024, force=True) + ns3_req = pb2.namespace_add_req( + subsystem_nqn=nqn, + rbd_pool_name=pool, + rbd_image_name=image, + block_size=4096, + create_image=True, + size=16 * 1024 * 1024, + force=True, + ) ret_ns3 = stubB.namespace_add(ns3_req) assert ret_ns3.status == 0 assert "The file is not current, will reload it and try again" not in caplog.text @@ -248,9 +363,9 @@ def test_multi_gateway_omap_reread(config, conn_omap_reread, caplog): assert len(bdevsB) == 3 assert bdevsA[0]["uuid"] == bdevsB[0]["uuid"] + def test_trying_to_lock_twice(config, image, conn_lock_twice, caplog): - """Tests an attempt to lock the OMAP file from two gateways at the same time - """ + """Tests an attempt to lock the OMAP file from two gateways at the same time""" caplog.clear() stubA, stubB = conn_lock_twice @@ -261,9 +376,9 @@ def test_trying_to_lock_twice(config, image, conn_lock_twice, caplog): assert "The OMAP file is locked, will try again in" in caplog.text assert "Unable to lock OMAP file" in caplog.text + def test_multi_gateway_concurrent_changes(config, image, conn_concurrent, caplog): - """Tests concurrent changes to the OMAP from two gateways - """ + """Tests concurrent changes to the OMAP from two gateways""" caplog.clear() gwA, gwB, stubA, stubB = conn_concurrent @@ -273,14 +388,19 @@ def test_multi_gateway_concurrent_changes(config, image, conn_concurrent, caplog else: create_resource_by_index(stubB, i, caplog) assert "failed" not in caplog.text.lower() - listener_req = pb2.create_listener_req(nqn=f"{subsystem_prefix}0", - host_name=gwA.host_name, - adrfam="ipv4", - traddr="127.0.0.1", - trsvcid=5001) + listener_req = pb2.create_listener_req( + nqn=f"{subsystem_prefix}0", + host_name=gwA.host_name, + adrfam="ipv4", + traddr="127.0.0.1", + trsvcid=5001, + ) listener_ret = stubA.create_listener(listener_req) assert listener_ret.status == 0 - assert f"Received request to create {gwA.host_name} TCP ipv4 listener for {subsystem_prefix}0 at 127.0.0.1:5001" in caplog.text + assert ( + f"Received request to create {gwA.host_name} TCP ipv4 listener for {subsystem_prefix}0 at 127.0.0.1:5001" + in caplog.text + ) assert f"create_listener: True" in caplog.text timeout = 15 # Maximum time to wait (in seconds) @@ -289,61 +409,94 @@ def test_multi_gateway_concurrent_changes(config, image, conn_concurrent, caplog while expected_warning_other_gw not in caplog.text: if time.time() - start_time > timeout: - pytest.fail(f"Timeout: '{expected_warning_other_gw}' not found in caplog.text within {timeout} seconds.") + pytest.fail( + f"Timeout: '{expected_warning_other_gw}' not found in caplog.text within {timeout} seconds." + ) time.sleep(0.1) assert expected_warning_other_gw in caplog.text caplog.clear() subsystem_list_req = pb2.list_subsystems_req() - subListA = json.loads(json_format.MessageToJson( - stubA.list_subsystems(subsystem_list_req), - preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] - subListB = json.loads(json_format.MessageToJson( - stubB.list_subsystems(subsystem_list_req), - preserving_proto_field_name=True, including_default_value_fields=True))['subsystems'] + subListA = json.loads( + json_format.MessageToJson( + stubA.list_subsystems(subsystem_list_req), + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + )["subsystems"] + subListB = json.loads( + json_format.MessageToJson( + stubB.list_subsystems(subsystem_list_req), + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + )["subsystems"] for i in range(created_resource_count): subsystem = f"{subsystem_prefix}{i}" host_list_req = pb2.list_hosts_req(subsystem=subsystem) - hostListA = json.loads(json_format.MessageToJson( - stubA.list_hosts(host_list_req), - preserving_proto_field_name=True, including_default_value_fields=True)) - hostListB = json.loads(json_format.MessageToJson( - stubB.list_hosts(host_list_req), - preserving_proto_field_name=True, including_default_value_fields=True)) + hostListA = json.loads( + json_format.MessageToJson( + stubA.list_hosts(host_list_req), + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + ) + hostListB = json.loads( + json_format.MessageToJson( + stubB.list_hosts(host_list_req), + preserving_proto_field_name=True, + including_default_value_fields=True, + ) + ) check_resource_by_index(i, subListA, hostListA) check_resource_by_index(i, subListB, hostListB) + def test_multi_gateway_listener_update(config, image, conn_concurrent, caplog): - """Tests listener update after subsystem deletion - """ + """Tests listener update after subsystem deletion""" gwA, gwB, stubA, stubB = conn_concurrent caplog.clear() subsystem = f"{subsystem_prefix}QQQ" - subsystem_add_req = pb2.create_subsystem_req(subsystem_nqn=subsystem, max_namespaces=256, enable_ha=True, no_group_append=True) + subsystem_add_req = pb2.create_subsystem_req( + subsystem_nqn=subsystem, + max_namespaces=256, + enable_ha=True, + no_group_append=True, + ) ret_subsystem = stubA.create_subsystem(subsystem_add_req) assert ret_subsystem.status == 0 assert f"create_subsystem {subsystem}: True" in caplog.text assert f"Failure creating subsystem {subsystem}" not in caplog.text caplog.clear() - listenerA_req = pb2.create_listener_req(nqn=subsystem, - host_name=gwA.host_name, - adrfam="ipv4", - traddr="127.0.0.1", - trsvcid=5101) + listenerA_req = pb2.create_listener_req( + nqn=subsystem, + host_name=gwA.host_name, + adrfam="ipv4", + traddr="127.0.0.1", + trsvcid=5101, + ) listener_ret = stubA.create_listener(listenerA_req) assert listener_ret.status == 0 - assert f"Received request to create {gwA.host_name} TCP ipv4 listener for {subsystem} at 127.0.0.1:5101" in caplog.text + assert ( + f"Received request to create {gwA.host_name} TCP ipv4 listener for {subsystem} at 127.0.0.1:5101" + in caplog.text + ) assert f"create_listener: True" in caplog.text caplog.clear() - listenerB_req = pb2.create_listener_req(nqn=subsystem, - host_name=gwB.host_name, - adrfam="ipv4", - traddr="127.0.0.1", - trsvcid=5102) + listenerB_req = pb2.create_listener_req( + nqn=subsystem, + host_name=gwB.host_name, + adrfam="ipv4", + traddr="127.0.0.1", + trsvcid=5102, + ) listener_ret = stubB.create_listener(listenerB_req) assert listener_ret.status == 0 - assert f"Received request to create {gwB.host_name} TCP ipv4 listener for {subsystem} at 127.0.0.1:5102" in caplog.text + assert ( + f"Received request to create {gwB.host_name} TCP ipv4 listener for {subsystem} at 127.0.0.1:5102" + in caplog.text + ) assert f"create_listener: True" in caplog.text caplog.clear() subsystem_del_req = pb2.delete_subsystem_req(subsystem_nqn=subsystem) @@ -359,12 +512,18 @@ def test_multi_gateway_listener_update(config, image, conn_concurrent, caplog): caplog.clear() listener_ret = stubA.create_listener(listenerA_req) assert listener_ret.status == 0 - assert f"Received request to create {gwA.host_name} TCP ipv4 listener for {subsystem} at 127.0.0.1:5101" in caplog.text + assert ( + f"Received request to create {gwA.host_name} TCP ipv4 listener for {subsystem} at 127.0.0.1:5101" + in caplog.text + ) assert f"create_listener: True" in caplog.text assert f"Failure adding {subsystem} listener at 127.0.0.1:5101" not in caplog.text caplog.clear() listener_ret = stubB.create_listener(listenerB_req) assert listener_ret.status == 0 - assert f"Received request to create {gwB.host_name} TCP ipv4 listener for {subsystem} at 127.0.0.1:5102" in caplog.text + assert ( + f"Received request to create {gwB.host_name} TCP ipv4 listener for {subsystem} at 127.0.0.1:5102" + in caplog.text + ) assert f"create_listener: True" in caplog.text assert f"Failure adding {subsystem} listener at 127.0.0.1:5102" not in caplog.text diff --git a/tests/test_psk.py b/tests/test_psk.py index 536035c8..2ac91a71 100644 --- a/tests/test_psk.py +++ b/tests/test_psk.py @@ -30,7 +30,7 @@ hostpsk1 = "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" hostpsk2 = "NVMeTLSkey-1:02:FTFds4vH4utVcfrOforxbrWIgv+Qq4GQHgMdWwzDdDxE1bAqK2mOoyXxmbJxGeueEVVa/Q==:" -hostpsk3 = "junk" +hostpsk3 = "junk" hostpsk4 = "NVMeTLSkey-1:01:YzrPElk4OYy1uUERriPwiiyEJE/+J5ckYpLB+5NHMsR2iBuT:" hostdhchap1 = "DHHC-1:00:MWPqcx1Ug1debg8fPIGpkqbQhLcYUt39k7UWirkblaKEH1kE:" @@ -39,6 +39,7 @@ addr = "127.0.0.1" config = "ceph-nvmeof.conf" + @pytest.fixture(scope="module") def gateway(config): """Sets up and tears down Gateway""" @@ -53,7 +54,11 @@ def gateway(config): # Start gateway gateway.gw_logger_object.set_log_level("debug") - ceph_utils.execute_ceph_monitor_command("{" + f'"prefix":"nvme-gw create", "id": "{gateway.name}", "pool": "{pool}", "group": ""' + "}") + ceph_utils.execute_ceph_monitor_command( + "{" + + f'"prefix":"nvme-gw create", "id": "{gateway.name}", "pool": "{pool}", "group": ""' + + "}" + ) gateway.serve() # Bind the client and Gateway @@ -64,43 +69,138 @@ def gateway(config): gateway.server.stop(grace=1) gateway.gateway_rpc.gateway_state.delete_state() + def test_setup(caplog, gateway): gw = gateway caplog.clear() cli(["subsystem", "add", "--subsystem", subsystem]) assert f"create_subsystem {subsystem}: True" in caplog.text caplog.clear() - cli(["namespace", "add", "--subsystem", subsystem, "--rbd-pool", pool, "--rbd-image", image, "--rbd-create-image", "--size", "16MB"]) + cli( + [ + "namespace", + "add", + "--subsystem", + subsystem, + "--rbd-pool", + pool, + "--rbd-image", + image, + "--rbd-create-image", + "--size", + "16MB", + ] + ) assert f"Adding namespace 1 to {subsystem}: Successful" in caplog.text + def test_create_secure_with_any_host(caplog, gateway): caplog.clear() cli(["host", "add", "--subsystem", subsystem, "--host-nqn", "*"]) assert f"Allowing open host access to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name, "-a", addr, "-s", "5001", "--secure"]) - assert f"Secure channel is only allowed for subsystems in which \"allow any host\" is off" in caplog.text + cli( + [ + "listener", + "add", + "--subsystem", + subsystem, + "--host-name", + host_name, + "-a", + addr, + "-s", + "5001", + "--secure", + ] + ) + assert ( + f'Secure channel is only allowed for subsystems in which "allow any host" is off' + in caplog.text + ) caplog.clear() cli(["host", "del", "--subsystem", subsystem, "--host-nqn", "*"]) assert f"Disabling open host access to {subsystem}: Successful" in caplog.text + def test_create_secure(caplog, gateway): caplog.clear() - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name, "-a", addr, "-s", "5001", "--secure"]) + cli( + [ + "listener", + "add", + "--subsystem", + subsystem, + "--host-name", + host_name, + "-a", + addr, + "-s", + "5001", + "--secure", + ] + ) assert f"Adding {subsystem} listener at {addr}:5001: Successful" in caplog.text caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn1, "--psk", hostpsk1]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn1, + "--psk", + hostpsk1, + ] + ) assert f"Adding host {hostnqn1} to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn2, "--psk", hostpsk2]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn2, + "--psk", + hostpsk2, + ] + ) assert f"Adding host {hostnqn2} to {subsystem}: Successful" in caplog.text caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn4, "--psk", hostpsk4]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn4, + "--psk", + hostpsk4, + ] + ) assert f"Adding host {hostnqn4} to {subsystem}: Successful" in caplog.text + def test_create_not_secure(caplog, gateway): caplog.clear() - cli(["listener", "add", "--subsystem", subsystem, "--host-name", host_name, "-a", addr, "-s", "5002"]) + cli( + [ + "listener", + "add", + "--subsystem", + subsystem, + "--host-name", + host_name, + "-a", + addr, + "-s", + "5002", + ] + ) assert f"Adding {subsystem} listener at {addr}:5002: Successful" in caplog.text caplog.clear() cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn6]) @@ -109,22 +209,52 @@ def test_create_not_secure(caplog, gateway): cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn7]) assert f"Adding host {hostnqn7} to {subsystem}: Successful" in caplog.text + def test_create_secure_list(caplog, gateway): caplog.clear() rc = 0 try: - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn8, hostnqn9, hostnqn10, "--psk", hostpsk1]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn8, + hostnqn9, + hostnqn10, + "--psk", + hostpsk1, + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass assert rc == 2 - assert f"error: Can't have more than one host NQN when PSK keys are used" in caplog.text + assert ( + f"error: Can't have more than one host NQN when PSK keys are used" + in caplog.text + ) + def test_create_secure_junk_key(caplog, gateway): caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn3, "--psk", hostpsk3]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn3, + "--psk", + hostpsk3, + ] + ) assert f"Failure adding host {hostnqn3} to {subsystem}" in caplog.text + def test_create_secure_no_key(caplog, gateway): caplog.clear() rc = 0 @@ -136,6 +266,7 @@ def test_create_secure_no_key(caplog, gateway): assert rc == 2 assert f"error: argument --psk/-p: expected one argument" in caplog.text + def test_list_psk_hosts(caplog, gateway): caplog.clear() hosts = cli_test(["host", "list", "--subsystem", subsystem]) @@ -161,22 +292,52 @@ def test_list_psk_hosts(caplog, gateway): assert False assert found == 5 + def test_allow_any_host_with_psk(caplog, gateway): caplog.clear() rc = 0 try: - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", "*", "--psk", hostpsk1]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + "*", + "--psk", + hostpsk1, + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass assert rc == 2 assert f"error: PSK key is only allowed for specific hosts" in caplog.text + def test_psk_with_dhchap(caplog, gateway): caplog.clear() - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn10, "--psk", hostpsk1, "--dhchap-key", hostdhchap1]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn10, + "--psk", + hostpsk1, + "--dhchap-key", + hostdhchap1, + ] + ) assert f"Adding host {hostnqn10} to {subsystem}: Successful" in caplog.text - assert f"Host {hostnqn10} has a DH-HMAC-CHAP key but subsystem {subsystem} has no key, a unidirectional authentication will be used" in caplog.text + assert ( + f"Host {hostnqn10} has a DH-HMAC-CHAP key but subsystem {subsystem} has no key, a unidirectional authentication will be used" + in caplog.text + ) + def test_list_listeners(caplog, gateway): caplog.clear() @@ -194,11 +355,24 @@ def test_list_listeners(caplog, gateway): assert False assert found == 2 + def test_add_host_with_key_host_list(caplog, gateway): caplog.clear() rc = 0 try: - cli(["host", "add", "--subsystem", subsystem, "--host-nqn", hostnqn11, hostnqn12, "--psk", "junk"]) + cli( + [ + "host", + "add", + "--subsystem", + subsystem, + "--host-nqn", + hostnqn11, + hostnqn12, + "--psk", + "junk", + ] + ) except SystemExit as sysex: rc = int(str(sysex)) pass diff --git a/tests/test_server.py b/tests/test_server.py index fdfd209c..478eb60a 100644 --- a/tests/test_server.py +++ b/tests/test_server.py @@ -7,27 +7,29 @@ import unittest from control.server import GatewayServer + class TestServer(unittest.TestCase): # Location of core files in test env. - core_dir="/tmp/coredump" + core_dir = "/tmp/coredump" @pytest.fixture(autouse=True) def _config(self, config): self.config = config def validate_exception(self, e): - pattern = r'Gateway subprocess terminated pid=(\d+) exit_code=(-?\d+)' + pattern = r"Gateway subprocess terminated pid=(\d+) exit_code=(-?\d+)" m = re.match(pattern, e.code) - assert(m) + assert m pid = int(m.group(1)) code = int(m.group(2)) - assert(pid > 0) - assert(code) + assert pid > 0 + assert code def remove_core_files(self, directory_path): # List all files starting with "core." in the core directory files = [ - f for f in os.listdir(directory_path) + f + for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f)) and f.startswith("core.") ] @@ -38,9 +40,13 @@ def remove_core_files(self, directory_path): print(f"Removed: {file_path}") def assert_no_core_files(self, directory_path): - assert(os.path.exists(directory_path) and os.path.isdir(directory_path)) - files = [f for f in os.listdir(directory_path) if os.path.isfile(os.path.join(directory_path, f)) and f.startswith("core.")] - assert(len(files) == 0) + assert os.path.exists(directory_path) and os.path.isdir(directory_path) + files = [ + f + for f in os.listdir(directory_path) + if os.path.isfile(os.path.join(directory_path, f)) and f.startswith("core.") + ] + assert len(files) == 0 def test_spdk_exception(self): """Tests spdk sub process exiting with error.""" @@ -62,13 +68,13 @@ def test_no_coredumps_on_gracefull_shutdown(self): gateway.serve() time.sleep(10) # exited context, sub processes should terminate gracefully - time.sleep(10) # let it dump + time.sleep(10) # let it dump self.assert_no_core_files(self.core_dir) def test_monc_exit(self): """Tests monitor client sub process abort.""" config_monc_abort = copy.deepcopy(self.config) - signals = [ signal.SIGABRT, signal.SIGTERM, signal.SIGKILL ] + signals = [signal.SIGABRT, signal.SIGTERM, signal.SIGKILL] for sig in signals: with self.assertRaises(SystemExit) as cm: @@ -80,7 +86,7 @@ def test_monc_exit(self): time.sleep(2) # Send SIGABRT (abort signal) to the monitor client process - assert(gateway.monitor_client_process) + assert gateway.monitor_client_process gateway.monitor_client_process.send_signal(signal.SIGABRT) # Block on running keep alive ping @@ -109,12 +115,13 @@ def test_spdk_multi_gateway_exception(self): with ( GatewayServer(configA) as gatewayA, GatewayServer(configB) as gatewayB, - ): + ): gatewayA.set_group_id(0) gatewayA.serve() gatewayB.set_group_id(1) gatewayB.serve() self.validate_exception(cm.exception) -if __name__ == '__main__': + +if __name__ == "__main__": unittest.main() diff --git a/tests/test_state.py b/tests/test_state.py index fda1366d..6b569244 100644 --- a/tests/test_state.py +++ b/tests/test_state.py @@ -75,8 +75,9 @@ def _state_polling_update(update, is_add_req): version = 1 update_interval_sec = 1 - state = GatewayStateHandler(config, local_state, omap_state, - _state_polling_update, "test") + state = GatewayStateHandler( + config, local_state, omap_state, _state_polling_update, "test" + ) state.update_interval = update_interval_sec state.use_notify = False key = "namespace_test" @@ -84,20 +85,26 @@ def _state_polling_update(update, is_add_req): # Add namespace key to OMAP and update version number version += 1 - add_key(ioctx, key, "add", version, omap_state.omap_name, - omap_state.OMAP_VERSION_KEY) + add_key( + ioctx, key, "add", version, omap_state.omap_name, omap_state.OMAP_VERSION_KEY + ) time.sleep(update_interval_sec + 1) # Allow time for polling # Change namespace key and update version number version += 1 - add_key(ioctx, key, "changed", version, omap_state.omap_name, - omap_state.OMAP_VERSION_KEY) + add_key( + ioctx, + key, + "changed", + version, + omap_state.omap_name, + omap_state.OMAP_VERSION_KEY, + ) time.sleep(update_interval_sec + 1) # Allow time for polling # Remove namespace key and update version number version += 1 - remove_key(ioctx, key, version, omap_state.omap_name, - omap_state.OMAP_VERSION_KEY) + remove_key(ioctx, key, version, omap_state.omap_name, omap_state.OMAP_VERSION_KEY) time.sleep(update_interval_sec + 1) # Allow time for polling assert update_counter == 4 @@ -107,7 +114,7 @@ def test_state_notify_update(config, ioctx, local_state, omap_state): """Confirms use of OMAP watch/notify for updates.""" update_counter = 0 - notify_event = threading.Event() # Event to signal when notify is called + notify_event = threading.Event() # Event to signal when notify is called def _state_notify_update(update, is_add_req): nonlocal update_counter @@ -139,8 +146,9 @@ def _state_notify_update(update, is_add_req): version = 1 update_interval_sec = 10 - state = GatewayStateHandler(config, local_state, omap_state, - _state_notify_update, "test") + state = GatewayStateHandler( + config, local_state, omap_state, _state_notify_update, "test" + ) key = "namespace_test" state.update_interval = update_interval_sec state.use_notify = True @@ -149,9 +157,10 @@ def _state_notify_update(update, is_add_req): # Add namespace key to OMAP and update version number version += 1 - add_key(ioctx, key, "add", version, omap_state.omap_name, - omap_state.OMAP_VERSION_KEY) - assert (ioctx.notify(omap_state.omap_name)) # Send notify signal + add_key( + ioctx, key, "add", version, omap_state.omap_name, omap_state.OMAP_VERSION_KEY + ) + assert ioctx.notify(omap_state.omap_name) # Send notify signal # Wait for the notify to be triggered assert notify_event.wait(update_interval_sec - 0.5) notify_event.clear() # Reset the event for the next notification @@ -159,9 +168,15 @@ def _state_notify_update(update, is_add_req): # Change namespace key and update version number version += 1 - add_key(ioctx, key, "changed", version, omap_state.omap_name, - omap_state.OMAP_VERSION_KEY) - assert (ioctx.notify(omap_state.omap_name)) # Send notify signal + add_key( + ioctx, + key, + "changed", + version, + omap_state.omap_name, + omap_state.OMAP_VERSION_KEY, + ) + assert ioctx.notify(omap_state.omap_name) # Send notify signal # Wait for the notify to be triggered assert notify_event.wait(update_interval_sec - 0.5) notify_event.clear() # Reset the event for the next notification @@ -169,9 +184,8 @@ def _state_notify_update(update, is_add_req): # Remove namespace key and update version number version += 1 - remove_key(ioctx, key, version, omap_state.omap_name, - omap_state.OMAP_VERSION_KEY) - assert (ioctx.notify(omap_state.omap_name)) # Send notify signal + remove_key(ioctx, key, version, omap_state.omap_name, omap_state.OMAP_VERSION_KEY) + assert ioctx.notify(omap_state.omap_name) # Send notify signal # Wait for the notify to be triggered assert notify_event.wait(update_interval_sec - 0.5) notify_event.clear() # Reset the event for the next notification @@ -181,8 +195,8 @@ def _state_notify_update(update, is_add_req): # to test notify capability elapsed = time.time() - start wait_interval = update_interval_sec - elapsed - 0.5 - assert(wait_interval > 0) - assert(wait_interval < update_interval_sec) + assert wait_interval > 0 + assert wait_interval < update_interval_sec time.sleep(wait_interval) # expect 4 updates: addition, two-step change and removal