From fa6988b6d42d008b35388a3f101d1db60f424f5a Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Sat, 2 Dec 2023 03:51:59 +0000 Subject: [PATCH 01/47] module/raid: add superblock struct for future realisation of superblock also add superblock to the base bdev struct --- module/bdev/raid/bdev_raid.h | 42 ++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index bd894ceef8c..3debdc09d79 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -45,6 +45,45 @@ enum raid_bdev_state { typedef void (*raid_bdev_remove_base_bdev_cb)(void *ctx, int status); +/* + * Superblock for operation with metadata of the base bdev which part of some raid. + * It stores some metadata of the base bdev and the raid + */ +struct raid_superblock { + /* SPDK raid magic number "SKRd" */ + uint32_t magic; + + /* The version of metadata. Currently, only 01 version exists */ + uint32_t version; + + /* Logical block size of base bdev */ + uint32_t blocklen; + + /* Number of base bdevs */ + uint8_t num_base_bdevs; + + /* Raid Level of device's raid */ + enum raid_level level; + + /* Position of device in raid */ + uint32_t array_position; + + /* strip size of device's raid in blocks */ + uint32_t strip_size; + + /* Number of blocks held by this base bdev */ + uint64_t blockcnt; + + /* Number of blocks held by device's raid */ + uint64_t raid_blockcnt; + + /* Timestamp to know the freshest device in raid */ + struct timespec timestamp; + + /* UUID of this base bdev */ + struct spdk_uuid uuid; +}; + /* * raid_base_bdev_info contains information for the base bdevs which are part of some * raid. This structure contains the per base bdev information. Whatever is @@ -60,6 +99,9 @@ struct raid_base_bdev_info { /* pointer to base bdev descriptor opened by raid bdev */ struct spdk_bdev_desc *desc; + /* base bdev's superblock */ + struct raid_superblock *raid_sb; + /* offset in blocks from the start of the base bdev to the start of the data region */ uint64_t data_offset; From cc0a8879b60941778a10754e64addc5a287832f6 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Mon, 4 Dec 2023 19:04:05 +0000 Subject: [PATCH 02/47] module/raid: update superblock struct - add raid metadata's magic number - add enum for the metadata versions - change uuid from base bdev to raid bdev - add physical blocklen --- module/bdev/raid/bdev_raid.h | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index 3debdc09d79..640f92e5715 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -11,6 +11,8 @@ #define RAID_BDEV_MIN_DATA_OFFSET_SIZE (1024*1024) /* 1 MiB */ +#define RAID_SUPERBLOCK_MAGIC 0x534B5244 /* "SKRD" */ + enum raid_level { INVALID_RAID_LEVEL = -1, RAID0 = 0, @@ -45,20 +47,27 @@ enum raid_bdev_state { typedef void (*raid_bdev_remove_base_bdev_cb)(void *ctx, int status); +enum metadata_version { + RAID_METADATA_VERSION_01 = 01 +}; + /* * Superblock for operation with metadata of the base bdev which part of some raid. * It stores some metadata of the base bdev and the raid */ struct raid_superblock { - /* SPDK raid magic number "SKRd" */ + /* SPDK raid magic number "SKRD" */ uint32_t magic; /* The version of metadata. Currently, only 01 version exists */ - uint32_t version; + enum metadata_version version; /* Logical block size of base bdev */ uint32_t blocklen; + /* Physical block size of base bdev */ + uint32_t phys_blocklen; + /* Number of base bdevs */ uint8_t num_base_bdevs; @@ -80,7 +89,7 @@ struct raid_superblock { /* Timestamp to know the freshest device in raid */ struct timespec timestamp; - /* UUID of this base bdev */ + /* UUID of raid bdev */ struct spdk_uuid uuid; }; From 0937b77076dfae039c453e4a424aef7927fbe4c9 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Mon, 4 Dec 2023 19:06:24 +0000 Subject: [PATCH 03/47] module/raid: change in raid_base_bdev_info and raid_bdev structs for superblock add position, is_new_ sb_loaded to raid_base_bdev_info is_new to raid_bdev for operation with raid's superblocks --- module/bdev/raid/bdev_raid.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index 640f92e5715..ed78bec1375 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -108,6 +108,15 @@ struct raid_base_bdev_info { /* pointer to base bdev descriptor opened by raid bdev */ struct spdk_bdev_desc *desc; + /* position of the base bdev in raid_bdev's base_bdev_info (slot) */ + uint32_t position; + + /* indicate if the metadata was on disk or not */ + bool is_new; + + /* true if sb has loaded */ + bool sb_loaded; + /* base bdev's superblock */ struct raid_superblock *raid_sb; @@ -190,6 +199,9 @@ struct raid_bdev { /* state of raid bdev */ enum raid_bdev_state state; + /* indicate if raid is new for raid_bdev_base_bdev_sb_validate */ + bool is_new; + /* number of base bdevs comprising raid bdev */ uint8_t num_base_bdevs; From 7bef123fbe813e8dfe6c92c5a10edf564b1045f6 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Mon, 4 Dec 2023 19:09:50 +0000 Subject: [PATCH 04/47] module/raid: add super_sync function for raid's superblock --- module/bdev/raid/bdev_raid.c | 113 +++++++++++++++++++++++++++++++++++ 1 file changed, 113 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 79932a600f4..fd6278da0b8 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1116,6 +1116,119 @@ raid_bdev_configure_md(struct raid_bdev *raid_bdev) return 0; } +static int +raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { + struct raid_base_bdev_info *base_info = cb_arg; + + if (success) { + SPDK_NOTICELOG("Read superblock from base bdev : %s\n", base_info->name); + } else { + SPDK_ERRLOG("base bdev '%s' superblock io read error\n", base_info->name); + } + + spdk_bdev_free_io(bdev_io); + + base_info->sb_loaded = true; +} + +static int +raid_bdev_load_base_bdevs_sb(struct raid_base_bdev_info *base_info, uint32_t sb_size) +{ + struct spdk_io_channel *ch; + struct spdk_bdev_channel *ctx; + struct spdk_bdev_io *bdev_io; + + if (base_info->sb_loaded) + return 0; + + ch = spdk_bdev_get_io_channel(base_info->desc); + + if (!ch) { + SPDK_ERRLOG("Unable to create io channel for bdev '%s'\n", base_info->name); + return -ENOMEM; + } + + ctx = spdk_io_channel_get_ctx(ch); + + spdk_bdev_read_blocks(base_info->desc, ch, base_info->raid_sb, 0, + sb_size, (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_read_sb_compete, + base_info); + return 0; +} + +static int +raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) +{ + struct raid_superblock *sb = base_info->raid_sb; + struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); + struct raid_bdev *raid = base_info->raid_bdev; + + sb->magic = RAID_SUPERBLOCK_MAGIC; + sb->version = RAID_METADATA_VERSION_01; + sb->blocklen = base_bdev->blocklen; + sb->phys_blocklen = base_bdev->phys_blocklen; + sb->num_base_bdevs = raid->num_base_bdevs; + sb->level = raid->level; + sb->array_position = base_info->position; + sb->strip_size = raid->strip_size; + sb->blockcnt = base_bdev->blockcnt; + sb->raid_blockcnt = raid->bdev.blockcnt; + + clock_gettime(CLOCK_REALTIME, &sb->timestamp); + + sb->uuid = raid->bdev.uuid; + return 0; +} + +static int +raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct raid_superblock **freshest) +{ + int rc = 0; + uint32_t sb_blocks = spdk_divide_round_up(sizeof(struct raid_superblock), + spdk_bdev_desc_get_bdev(base_info->desc)->blocklen); + + base_info->raid_sb = calloc(1, sb_blocks); + if (!base_info->raid_sb) { + SPDK_ERRLOG("Failed to allocate memory for raid_sb\n"); + return -ENOMEM; + } + + rc = raid_bdev_load_base_bdevs_sb(base_info, sb_blocks); + if (rc) { + SPDK_ERRLOG("Failed while read superblock from base bdev '%s'\n", base_info->name); + goto clean; + } + + struct raid_superblock *sb = base_info->raid_sb; + + if (sb->magic != RAID_SUPERBLOCK_MAGIC) + base_info->is_new = true; + + if (base_info->is_new) { + rc = raid_bdev_base_bdev_super_sync(base_info); + if (rc) { + SPDK_ERRLOG("Failed in super_sync for base bdev '%s'\n", base_info->name); + goto clean; + } + + *freshest = *freshest ? : sb; + return 0; + } + + if (!*freshest) { + *freshest = sb; + return 0; + } + + *freshest = (*freshest)->timestamp.tv_nsec > sb->timestamp.tv_nsec ? *freshest : sb; + return 0; + +clean: + free(base_info->raid_sb); + base_info->raid_sb = NULL; + base_info->sb_loaded = false; + return rc; +} /* * brief: * If raid bdev config is complete, then only register the raid bdev to From db72808c244d5fbc04c9c7b36533a796d1f7359e Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Mon, 4 Dec 2023 19:23:41 +0000 Subject: [PATCH 05/47] module/raid: add raid_sb free to free_base_bdev_resource --- module/bdev/raid/bdev_raid.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index fd6278da0b8..17254bf6501 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -221,6 +221,9 @@ raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info) free(base_info->name); base_info->name = NULL; + free(base_info->raid_sb); + base_info->raid_sb = NULL; + if (base_info->desc == NULL) { return; } From 78a78adcbcec8d280cb9698c6665873a1e39f0db Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Mon, 4 Dec 2023 19:33:10 +0000 Subject: [PATCH 06/47] module/raid: remove free of raid_sb in super_load --- module/bdev/raid/bdev_raid.c | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 17254bf6501..50f09a46972 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1199,7 +1199,7 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai rc = raid_bdev_load_base_bdevs_sb(base_info, sb_blocks); if (rc) { SPDK_ERRLOG("Failed while read superblock from base bdev '%s'\n", base_info->name); - goto clean; + goto bad; } struct raid_superblock *sb = base_info->raid_sb; @@ -1211,7 +1211,7 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai rc = raid_bdev_base_bdev_super_sync(base_info); if (rc) { SPDK_ERRLOG("Failed in super_sync for base bdev '%s'\n", base_info->name); - goto clean; + goto bad; } *freshest = *freshest ? : sb; @@ -1226,9 +1226,7 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai *freshest = (*freshest)->timestamp.tv_nsec > sb->timestamp.tv_nsec ? *freshest : sb; return 0; -clean: - free(base_info->raid_sb); - base_info->raid_sb = NULL; +bad: base_info->sb_loaded = false; return rc; } From 85de375c68a5ce4132a42716ae79cb53fdb08273 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Mon, 4 Dec 2023 20:43:21 +0000 Subject: [PATCH 07/47] module/raid: fix timestamp for new base bdevs, change freshest type to raid_base_bdev_info --- module/bdev/raid/bdev_raid.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 50f09a46972..bde44cd590b 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1165,6 +1165,7 @@ raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) struct raid_superblock *sb = base_info->raid_sb; struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); struct raid_bdev *raid = base_info->raid_bdev; + struct timespec time_0 = {.tv_nsec = 0, .tv_sec = 0}; sb->magic = RAID_SUPERBLOCK_MAGIC; sb->version = RAID_METADATA_VERSION_01; @@ -1176,15 +1177,13 @@ raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) sb->strip_size = raid->strip_size; sb->blockcnt = base_bdev->blockcnt; sb->raid_blockcnt = raid->bdev.blockcnt; - - clock_gettime(CLOCK_REALTIME, &sb->timestamp); - + sb->timestamp = time_0; sb->uuid = raid->bdev.uuid; return 0; } static int -raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct raid_superblock **freshest) +raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct raid_base_bdev_info **freshest) { int rc = 0; uint32_t sb_blocks = spdk_divide_round_up(sizeof(struct raid_superblock), @@ -1214,16 +1213,19 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai goto bad; } - *freshest = *freshest ? : sb; + *freshest = *freshest ? : base_info; return 0; } if (!*freshest) { - *freshest = sb; + *freshest = base_info; return 0; } - *freshest = (*freshest)->timestamp.tv_nsec > sb->timestamp.tv_nsec ? *freshest : sb; + if ((*freshest)->raid_sb->timestamp.tv_sec <= sb->timestamp.tv_sec && + (*freshest)->raid_sb->timestamp.tv_nsec < sb->timestamp.tv_nsec) + *freshest = base_info; + return 0; bad: From e7b955a01619c760ef3660227a56fb6da6185fd2 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Mon, 4 Dec 2023 21:11:22 +0000 Subject: [PATCH 08/47] module/raid: add sb_validate function for metadata --- module/bdev/raid/bdev_raid.c | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index bde44cd590b..08702492a07 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1217,6 +1217,8 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai return 0; } + base_info->raid_bdev->is_new = false; + if (!*freshest) { *freshest = base_info; return 0; @@ -1232,6 +1234,56 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai base_info->sb_loaded = false; return rc; } + +static int +raid_bdev_sb_init_validation(struct raid_bdev *raid, struct raid_base_bdev_info *base_info) +{ + int rc = 0; + struct raid_superblock *sb = base_info->raid_sb; + + assert(raid->num_base_bdevs == raid->num_base_bdevs_discovered); + + raid->num_base_bdevs = sb->num_base_bdevs; + raid->num_base_bdevs_discovered = sb->num_base_bdevs; + raid->level = sb->level; + raid->bdev.blocklen = sb->blocklen; + raid->bdev.blockcnt = sb->raid_blockcnt; + raid->bdev.uuid = sb->uuid; + raid->strip_size = sb->strip_size; + raid->strip_size_kb = sb->strip_size * sb->blocklen; +} + +static int +raid_bdev_base_bdev_sb_validate(struct raid_base_bdev_info *base_info) +{ + int rc = 0; + struct raid_superblock *sb = base_info->raid_sb; + struct raid_bdev *raid = base_info->raid_bdev; + + if (raid->num_base_bdevs <= 0 || !sb) + return 0; + + if (raid->is_new && raid_bdev_sb_init_validation(raid, base_info)) + return -EINVAL; + + if (sb->version != RAID_METADATA_VERSION_01) { + SPDK_ERRLOG("Unsupported version of raid metadata has been found in base bdev '%s' superblock\n", base_info->name); + return -EINVAL; + } + + if (!raid->is_new) { + sb->blocklen = raid->bdev.blocklen; + sb->blockcnt = spdk_bdev_desc_get_bdev(base_info->desc)->blockcnt; + sb->raid_blockcnt = raid->bdev.blockcnt; + + if (sb->array_position != base_info->position) + SPDK_WARNLOG("The base bdev '%s' has changed position in the raid from '%n' to '%n'\n", base_info->name, + sb->array_position, base_info->position); + } + + return rc; +} + /* * brief: * If raid bdev config is complete, then only register the raid bdev to From 10853c67c71318a2c4ea9017a94baa23cbb01463 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 12:20:30 +0000 Subject: [PATCH 09/47] module/raid: add macros RAID_SB_BLOCKS --- module/bdev/raid/bdev_raid.c | 5 ++--- module/bdev/raid/bdev_raid.h | 2 ++ 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 08702492a07..a2c01729f82 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1186,8 +1186,7 @@ static int raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct raid_base_bdev_info **freshest) { int rc = 0; - uint32_t sb_blocks = spdk_divide_round_up(sizeof(struct raid_superblock), - spdk_bdev_desc_get_bdev(base_info->desc)->blocklen); + struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); base_info->raid_sb = calloc(1, sb_blocks); if (!base_info->raid_sb) { @@ -1195,7 +1194,7 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai return -ENOMEM; } - rc = raid_bdev_load_base_bdevs_sb(base_info, sb_blocks); + rc = raid_bdev_load_base_bdevs_sb(base_info, RAID_SB_BLOCKS(base_bdev->blocklen)); if (rc) { SPDK_ERRLOG("Failed while read superblock from base bdev '%s'\n", base_info->name); goto bad; diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index ed78bec1375..6733a617e75 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -13,6 +13,8 @@ #define RAID_SUPERBLOCK_MAGIC 0x534B5244 /* "SKRD" */ +#define RAID_SB_BLOCKS(bdev_blocklen) spdk_divide_round_up(sizeof(struct raid_superblock), bdev_blocklen) + enum raid_level { INVALID_RAID_LEVEL = -1, RAID0 = 0, From de041a2e9ffb6a0053ae5271679a27106ffb6cc0 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 12:32:55 +0000 Subject: [PATCH 10/47] module/raid: change calloc and free for superblock to dma analogs add memset 0 for remaining of superblock in super_sync --- module/bdev/raid/bdev_raid.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index a2c01729f82..0518700ed49 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -221,7 +221,7 @@ raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info) free(base_info->name); base_info->name = NULL; - free(base_info->raid_sb); + spdk_dma_free(base_info->raid_sb); base_info->raid_sb = NULL; if (base_info->desc == NULL) { @@ -1179,6 +1179,9 @@ raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) sb->raid_blockcnt = raid->bdev.blockcnt; sb->timestamp = time_0; sb->uuid = raid->bdev.uuid; + + memset(sb+1, 0, RAID_SB_BLOCKS(base_bdev->blocklen)) + return 0; } @@ -1188,7 +1191,7 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai int rc = 0; struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); - base_info->raid_sb = calloc(1, sb_blocks); + base_info->raid_sb = spdk_dma_malloc(RAID_SB_BLOCKS(base_bdev->blocklen) * base_bdev->blocklen, 0, NULL); if (!base_info->raid_sb) { SPDK_ERRLOG("Failed to allocate memory for raid_sb\n"); return -ENOMEM; From 86924467d0234a6478f1f70f6076babb56eb89aa Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 12:35:56 +0000 Subject: [PATCH 11/47] module/raid: disable alignment for raid_superblock --- module/bdev/raid/bdev_raid.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index 6733a617e75..e1825d5559c 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -53,6 +53,7 @@ enum metadata_version { RAID_METADATA_VERSION_01 = 01 }; +#pragma pack(push, 1) /* * Superblock for operation with metadata of the base bdev which part of some raid. * It stores some metadata of the base bdev and the raid @@ -94,6 +95,7 @@ struct raid_superblock { /* UUID of raid bdev */ struct spdk_uuid uuid; }; +#pragma pop /* * raid_base_bdev_info contains information for the base bdevs which are part of some From 5e82fb0c84eae208756d43237fc26431c837c504 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 16:56:29 +0000 Subject: [PATCH 12/47] module/raid: fix raid superblock change enum to uint32_t remove useless phys_blocklen --- module/bdev/raid/bdev_raid.h | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index e1825d5559c..7b9a0fb319f 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -63,19 +63,16 @@ struct raid_superblock { uint32_t magic; /* The version of metadata. Currently, only 01 version exists */ - enum metadata_version version; + uint32_t version; /* Logical block size of base bdev */ uint32_t blocklen; - /* Physical block size of base bdev */ - uint32_t phys_blocklen; - /* Number of base bdevs */ uint8_t num_base_bdevs; /* Raid Level of device's raid */ - enum raid_level level; + uint32_t level; /* Position of device in raid */ uint32_t array_position; From d2bf700467dfb355bb53dfb347aa3c24a24e77e5 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 17:17:13 +0000 Subject: [PATCH 13/47] module/raid: add function raid_bdev_is_raid_level for super_init_validation function --- module/bdev/raid/bdev_raid.c | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 0518700ed49..a2c4d678a29 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -795,6 +795,16 @@ static struct { { } }; +bool +raid_bdev_is_raid_level(uint32_t level) { + for (int i = 0; g_raid_level_names[i].name != NULL; i++) { + if (level == g_raid_level_names[i].value) + return true; + } + + return false; +} + /* We have to use the typedef in the function declaration to appease astyle. */ typedef enum raid_level raid_level_t; typedef enum raid_bdev_state raid_bdev_state_t; From e8f9cf813aa440d6e244ca3d9c0bb484dcfcd556 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 17:20:22 +0000 Subject: [PATCH 14/47] module/raid: fix and clean base_bdev_super_load raid_bdev->is_new shouldn't be changed in this function super_sync will be in base_bdev_super_validate --- module/bdev/raid/bdev_raid.c | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index a2c4d678a29..b98c4321420 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1161,8 +1161,6 @@ raid_bdev_load_base_bdevs_sb(struct raid_base_bdev_info *base_info, uint32_t sb_ return -ENOMEM; } - ctx = spdk_io_channel_get_ctx(ch); - spdk_bdev_read_blocks(base_info->desc, ch, base_info->raid_sb, 0, sb_size, (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_read_sb_compete, base_info); @@ -1219,18 +1217,10 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai base_info->is_new = true; if (base_info->is_new) { - rc = raid_bdev_base_bdev_super_sync(base_info); - if (rc) { - SPDK_ERRLOG("Failed in super_sync for base bdev '%s'\n", base_info->name); - goto bad; - } - *freshest = *freshest ? : base_info; return 0; } - base_info->raid_bdev->is_new = false; - if (!*freshest) { *freshest = base_info; return 0; From b434e833cf95dec65c9531d0a480810c23404393 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 17:26:36 +0000 Subject: [PATCH 15/47] module/raid: refactor base_bdev_super_sync remove rewrite of blockcnt, timestamp --- module/bdev/raid/bdev_raid.c | 48 +++++++++++++++++------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index b98c4321420..19bdd99571e 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1167,32 +1167,6 @@ raid_bdev_load_base_bdevs_sb(struct raid_base_bdev_info *base_info, uint32_t sb_ return 0; } -static int -raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) -{ - struct raid_superblock *sb = base_info->raid_sb; - struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); - struct raid_bdev *raid = base_info->raid_bdev; - struct timespec time_0 = {.tv_nsec = 0, .tv_sec = 0}; - - sb->magic = RAID_SUPERBLOCK_MAGIC; - sb->version = RAID_METADATA_VERSION_01; - sb->blocklen = base_bdev->blocklen; - sb->phys_blocklen = base_bdev->phys_blocklen; - sb->num_base_bdevs = raid->num_base_bdevs; - sb->level = raid->level; - sb->array_position = base_info->position; - sb->strip_size = raid->strip_size; - sb->blockcnt = base_bdev->blockcnt; - sb->raid_blockcnt = raid->bdev.blockcnt; - sb->timestamp = time_0; - sb->uuid = raid->bdev.uuid; - - memset(sb+1, 0, RAID_SB_BLOCKS(base_bdev->blocklen)) - - return 0; -} - static int raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct raid_base_bdev_info **freshest) { @@ -1253,6 +1227,28 @@ raid_bdev_sb_init_validation(struct raid_bdev *raid, struct raid_base_bdev_info raid->bdev.uuid = sb->uuid; raid->strip_size = sb->strip_size; raid->strip_size_kb = sb->strip_size * sb->blocklen; + +static int +raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) +{ + struct raid_superblock *sb = base_info->raid_sb; + struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); + struct raid_bdev *raid = base_info->raid_bdev; + + sb->magic = RAID_SUPERBLOCK_MAGIC; + sb->version = RAID_METADATA_VERSION_01; + sb->blocklen = raid->bdev.blocklen; + sb->num_base_bdevs = raid->num_base_bdevs; + sb->level = raid->level; + sb->array_position = base_info->position; + sb->strip_size = raid->strip_size; + sb->raid_blockcnt = raid->bdev.blockcnt; + sb->uuid = raid->bdev.uuid; + + memset(sb+1, 0, RAID_SB_BLOCKS(base_bdev->blocklen)); + + return 0; +} } static int From 1613991c5e32372d43df448002e2e0373f74199d Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 17:28:42 +0000 Subject: [PATCH 16/47] module/raid: refactor of sb_init_validation rename to super_init_validation add validation of metadata add return --- module/bdev/raid/bdev_raid.c | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 19bdd99571e..4096e57ff32 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1212,13 +1212,31 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai } static int -raid_bdev_sb_init_validation(struct raid_bdev *raid, struct raid_base_bdev_info *base_info) +raid_bdev_super_init_validation(struct raid_bdev *raid, struct raid_base_bdev_info *base_info) { int rc = 0; struct raid_superblock *sb = base_info->raid_sb; assert(raid->num_base_bdevs == raid->num_base_bdevs_discovered); + if (sb->version != RAID_METADATA_VERSION_01) { + SPDK_ERRLOG("Unsupported version of raid metadata has been found in base '%s' bdev's superblock\n", + base_info->name); + return -EINVAL; + } + + if (sb->num_base_bdevs != raid->num_base_bdevs) { + SPDK_ERRLOG("The '%s' bdev's number of devices isn't equal to RAID's\n", + base_info->name); + return -EINVAL; + } + + if (!raid_bdev_is_raid_level(sb->level)) { + SPDK_ERRLOG("Invalid RAID level in base '%s' bdev's metadata. Retrieving of array isn't possible\n", + base_info->name); + return -EINVAL; + } + raid->num_base_bdevs = sb->num_base_bdevs; raid->num_base_bdevs_discovered = sb->num_base_bdevs; raid->level = sb->level; @@ -1228,6 +1246,9 @@ raid_bdev_sb_init_validation(struct raid_bdev *raid, struct raid_base_bdev_info raid->strip_size = sb->strip_size; raid->strip_size_kb = sb->strip_size * sb->blocklen; + return 0; +} + static int raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) { From adda1c261ef47b4b17af9969c3f32dabb8a20c65 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 17:30:00 +0000 Subject: [PATCH 17/47] module/raid: add base_bdev_capture_super_validate and base_bdev_super_init functions --- module/bdev/raid/bdev_raid.c | 48 ++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 4096e57ff32..9ab1ce1d3f7 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1270,6 +1270,54 @@ raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) return 0; } + +static int +raid_bdev_base_bdev_capture_super_validate(struct raid_base_bdev_info *base_info) { + struct raid_superblock *sb = base_info->raid_sb; + struct raid_bdev *raid = base_info->raid_bdev; + + /* The absence of a rebuild does not allow to capture not SPDK RAID disks or disks from the another RAID now. + * Redo when the rebuild will be done. + */ + if (base_info->is_new) { + SPDK_ERRLOG("The bdev '%s' haven't been in a SPDK RAID\n", + base_info->name); + return -EINVAL; + } else if (spdk_uuid_compare(&sb->uuid, &raid->bdev.uuid)) { + SPDK_ERRLOG("The bdev '%s' have been in another RAID bdev\n", + base_info->name); + return -EINVAL; + } + + if (sb->array_position != base_info->position) { + SPDK_WARNLOG("The base bdev '%s' has changed position in the raid from '%n' to '%n'\n", base_info->name, + sb->array_position, base_info->position); + } + + if (sb->blocklen != raid->bdev.blocklen) { + SPDK_ERRLOG("The logical block length stored in metadata of base '%s' bdev differ from RAID block length\n", + base_info->name); + return -EINVAL; + } + + if (sb->strip_size != raid->strip_size) { + SPDK_WARNLOG("The strip size stored in metadata of base '%s' bdev differ from RAID strip size and" + "it will be overwritten\n", base_info->name); + } + + return 0; +} + +static int +raid_bdev_base_bdev_super_init(struct raid_base_bdev_info *base_info) { + struct raid_superblock *sb = base_info->raid_sb; + struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); + struct timespec time_0 = {.tv_nsec = 0, .tv_sec = 0}; + + sb->blockcnt = base_bdev->blockcnt; + sb->timestamp = time_0; + + return 0; } static int From 4e6a00538b68009de9d6eab189ad03066d26b4ec Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 17:31:03 +0000 Subject: [PATCH 18/47] module/raid: refactor in base_bdev_super_validate --- module/bdev/raid/bdev_raid.c | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 9ab1ce1d3f7..e6ce1da6fe1 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1321,7 +1321,7 @@ raid_bdev_base_bdev_super_init(struct raid_base_bdev_info *base_info) { } static int -raid_bdev_base_bdev_sb_validate(struct raid_base_bdev_info *base_info) +raid_bdev_base_bdev_super_validate(struct raid_base_bdev_info *base_info, bool recreate) { int rc = 0; struct raid_superblock *sb = base_info->raid_sb; @@ -1330,22 +1330,17 @@ raid_bdev_base_bdev_sb_validate(struct raid_base_bdev_info *base_info) if (raid->num_base_bdevs <= 0 || !sb) return 0; - if (raid->is_new && raid_bdev_sb_init_validation(raid, base_info)) + if (!recreate && raid->is_new && raid_bdev_super_init_validation(raid, base_info)) return -EINVAL; - if (sb->version != RAID_METADATA_VERSION_01) { - SPDK_ERRLOG("Unsupported version of raid metadata has been found in base bdev '%s' superblock\n", base_info->name); - return -EINVAL; - } - - if (!raid->is_new) { - sb->blocklen = raid->bdev.blocklen; - sb->blockcnt = spdk_bdev_desc_get_bdev(base_info->desc)->blockcnt; - sb->raid_blockcnt = raid->bdev.blockcnt; + if (!raid->is_new || recreate) { + if (!recreate && raid_bdev_base_bdev_capture_super_validate(base_info)) + return -EINVAL; + + raid_bdev_base_bdev_super_sync(base_info); - if (sb->array_position != base_info->position) - SPDK_WARNLOG("The base bdev '%s' has changed position in the raid from '%n' to '%n'\n", base_info->name, - sb->array_position, base_info->position); + if (recreate) + raid_bdev_base_bdev_super_init(base_info); } return rc; From 069b7b6e5bc8546c68103e49d3fc4b4c72a677a0 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 5 Dec 2023 17:33:07 +0000 Subject: [PATCH 19/47] module/raid: add analyse_superblocks function it's main function for operate with raid metadata during initialization --- module/bdev/raid/bdev_raid.c | 37 ++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index e6ce1da6fe1..ff8ba6cdfe3 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1346,6 +1346,43 @@ raid_bdev_base_bdev_super_validate(struct raid_base_bdev_info *base_info, bool r return rc; } +static int +raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) +{ + int rc = 0; + struct raid_base_bdev_info *base_info; + struct raid_base_bdev_info *freshest = NULL; + + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + rc = raid_bdev_base_bdev_super_load(base_info, &freshest); + if (rc) { + SPDK_DEBUGLOG("super_load for base bdev failed'%s'\n", base_info->name); + return rc; + } + } + + if (!freshest) { + SPDK_DEBUGLOG("There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); + return 0; + } + + rc = raid_bdev_base_bdev_super_validate(freshest, sb_recreate); + if (rc) { + SPDK_DEBUGLOG("super_validate for freshest '%s' bdev have failed\n", base_info->name); + return rc; + } + + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + raid_bdev_base_bdev_super_validate(base_info, sb_recreate); + if (rc) { + SPDK_DEBUGLOG("super_validate for '%s' bdev have failed\n", base_info.name); + return rc; + } + } + + return rc; +} + /* * brief: * If raid bdev config is complete, then only register the raid bdev to From 3d626e31ad189a49ea72619b3c0f0cf6ca2d1a2f Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 19 Dec 2023 21:03:38 +0000 Subject: [PATCH 20/47] module/raid: fix load_base_bdevs_sb add spdk_bdev_read_blocks error return add free of io channel change message in read callback --- module/bdev/raid/bdev_raid.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index ff8ba6cdfe3..980eaa956e3 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1134,9 +1134,9 @@ raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, struct raid_base_bdev_info *base_info = cb_arg; if (success) { - SPDK_NOTICELOG("Read superblock from base bdev : %s\n", base_info->name); + SPDK_NOTICELOG("Read superblock from base bdev : '%s'\n", base_info->name); } else { - SPDK_ERRLOG("base bdev '%s' superblock io read error\n", base_info->name); + SPDK_ERRLOG("base bdev '%s' superblock io read has failed\n", base_info->name); } spdk_bdev_free_io(bdev_io); @@ -1147,6 +1147,7 @@ raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, static int raid_bdev_load_base_bdevs_sb(struct raid_base_bdev_info *base_info, uint32_t sb_size) { + int rc = 0; struct spdk_io_channel *ch; struct spdk_bdev_channel *ctx; struct spdk_bdev_io *bdev_io; @@ -1161,10 +1162,12 @@ raid_bdev_load_base_bdevs_sb(struct raid_base_bdev_info *base_info, uint32_t sb_ return -ENOMEM; } - spdk_bdev_read_blocks(base_info->desc, ch, base_info->raid_sb, 0, + rc = spdk_bdev_read_blocks(base_info->desc, ch, base_info->raid_sb, 0, sb_size, (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_read_sb_compete, base_info); - return 0; + + spdk_put_io_channel(ch); + return rc; } static int From 5652b037ee039f60e9378d16d7be20af1a274850 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 19 Dec 2023 21:04:09 +0000 Subject: [PATCH 21/47] module/raid: add base_bdev_write_sb function --- module/bdev/raid/bdev_raid.c | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 980eaa956e3..cbf490e9578 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1129,6 +1129,42 @@ raid_bdev_configure_md(struct raid_bdev *raid_bdev) return 0; } +static int +raid_bdev_base_bdev_write_sb_compete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { + struct raid_base_bdev_info *base_info = cb_arg; + + if (success) { + SPDK_NOTICELOG("Write superblock to base bdev : '%s'\n", base_info->name); + } else { + SPDK_ERRLOG("Base bdev '%s' superblock io write has failed\n", base_info->name); + } + + spdk_bdev_free_io(bdev_io); +} + +static int +raid_bdev_base_bdev_write_sb(struct raid_base_bdev_info *base_info) +{ + int rc = 0; + struct spdk_io_channel *ch; + struct spdk_bdev_io *bdev_io; + + ch = spdk_bdev_get_io_channel(base_info->desc); + + if (!ch) { + SPDK_ERRLOG("Unable to create io channel for bdev '%s'\n", base_info->name); + return -ENOMEM; + } + + rc = spdk_bdev_write_blocks(base_info->desc, ch, base_info->raid_sb, 0, + RAID_SB_BLOCKS(spdk_bdev_desc_get_bdev(base_info)->blocklen), + (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_write_sb_compete, + base_info); + + spdk_put_io_channel(ch); + return rc; +} + static int raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { struct raid_base_bdev_info *base_info = cb_arg; From 0e04416c69dbf0d21454c13a76b00d87c1070403 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 19 Dec 2023 21:07:22 +0000 Subject: [PATCH 22/47] module/raid: refactor of super load delete useless ctx in load_base_bdevs_sb rename load_base_bdevs_sb to base_bdev_read_sb replace bad from bad to if base_bdev_read_sb --- module/bdev/raid/bdev_raid.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index cbf490e9578..389169b615c 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1181,11 +1181,10 @@ raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, } static int -raid_bdev_load_base_bdevs_sb(struct raid_base_bdev_info *base_info, uint32_t sb_size) +raid_bdev_base_bdev_read_sb(struct raid_base_bdev_info *base_info, uint32_t sb_size) { int rc = 0; struct spdk_io_channel *ch; - struct spdk_bdev_channel *ctx; struct spdk_bdev_io *bdev_io; if (base_info->sb_loaded) @@ -1218,10 +1217,11 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai return -ENOMEM; } - rc = raid_bdev_load_base_bdevs_sb(base_info, RAID_SB_BLOCKS(base_bdev->blocklen)); + rc = raid_bdev_base_bdev_read_sb(base_info, RAID_SB_BLOCKS(base_bdev->blocklen)); if (rc) { SPDK_ERRLOG("Failed while read superblock from base bdev '%s'\n", base_info->name); - goto bad; + base_info->sb_loaded = false; + return rc; } struct raid_superblock *sb = base_info->raid_sb; @@ -1244,10 +1244,6 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai *freshest = base_info; return 0; - -bad: - base_info->sb_loaded = false; - return rc; } static int From 4cee03219cebb39c5363fe072068de8e16a7d5e7 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Tue, 19 Dec 2023 21:16:48 +0000 Subject: [PATCH 23/47] module/raid: add write of superblock to analyse_superblocks also fix typo in DEBUGLOGSs --- module/bdev/raid/bdev_raid.c | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 389169b615c..6492865475b 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1391,7 +1391,7 @@ raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { rc = raid_bdev_base_bdev_super_load(base_info, &freshest); if (rc) { - SPDK_DEBUGLOG("super_load for base bdev failed'%s'\n", base_info->name); + SPDK_DEBUGLOG("super_load for base bdev has failed'%s'\n", base_info->name); return rc; } } @@ -1403,19 +1403,26 @@ raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) rc = raid_bdev_base_bdev_super_validate(freshest, sb_recreate); if (rc) { - SPDK_DEBUGLOG("super_validate for freshest '%s' bdev have failed\n", base_info->name); + SPDK_DEBUGLOG("super_validate for freshest '%s' bdev has failed\n", base_info->name); return rc; } RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { raid_bdev_base_bdev_super_validate(base_info, sb_recreate); if (rc) { - SPDK_DEBUGLOG("super_validate for '%s' bdev have failed\n", base_info.name); + SPDK_DEBUGLOG("super_validate for '%s' bdev has failed\n", base_info.name); return rc; } } - return rc; + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + rc = raid_bdev_base_bdev_write_sb(base_info); + if (rc) { + SPDK_DEBUGLOG("Write superblock to '%s' bdev has failed"); + } + } + + return 0; } /* From fa3b1a6880c756c32d20dee60878bd67b8874ee9 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 03:07:04 +0000 Subject: [PATCH 24/47] rpc.py: add retrieve argument to bdev_raid_create --- python/spdk/rpc/bdev.py | 6 ++++-- scripts/rpc.py | 5 ++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/python/spdk/rpc/bdev.py b/python/spdk/rpc/bdev.py index 25fd6f83042..b674653d492 100644 --- a/python/spdk/rpc/bdev.py +++ b/python/spdk/rpc/bdev.py @@ -402,7 +402,8 @@ def bdev_raid_get_bdevs(client, category): return client.call('bdev_raid_get_bdevs', params) -def bdev_raid_create(client, name, raid_level, base_bdevs, strip_size=None, strip_size_kb=None, uuid=None, superblock=False): +def bdev_raid_create(client, name, raid_level, base_bdevs, strip_size=None, strip_size_kb=None, uuid=None, superblock=False, + retrieve=False): """Create raid bdev. Either strip size arg will work but one is required. Args: @@ -414,11 +415,12 @@ def bdev_raid_create(client, name, raid_level, base_bdevs, strip_size=None, stri uuid: UUID for this raid bdev (optional) superblock: information about raid bdev will be stored in superblock on each base bdev, disabled by default due to backward compatibility + retrieve: if specified, try to retrieve raid from superblock in base bdevs Returns: None """ - params = {'name': name, 'raid_level': raid_level, 'base_bdevs': base_bdevs, 'superblock': superblock} + params = {'name': name, 'raid_level': raid_level, 'base_bdevs': base_bdevs, 'superblock': superblock, 'retrieve': retrieve} if strip_size: params['strip_size'] = strip_size diff --git a/scripts/rpc.py b/scripts/rpc.py index 1ce9a8e6af0..b81e58d41d1 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2130,7 +2130,8 @@ def bdev_raid_create(args): raid_level=args.raid_level, base_bdevs=base_bdevs, uuid=args.uuid, - superblock=args.superblock) + superblock=args.superblock, + retrieve=args.retrieve) p = subparsers.add_parser('bdev_raid_create', help='Create new raid bdev') p.add_argument('-n', '--name', help='raid bdev name', required=True) p.add_argument('-z', '--strip-size-kb', help='strip size in KB', type=int) @@ -2139,6 +2140,8 @@ def bdev_raid_create(args): p.add_argument('--uuid', help='UUID for this raid bdev', required=False) p.add_argument('-s', '--superblock', help='information about raid bdev will be stored in superblock on each base bdev, ' 'disabled by default due to backward compatibility', action='store_true') + p.add_argument('--retrieve', help='try to recreate raid from superblock on base bdevs, it works only if superblock' + 'option specified', action='store_true') p.set_defaults(func=bdev_raid_create) def bdev_raid_delete(args): From 567da2dd317434366df027fc5c29bcf99d82af61 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 03:07:54 +0000 Subject: [PATCH 25/47] rpc.py: add bdev_raid_retrieve function as the wrapper for bdev_raid_create --- python/spdk/rpc/bdev.py | 16 ++++++++++++++++ scripts/rpc.py | 14 ++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/python/spdk/rpc/bdev.py b/python/spdk/rpc/bdev.py index b674653d492..ecc2692e009 100644 --- a/python/spdk/rpc/bdev.py +++ b/python/spdk/rpc/bdev.py @@ -434,6 +434,22 @@ def bdev_raid_create(client, name, raid_level, base_bdevs, strip_size=None, stri return client.call('bdev_raid_create', params) +def bdev_raid_retrieve(client, name, base_bdevs): + """Wrapper of bdev_raid_create. Tried to retrieve raid array from base bdevs. + + Args: + name: user defined raid bdev name for retrieved raid + base_bdevs: Space separated names of retrieving array base bdevs in double quotes, + like "Nvme0n1 Nvme1n1 Nvme2n1" + + Returns: + None + """ + params = {'name': name, 'raid_level': "raid1", 'base_bdevs': base_bdevs, 'superblock': True, 'retrieve': True} + + return client.call('bdev_raid_retrieve', params) + + def bdev_raid_delete(client, name): """Delete raid bdev diff --git a/scripts/rpc.py b/scripts/rpc.py index b81e58d41d1..1fe477a0d89 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2144,6 +2144,20 @@ def bdev_raid_create(args): 'option specified', action='store_true') p.set_defaults(func=bdev_raid_create) + def bdev_raid_retrieve(args): + base_bdevs = [] + for u in args.base_bdevs.strip().split(" "): + base_bdevs.append(u) + + rpc.bdev.bdev_raid_create(args.client, + name=args.name, + base_bdevs=base_bdevs) + p = subparsers.add_parser('bdev_raid_retrieve', help='Try to restore raid from metadata in base bdevs') + p.add_argument('-n', '--name', help='new name for retrieved raid', required=True) + p.add_argument('-b', '--base-bdevs', help='base bdevs name whiche have the superblock with raid metadata, ' + 'whitespace separated list in quotes', required=True) + p.set_defaults(func=bdev_raid_retrieve) + def bdev_raid_delete(args): rpc.bdev.bdev_raid_delete(args.client, name=args.name) From f0adcc046f3c6aa0e7685491202f647982fb6e06 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 03:09:24 +0000 Subject: [PATCH 26/47] module/raid: add retrieve argument to bdev_raid_rpc.c --- module/bdev/raid/bdev_raid_rpc.c | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/module/bdev/raid/bdev_raid_rpc.c b/module/bdev/raid/bdev_raid_rpc.c index e108d2f856a..67a65a9d93d 100644 --- a/module/bdev/raid/bdev_raid_rpc.c +++ b/module/bdev/raid/bdev_raid_rpc.c @@ -133,6 +133,9 @@ struct rpc_bdev_raid_create { /* If set, information about raid bdev will be stored in superblock on each base bdev */ bool superblock_enabled; + + /* If set, it tries to retrieve raid array from the base bdevs and fail if it can't */ + bool retrieve; }; /* @@ -199,6 +202,7 @@ static const struct spdk_json_object_decoder rpc_bdev_raid_create_decoders[] = { {"base_bdevs", offsetof(struct rpc_bdev_raid_create, base_bdevs), decode_base_bdevs}, {"uuid", offsetof(struct rpc_bdev_raid_create, uuid), spdk_json_decode_uuid, true}, {"superblock", offsetof(struct rpc_bdev_raid_create, superblock_enabled), spdk_json_decode_bool, true}, + {"retrieve", offsetof(struct rpc_bdev_raid_create, retrieve), spdk_json_decode_bool, true} }; /* @@ -228,6 +232,13 @@ rpc_bdev_raid_create(struct spdk_jsonrpc_request *request, goto cleanup; } + if (!req.superblock_enabled && req.retrieve) { + spdk_jsonrpc_send_error_response_fmt(request, -EINVAL, + "Retrieve option can be only if superblock is specified: %s", + spdk_strerror(-EINVAL)); + goto cleanup; + } + rc = raid_bdev_create(req.name, req.strip_size_kb, req.base_bdevs.num_base_bdevs, req.level, req.superblock_enabled, &req.uuid, &raid_bdev); if (rc != 0) { @@ -237,6 +248,10 @@ rpc_bdev_raid_create(struct spdk_jsonrpc_request *request, goto cleanup; } + if (req.retrieve) { + raid_bdev->is_new = false; + } + for (i = 0; i < req.base_bdevs.num_base_bdevs; i++) { const char *base_bdev_name = req.base_bdevs.base_bdevs[i]; From f4e2ac5637424d05899f9b7b8ab45debfe1eaf19 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 03:10:34 +0000 Subject: [PATCH 27/47] module/raid: change comment is_new in raid_bdev struct --- module/bdev/raid/bdev_raid.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index 7b9a0fb319f..5083ef9e192 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -200,7 +200,7 @@ struct raid_bdev { /* state of raid bdev */ enum raid_bdev_state state; - /* indicate if raid is new for raid_bdev_base_bdev_sb_validate */ + /* indicate if raid is new for */ bool is_new; /* number of base bdevs comprising raid bdev */ From a2b945533d8390ec882d27542d54b557f76c8e93 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 03:12:07 +0000 Subject: [PATCH 28/47] module/raid: raid_bdev's is_new indicate now if raid must be retrieved --- module/bdev/raid/bdev_raid.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 6492865475b..e671adcef8e 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1062,6 +1062,7 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs, raid_bdev->level = level; raid_bdev->min_base_bdevs_operational = min_operational; raid_bdev->superblock_enabled = superblock_enabled; + raid_bdev->is_new = true; raid_bdev_gen = &raid_bdev->bdev; @@ -1401,6 +1402,8 @@ raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) return 0; } + raid_bdev->is_new = true; + rc = raid_bdev_base_bdev_super_validate(freshest, sb_recreate); if (rc) { SPDK_DEBUGLOG("super_validate for freshest '%s' bdev has failed\n", base_info->name); @@ -1485,6 +1488,17 @@ raid_bdev_configure(struct raid_bdev *raid_bdev) return rc; } + if (raid_bdev->superblock_enabled) { + /* majoranta for superblock metadata configuration */ + raid_bdev_gen->blockcnt = blocklen * raid_bdev->num_base_bdevs; + + rc = raid_bdev_analyse_superblocks(raid_bdev, raid_bdev->is_new); + if (rc != 0) { + SPDK_ERRLOG("raid module superblock analyse failed\n"); + return rc; + } + } + rc = raid_bdev->module->start(raid_bdev); if (rc != 0) { SPDK_ERRLOG("raid module startup callback failed\n"); From 78b267015bf08cf3c88e2e71610e0e9459489972 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 03:12:47 +0000 Subject: [PATCH 29/47] module/raid: fix data_offset in configure_base_bdev --- module/bdev/raid/bdev_raid.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index e671adcef8e..893397305c1 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1887,7 +1887,8 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info) if (raid_bdev->superblock_enabled) { assert((RAID_BDEV_MIN_DATA_OFFSET_SIZE % bdev->blocklen) == 0); - base_info->data_offset = RAID_BDEV_MIN_DATA_OFFSET_SIZE / bdev->blocklen; + base_info->data_offset = spdk_max(RAID_BDEV_MIN_DATA_OFFSET_SIZE/bdev->blocklen, + RAID_SB_BLOCKS(bdev->blocklen)); if (bdev->optimal_io_boundary) { base_info->data_offset = spdk_divide_round_up(base_info->data_offset, From e8eeceda5cf2f0383f9198da9f4727e809cafefe Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 20:12:15 +0000 Subject: [PATCH 30/47] module/raid: move module functionality in raid_bdev_create to new raid_bdev_module_init function --- module/bdev/raid/bdev_raid.c | 103 +++++++++++++++++++---------------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 893397305c1..e4f3adb743c 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -949,6 +949,58 @@ raid_bdev_init(void) return 0; } +static int +raid_bdev_module_init(struct raid_bdev *raid_bdev) { + struct raid_bdev_module *module; + uint8_t min_operational; + + module = raid_bdev_module_find(raid_bdev->level); + if (module == NULL) { + SPDK_ERRLOG("Unsupported raid_bdev level '%d'\n", raid_bdev->level); + return -EINVAL; + } + + assert(module->base_bdevs_min != 0); + if (raid_bdev->num_base_bdevs < module->base_bdevs_min) { + SPDK_ERRLOG("At least %u base devices required for %s\n", + module->base_bdevs_min, + raid_bdev_level_to_str(raid_bdev->level)); + return -EINVAL; + } + + switch (module->base_bdevs_constraint.type) { + case CONSTRAINT_MAX_BASE_BDEVS_REMOVED: + min_operational = raid_bdev->num_base_bdevs - module->base_bdevs_constraint.value; + break; + case CONSTRAINT_MIN_BASE_BDEVS_OPERATIONAL: + min_operational = module->base_bdevs_constraint.value; + break; + case CONSTRAINT_UNSET: + if (module->base_bdevs_constraint.value != 0) { + SPDK_ERRLOG("Unexpected constraint value '%u' provided for raid_bdev bdev '%s'.\n", + (uint8_t)module->base_bdevs_constraint.value, raid_bdev->bdev.name); + return -EINVAL; + } + min_operational = raid_bdev->num_base_bdevs; + break; + default: + SPDK_ERRLOG("Unrecognised constraint type '%u' in module for raid_bdev level '%s'.\n", + (uint8_t)module->base_bdevs_constraint.type, + raid_bdev_level_to_str(module->level)); + return -EINVAL; + }; + + if (min_operational == 0 || min_operational > raid_bdev->num_base_bdevs) { + SPDK_ERRLOG("Wrong constraint value for raid_bdev level '%s'.\n", + raid_bdev_level_to_str(module->level)); + return -EINVAL; + } + + raid_bdev->min_base_bdevs_operational = min_operational; + + return 0; +} + /* * brief: * raid_bdev_create allocates raid bdev based on passed configuration @@ -971,9 +1023,7 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs, { struct raid_bdev *raid_bdev; struct spdk_bdev *raid_bdev_gen; - struct raid_bdev_module *module; struct raid_base_bdev_info *base_info; - uint8_t min_operational; if (raid_bdev_find_by_name(name) != NULL) { SPDK_ERRLOG("Duplicate raid bdev name found: %s\n", name); @@ -990,48 +1040,6 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs, return -EINVAL; } - module = raid_bdev_module_find(level); - if (module == NULL) { - SPDK_ERRLOG("Unsupported raid level '%d'\n", level); - return -EINVAL; - } - - assert(module->base_bdevs_min != 0); - if (num_base_bdevs < module->base_bdevs_min) { - SPDK_ERRLOG("At least %u base devices required for %s\n", - module->base_bdevs_min, - raid_bdev_level_to_str(level)); - return -EINVAL; - } - - switch (module->base_bdevs_constraint.type) { - case CONSTRAINT_MAX_BASE_BDEVS_REMOVED: - min_operational = num_base_bdevs - module->base_bdevs_constraint.value; - break; - case CONSTRAINT_MIN_BASE_BDEVS_OPERATIONAL: - min_operational = module->base_bdevs_constraint.value; - break; - case CONSTRAINT_UNSET: - if (module->base_bdevs_constraint.value != 0) { - SPDK_ERRLOG("Unexpected constraint value '%u' provided for raid bdev '%s'.\n", - (uint8_t)module->base_bdevs_constraint.value, name); - return -EINVAL; - } - min_operational = num_base_bdevs; - break; - default: - SPDK_ERRLOG("Unrecognised constraint type '%u' in module for raid level '%s'.\n", - (uint8_t)module->base_bdevs_constraint.type, - raid_bdev_level_to_str(module->level)); - return -EINVAL; - }; - - if (min_operational == 0 || min_operational > num_base_bdevs) { - SPDK_ERRLOG("Wrong constraint value for raid level '%s'.\n", - raid_bdev_level_to_str(module->level)); - return -EINVAL; - } - raid_bdev = calloc(1, sizeof(*raid_bdev)); if (!raid_bdev) { SPDK_ERRLOG("Unable to allocate memory for raid bdev\n"); @@ -1039,7 +1047,6 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs, } spdk_spin_init(&raid_bdev->base_bdev_lock); - raid_bdev->module = module; raid_bdev->num_base_bdevs = num_base_bdevs; raid_bdev->base_bdev_info = calloc(raid_bdev->num_base_bdevs, sizeof(struct raid_base_bdev_info)); @@ -1060,10 +1067,14 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs, raid_bdev->strip_size_kb = strip_size; raid_bdev->state = RAID_BDEV_STATE_CONFIGURING; raid_bdev->level = level; - raid_bdev->min_base_bdevs_operational = min_operational; raid_bdev->superblock_enabled = superblock_enabled; raid_bdev->is_new = true; + if (raid_bdev_module_init(raid_bdev)) { + raid_bdev_free(raid_bdev); + return -EINVAL; + } + raid_bdev_gen = &raid_bdev->bdev; raid_bdev_gen->name = strdup(name); From c1332c55ee6f2dbc6bc3da0f79be42db01d8c84e Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 20:14:27 +0000 Subject: [PATCH 31/47] module/raid: add module reinit when retrieve raid in raid_bdev_configure --- module/bdev/raid/bdev_raid.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index e4f3adb743c..cf17f0be32f 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1500,14 +1500,20 @@ raid_bdev_configure(struct raid_bdev *raid_bdev) } if (raid_bdev->superblock_enabled) { + bool recreate = raid_bdev->is_new; + /* majoranta for superblock metadata configuration */ raid_bdev_gen->blockcnt = blocklen * raid_bdev->num_base_bdevs; - rc = raid_bdev_analyse_superblocks(raid_bdev, raid_bdev->is_new); + rc = raid_bdev_analyse_superblocks(raid_bdev, recreate); if (rc != 0) { SPDK_ERRLOG("raid module superblock analyse failed\n"); return rc; } + + if (!recreate && raid_bdev_module_init(raid_bdev)) { + return -EINVAL; + } } rc = raid_bdev->module->start(raid_bdev); From 1a1b322c40e37f3f9077d5a9dd481b4bddcebe50 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 21:48:54 +0000 Subject: [PATCH 32/47] module/raid: fix raid_superblock struct change level type from uint32_t to int32_t fix pragma pack --- module/bdev/raid/bdev_raid.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index 5083ef9e192..cfb29564737 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -72,7 +72,7 @@ struct raid_superblock { uint8_t num_base_bdevs; /* Raid Level of device's raid */ - uint32_t level; + int32_t level; /* Position of device in raid */ uint32_t array_position; @@ -92,7 +92,7 @@ struct raid_superblock { /* UUID of raid bdev */ struct spdk_uuid uuid; }; -#pragma pop +#pragma pack(pop) /* * raid_base_bdev_info contains information for the base bdevs which are part of some From c4f2560dde06e13e439f6196eed3d31c0e5eda68 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 21:53:43 +0000 Subject: [PATCH 33/47] module/raid: fix raid_bdev_is_raid_level make it static change level type from uint32_t to int32_t --- module/bdev/raid/bdev_raid.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index cf17f0be32f..f8af2a3f548 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -795,10 +795,10 @@ static struct { { } }; -bool -raid_bdev_is_raid_level(uint32_t level) { +static bool +raid_bdev_is_raid_level(int32_t level) { for (int i = 0; g_raid_level_names[i].name != NULL; i++) { - if (level == g_raid_level_names[i].value) + if (level == (int32_t) g_raid_level_names[i].value) return true; } From d0d98d9df0013be6fceb00dd9d82055d6241deca Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 21:55:47 +0000 Subject: [PATCH 34/47] module/raid: refactor - clean code --- module/bdev/raid/bdev_raid.c | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index f8af2a3f548..f7ab16591cd 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -988,7 +988,7 @@ raid_bdev_module_init(struct raid_bdev *raid_bdev) { (uint8_t)module->base_bdevs_constraint.type, raid_bdev_level_to_str(module->level)); return -EINVAL; - }; + } if (min_operational == 0 || min_operational > raid_bdev->num_base_bdevs) { SPDK_ERRLOG("Wrong constraint value for raid_bdev level '%s'.\n", @@ -1159,7 +1159,6 @@ raid_bdev_base_bdev_write_sb(struct raid_base_bdev_info *base_info) { int rc = 0; struct spdk_io_channel *ch; - struct spdk_bdev_io *bdev_io; ch = spdk_bdev_get_io_channel(base_info->desc); @@ -1197,7 +1196,6 @@ raid_bdev_base_bdev_read_sb(struct raid_base_bdev_info *base_info, uint32_t sb_s { int rc = 0; struct spdk_io_channel *ch; - struct spdk_bdev_io *bdev_io; if (base_info->sb_loaded) return 0; @@ -1261,7 +1259,6 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai static int raid_bdev_super_init_validation(struct raid_bdev *raid, struct raid_base_bdev_info *base_info) { - int rc = 0; struct raid_superblock *sb = base_info->raid_sb; assert(raid->num_base_bdevs == raid->num_base_bdevs_discovered); From c6acbe4c5bc26eddf238f279db33e584136c9de2 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 21:56:42 +0000 Subject: [PATCH 35/47] module/raid: refactor - make some function static --- module/bdev/raid/bdev_raid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index f7ab16591cd..a20b1d520b3 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1141,7 +1141,7 @@ raid_bdev_configure_md(struct raid_bdev *raid_bdev) return 0; } -static int +static void raid_bdev_base_bdev_write_sb_compete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { struct raid_base_bdev_info *base_info = cb_arg; @@ -1176,7 +1176,7 @@ raid_bdev_base_bdev_write_sb(struct raid_base_bdev_info *base_info) return rc; } -static int +static void raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { struct raid_base_bdev_info *base_info = cb_arg; From 7b36b6255b0c634e30b121a2203496152da00977 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 21:59:24 +0000 Subject: [PATCH 36/47] module/raid: fix small error in bdev_capture_super_validate and base_bdev_write_sb --- module/bdev/raid/bdev_raid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index a20b1d520b3..5db65fc9c6a 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1168,7 +1168,7 @@ raid_bdev_base_bdev_write_sb(struct raid_base_bdev_info *base_info) } rc = spdk_bdev_write_blocks(base_info->desc, ch, base_info->raid_sb, 0, - RAID_SB_BLOCKS(spdk_bdev_desc_get_bdev(base_info)->blocklen), + RAID_SB_BLOCKS(spdk_bdev_desc_get_bdev(base_info->desc)->blocklen), (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_write_sb_compete, base_info); @@ -1335,7 +1335,7 @@ raid_bdev_base_bdev_capture_super_validate(struct raid_base_bdev_info *base_info if (sb->array_position != base_info->position) { SPDK_WARNLOG("The base bdev '%s' has changed position in the raid from '%n' to '%n'\n", base_info->name, - sb->array_position, base_info->position); + &sb->array_position, &base_info->position); } if (sb->blocklen != raid->bdev.blocklen) { From 819c048bbca852ba675f3b9bfd8f89dfed931254 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 23:42:01 +0000 Subject: [PATCH 37/47] module/raid: fix DEBUGLOG --- module/bdev/raid/bdev_raid.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 5db65fc9c6a..ec1b43ad9b9 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1400,13 +1400,13 @@ raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { rc = raid_bdev_base_bdev_super_load(base_info, &freshest); if (rc) { - SPDK_DEBUGLOG("super_load for base bdev has failed'%s'\n", base_info->name); + SPDK_DEBUGLOG(bdev_raid, "super_load for base bdev has failed'%s'\n", base_info->name); return rc; } } if (!freshest) { - SPDK_DEBUGLOG("There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); + SPDK_DEBUGLOG(bdev_raid, "There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); return 0; } @@ -1414,14 +1414,14 @@ raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) rc = raid_bdev_base_bdev_super_validate(freshest, sb_recreate); if (rc) { - SPDK_DEBUGLOG("super_validate for freshest '%s' bdev has failed\n", base_info->name); + SPDK_DEBUGLOG(bdev_raid, "super_validate for freshest '%s' bdev has failed\n", base_info->name); return rc; } RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { raid_bdev_base_bdev_super_validate(base_info, sb_recreate); if (rc) { - SPDK_DEBUGLOG("super_validate for '%s' bdev has failed\n", base_info.name); + SPDK_DEBUGLOG(bdev_raid, "super_validate for '%s' bdev has failed\n", base_info->name); return rc; } } @@ -1429,7 +1429,7 @@ raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { rc = raid_bdev_base_bdev_write_sb(base_info); if (rc) { - SPDK_DEBUGLOG("Write superblock to '%s' bdev has failed"); + SPDK_DEBUGLOG(bdev_raid, "Write superblock to '%s' bdev has failed\n", base_info->name); } } From 790da02df1a0520c89db53b841086b72aba11fad Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Wed, 20 Dec 2023 23:51:47 +0000 Subject: [PATCH 38/47] module/raid: remove useless sb_loaded from raid_base_bdev_info --- module/bdev/raid/bdev_raid.c | 6 ------ module/bdev/raid/bdev_raid.h | 3 --- 2 files changed, 9 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index ec1b43ad9b9..5fdeab5f639 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1187,8 +1187,6 @@ raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, } spdk_bdev_free_io(bdev_io); - - base_info->sb_loaded = true; } static int @@ -1197,9 +1195,6 @@ raid_bdev_base_bdev_read_sb(struct raid_base_bdev_info *base_info, uint32_t sb_s int rc = 0; struct spdk_io_channel *ch; - if (base_info->sb_loaded) - return 0; - ch = spdk_bdev_get_io_channel(base_info->desc); if (!ch) { @@ -1230,7 +1225,6 @@ raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct rai rc = raid_bdev_base_bdev_read_sb(base_info, RAID_SB_BLOCKS(base_bdev->blocklen)); if (rc) { SPDK_ERRLOG("Failed while read superblock from base bdev '%s'\n", base_info->name); - base_info->sb_loaded = false; return rc; } diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index cfb29564737..c7b8e881f30 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -115,9 +115,6 @@ struct raid_base_bdev_info { /* indicate if the metadata was on disk or not */ bool is_new; - /* true if sb has loaded */ - bool sb_loaded; - /* base bdev's superblock */ struct raid_superblock *raid_sb; From a089b485cdc321af995610e58d0d9b2e66cf8997 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 00:05:11 +0000 Subject: [PATCH 39/47] module/raid: now analyse superblock return EINVAL if there isn't superblock in raid --- module/bdev/raid/bdev_raid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 5fdeab5f639..870eb209284 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1400,8 +1400,8 @@ raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) } if (!freshest) { - SPDK_DEBUGLOG(bdev_raid, "There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); - return 0; + SPDK_ERRLOG("There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); + return -EINVAL; } raid_bdev->is_new = true; From 9065fc34de3b5d32490ca384bdaacbf7d6c621bb Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 05:48:13 +0000 Subject: [PATCH 40/47] module/raid: fix error with raid module --- module/bdev/raid/bdev_raid.c | 1 + 1 file changed, 1 insertion(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 870eb209284..c1955574fbd 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -997,6 +997,7 @@ raid_bdev_module_init(struct raid_bdev *raid_bdev) { } raid_bdev->min_base_bdevs_operational = min_operational; + raid_bdev->module = module; return 0; } From c460e261f5d1fbf786d7145e864ad8fab51e3a50 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 05:49:02 +0000 Subject: [PATCH 41/47] module/raid: fix uuid copy problem --- module/bdev/raid/bdev_raid.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index c1955574fbd..3c69f43346d 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1281,7 +1281,7 @@ raid_bdev_super_init_validation(struct raid_bdev *raid, struct raid_base_bdev_in raid->level = sb->level; raid->bdev.blocklen = sb->blocklen; raid->bdev.blockcnt = sb->raid_blockcnt; - raid->bdev.uuid = sb->uuid; + spdk_uuid_copy(&raid->bdev.uuid, &sb->uuid); raid->strip_size = sb->strip_size; raid->strip_size_kb = sb->strip_size * sb->blocklen; @@ -1303,7 +1303,7 @@ raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) sb->array_position = base_info->position; sb->strip_size = raid->strip_size; sb->raid_blockcnt = raid->bdev.blockcnt; - sb->uuid = raid->bdev.uuid; + spdk_uuid_copy(&sb->uuid, &raid->bdev.uuid); memset(sb+1, 0, RAID_SB_BLOCKS(base_bdev->blocklen)); From 29486a7125bd28d2ad46303b260fbfcf352352d1 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 05:50:42 +0000 Subject: [PATCH 42/47] module/raid: fix - add missing raid->is_new = false also add DEBUGLOG --- module/bdev/raid/bdev_raid.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 3c69f43346d..0750a16c97c 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1285,6 +1285,8 @@ raid_bdev_super_init_validation(struct raid_bdev *raid, struct raid_base_bdev_in raid->strip_size = sb->strip_size; raid->strip_size_kb = sb->strip_size * sb->blocklen; + raid->is_new = false; + return 0; } @@ -1400,6 +1402,8 @@ raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) } } + SPDK_DEBUGLOG(bdev_raid, "The freshest bdev is '%s'\n", freshest->name); + if (!freshest) { SPDK_ERRLOG("There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); return -EINVAL; From 867cc5df4a2ef4beaeedfcf9f0b774d3c150ef0f Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 07:29:08 +0000 Subject: [PATCH 43/47] module/raid: divide analyse_superblock into two parts in raid_bdev_configure --- module/bdev/raid/bdev_raid.c | 102 ++++++++++++++++------------------- 1 file changed, 45 insertions(+), 57 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 0750a16c97c..e8de31d09ab 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1147,7 +1147,7 @@ raid_bdev_base_bdev_write_sb_compete(struct spdk_bdev_io *bdev_io, bool success, struct raid_base_bdev_info *base_info = cb_arg; if (success) { - SPDK_NOTICELOG("Write superblock to base bdev : '%s'\n", base_info->name); + SPDK_DEBUGLOG(bdev_raid, "Write superblock to base bdev : '%s'\n", base_info->name); } else { SPDK_ERRLOG("Base bdev '%s' superblock io write has failed\n", base_info->name); } @@ -1182,7 +1182,7 @@ raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, struct raid_base_bdev_info *base_info = cb_arg; if (success) { - SPDK_NOTICELOG("Read superblock from base bdev : '%s'\n", base_info->name); + SPDK_DEBUGLOG(bdev_raid, "Read superblock from base bdev : '%s'\n", base_info->name); } else { SPDK_ERRLOG("base bdev '%s' superblock io read has failed\n", base_info->name); } @@ -1258,6 +1258,11 @@ raid_bdev_super_init_validation(struct raid_bdev *raid, struct raid_base_bdev_in assert(raid->num_base_bdevs == raid->num_base_bdevs_discovered); + if (base_info->is_new) { + SPDK_ERRLOG("There isn't any metadata on base bdevs for raid '%s'\n", raid->bdev.name); + return -EINVAL; + } + if (sb->version != RAID_METADATA_VERSION_01) { SPDK_ERRLOG("Unsupported version of raid metadata has been found in base '%s' bdev's superblock\n", base_info->name); @@ -1387,54 +1392,6 @@ raid_bdev_base_bdev_super_validate(struct raid_base_bdev_info *base_info, bool r return rc; } -static int -raid_bdev_analyse_superblocks(struct raid_bdev *raid_bdev, bool sb_recreate) -{ - int rc = 0; - struct raid_base_bdev_info *base_info; - struct raid_base_bdev_info *freshest = NULL; - - RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { - rc = raid_bdev_base_bdev_super_load(base_info, &freshest); - if (rc) { - SPDK_DEBUGLOG(bdev_raid, "super_load for base bdev has failed'%s'\n", base_info->name); - return rc; - } - } - - SPDK_DEBUGLOG(bdev_raid, "The freshest bdev is '%s'\n", freshest->name); - - if (!freshest) { - SPDK_ERRLOG("There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); - return -EINVAL; - } - - raid_bdev->is_new = true; - - rc = raid_bdev_base_bdev_super_validate(freshest, sb_recreate); - if (rc) { - SPDK_DEBUGLOG(bdev_raid, "super_validate for freshest '%s' bdev has failed\n", base_info->name); - return rc; - } - - RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { - raid_bdev_base_bdev_super_validate(base_info, sb_recreate); - if (rc) { - SPDK_DEBUGLOG(bdev_raid, "super_validate for '%s' bdev has failed\n", base_info->name); - return rc; - } - } - - RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { - rc = raid_bdev_base_bdev_write_sb(base_info); - if (rc) { - SPDK_DEBUGLOG(bdev_raid, "Write superblock to '%s' bdev has failed\n", base_info->name); - } - } - - return 0; -} - /* * brief: * If raid bdev config is complete, then only register the raid bdev to @@ -1453,6 +1410,8 @@ raid_bdev_configure(struct raid_bdev *raid_bdev) struct spdk_bdev *raid_bdev_gen; struct raid_base_bdev_info *base_info; struct spdk_bdev *base_bdev; + bool sb_recreate = raid_bdev->is_new; + struct raid_base_bdev_info *freshest = NULL; int rc = 0; assert(raid_bdev->state == RAID_BDEV_STATE_CONFIGURING); @@ -1496,18 +1455,30 @@ raid_bdev_configure(struct raid_bdev *raid_bdev) } if (raid_bdev->superblock_enabled) { - bool recreate = raid_bdev->is_new; + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + rc = raid_bdev_base_bdev_super_load(base_info, &freshest); + if (rc) { + SPDK_DEBUGLOG(bdev_raid, "super_load for base bdev has failed'%s'\n", base_info->name); + return rc; + } + } - /* majoranta for superblock metadata configuration */ - raid_bdev_gen->blockcnt = blocklen * raid_bdev->num_base_bdevs; + SPDK_DEBUGLOG(bdev_raid, "The freshest bdev is '%s'\n", freshest->name); - rc = raid_bdev_analyse_superblocks(raid_bdev, recreate); - if (rc != 0) { - SPDK_ERRLOG("raid module superblock analyse failed\n"); + if (!freshest) { + SPDK_ERRLOG("There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); + return -EINVAL; + } + + raid_bdev->is_new = true; + + rc = raid_bdev_base_bdev_super_validate(freshest, sb_recreate); + if (rc) { + SPDK_DEBUGLOG(bdev_raid, "super_validate for freshest '%s' bdev has failed\n", base_info->name); return rc; } - if (!recreate && raid_bdev_module_init(raid_bdev)) { + if (!sb_recreate && raid_bdev_module_init(raid_bdev)) { return -EINVAL; } } @@ -1538,6 +1509,23 @@ raid_bdev_configure(struct raid_bdev *raid_bdev) SPDK_DEBUGLOG(bdev_raid, "raid bdev is created with name %s, raid_bdev %p\n", raid_bdev_gen->name, raid_bdev); + if (raid_bdev->superblock_enabled) { + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + raid_bdev_base_bdev_super_validate(base_info, sb_recreate); + if (rc) { + SPDK_DEBUGLOG(bdev_raid, "super_validate for '%s' bdev has failed\n", base_info->name); + return rc; + } + } + + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + rc = raid_bdev_base_bdev_write_sb(base_info); + if (rc) { + SPDK_DEBUGLOG(bdev_raid, "Write superblock to '%s' bdev has failed\n", base_info->name); + } + } + } + return 0; } From 4f76442f1491f84171e2716c5cf488fa7efa2a2c Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 07:30:23 +0000 Subject: [PATCH 44/47] rpc.py: fix bdev_raid_retrieve --- python/spdk/rpc/bdev.py | 4 ++-- scripts/rpc.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/spdk/rpc/bdev.py b/python/spdk/rpc/bdev.py index ecc2692e009..8cf25dacbed 100644 --- a/python/spdk/rpc/bdev.py +++ b/python/spdk/rpc/bdev.py @@ -445,9 +445,9 @@ def bdev_raid_retrieve(client, name, base_bdevs): Returns: None """ - params = {'name': name, 'raid_level': "raid1", 'base_bdevs': base_bdevs, 'superblock': True, 'retrieve': True} + params = {'name': name, 'raid_level': 'raid1', 'base_bdevs': base_bdevs, 'superblock': True, 'retrieve': True} - return client.call('bdev_raid_retrieve', params) + return client.call('bdev_raid_create', params) def bdev_raid_delete(client, name): diff --git a/scripts/rpc.py b/scripts/rpc.py index 1fe477a0d89..dc22b42e46d 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2149,7 +2149,7 @@ def bdev_raid_retrieve(args): for u in args.base_bdevs.strip().split(" "): base_bdevs.append(u) - rpc.bdev.bdev_raid_create(args.client, + rpc.bdev.bdev_raid_retrieve(args.client, name=args.name, base_bdevs=base_bdevs) p = subparsers.add_parser('bdev_raid_retrieve', help='Try to restore raid from metadata in base bdevs') From 90d01d6d596462763f5bc44ba779c837d2afb4ea Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 11:13:09 +0000 Subject: [PATCH 45/47] rpc.py: fix typos in bdev_raid_retrieve and bdev_raid_create --- scripts/rpc.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/rpc.py b/scripts/rpc.py index dc22b42e46d..00ee5019579 100755 --- a/scripts/rpc.py +++ b/scripts/rpc.py @@ -2140,8 +2140,8 @@ def bdev_raid_create(args): p.add_argument('--uuid', help='UUID for this raid bdev', required=False) p.add_argument('-s', '--superblock', help='information about raid bdev will be stored in superblock on each base bdev, ' 'disabled by default due to backward compatibility', action='store_true') - p.add_argument('--retrieve', help='try to recreate raid from superblock on base bdevs, it works only if superblock' - 'option specified', action='store_true') + p.add_argument('--retrieve', help='try to recreate raid from the superblock on the base bdevs, it only works if superblock' + 'option is specified', action='store_true') p.set_defaults(func=bdev_raid_create) def bdev_raid_retrieve(args): @@ -2152,9 +2152,9 @@ def bdev_raid_retrieve(args): rpc.bdev.bdev_raid_retrieve(args.client, name=args.name, base_bdevs=base_bdevs) - p = subparsers.add_parser('bdev_raid_retrieve', help='Try to restore raid from metadata in base bdevs') - p.add_argument('-n', '--name', help='new name for retrieved raid', required=True) - p.add_argument('-b', '--base-bdevs', help='base bdevs name whiche have the superblock with raid metadata, ' + p = subparsers.add_parser('bdev_raid_retrieve', help='Try to restore the raid from metadata on the base bdevs') + p.add_argument('-n', '--name', help='new name for the retrieved raid', required=True) + p.add_argument('-b', '--base-bdevs', help='name of the base bdevs which have the superblock with raid metadata, ' 'whitespace separated list in quotes', required=True) p.set_defaults(func=bdev_raid_retrieve) From 1785bd136b4d2c123438e9bc4b96a9b3659739b3 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 19:54:28 +0000 Subject: [PATCH 46/47] module/raid: fix raid0/concat retrieve error add strip_size_shift and blocklen_shift calculation to super_init_validation --- module/bdev/raid/bdev_raid.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index e8de31d09ab..5d5649bff53 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -1289,6 +1289,8 @@ raid_bdev_super_init_validation(struct raid_bdev *raid, struct raid_base_bdev_in spdk_uuid_copy(&raid->bdev.uuid, &sb->uuid); raid->strip_size = sb->strip_size; raid->strip_size_kb = sb->strip_size * sb->blocklen; + raid->strip_size_shift = spdk_u32log2(sb->strip_size); + raid->blocklen_shift = spdk_u32log2(sb->blocklen); raid->is_new = false; From bb2fafa09d7208dbe223fd3a5bb901f91680e249 Mon Sep 17 00:00:00 2001 From: Arseniy Kuznetsov Date: Thu, 21 Dec 2023 21:27:01 +0000 Subject: [PATCH 47/47] module/raid: change spaces to tabs --- module/bdev/raid/bdev_raid.c | 550 +++++++++++++++---------------- module/bdev/raid/bdev_raid.h | 62 ++-- module/bdev/raid/bdev_raid_rpc.c | 24 +- 3 files changed, 318 insertions(+), 318 deletions(-) diff --git a/module/bdev/raid/bdev_raid.c b/module/bdev/raid/bdev_raid.c index 5d5649bff53..47a48363530 100644 --- a/module/bdev/raid/bdev_raid.c +++ b/module/bdev/raid/bdev_raid.c @@ -221,8 +221,8 @@ raid_bdev_free_base_bdev_resource(struct raid_base_bdev_info *base_info) free(base_info->name); base_info->name = NULL; - spdk_dma_free(base_info->raid_sb); - base_info->raid_sb = NULL; + spdk_dma_free(base_info->raid_sb); + base_info->raid_sb = NULL; if (base_info->desc == NULL) { return; @@ -797,12 +797,12 @@ static struct { static bool raid_bdev_is_raid_level(int32_t level) { - for (int i = 0; g_raid_level_names[i].name != NULL; i++) { - if (level == (int32_t) g_raid_level_names[i].value) - return true; - } + for (int i = 0; g_raid_level_names[i].name != NULL; i++) { + if (level == (int32_t) g_raid_level_names[i].value) + return true; + } - return false; + return false; } /* We have to use the typedef in the function declaration to appease astyle. */ @@ -951,55 +951,55 @@ raid_bdev_init(void) static int raid_bdev_module_init(struct raid_bdev *raid_bdev) { - struct raid_bdev_module *module; - uint8_t min_operational; - - module = raid_bdev_module_find(raid_bdev->level); - if (module == NULL) { - SPDK_ERRLOG("Unsupported raid_bdev level '%d'\n", raid_bdev->level); - return -EINVAL; - } - - assert(module->base_bdevs_min != 0); - if (raid_bdev->num_base_bdevs < module->base_bdevs_min) { - SPDK_ERRLOG("At least %u base devices required for %s\n", - module->base_bdevs_min, - raid_bdev_level_to_str(raid_bdev->level)); - return -EINVAL; - } - - switch (module->base_bdevs_constraint.type) { - case CONSTRAINT_MAX_BASE_BDEVS_REMOVED: - min_operational = raid_bdev->num_base_bdevs - module->base_bdevs_constraint.value; - break; - case CONSTRAINT_MIN_BASE_BDEVS_OPERATIONAL: - min_operational = module->base_bdevs_constraint.value; - break; - case CONSTRAINT_UNSET: - if (module->base_bdevs_constraint.value != 0) { - SPDK_ERRLOG("Unexpected constraint value '%u' provided for raid_bdev bdev '%s'.\n", - (uint8_t)module->base_bdevs_constraint.value, raid_bdev->bdev.name); - return -EINVAL; - } - min_operational = raid_bdev->num_base_bdevs; - break; - default: - SPDK_ERRLOG("Unrecognised constraint type '%u' in module for raid_bdev level '%s'.\n", - (uint8_t)module->base_bdevs_constraint.type, - raid_bdev_level_to_str(module->level)); - return -EINVAL; - } - - if (min_operational == 0 || min_operational > raid_bdev->num_base_bdevs) { - SPDK_ERRLOG("Wrong constraint value for raid_bdev level '%s'.\n", - raid_bdev_level_to_str(module->level)); - return -EINVAL; - } - - raid_bdev->min_base_bdevs_operational = min_operational; - raid_bdev->module = module; - - return 0; + struct raid_bdev_module *module; + uint8_t min_operational; + + module = raid_bdev_module_find(raid_bdev->level); + if (module == NULL) { + SPDK_ERRLOG("Unsupported raid_bdev level '%d'\n", raid_bdev->level); + return -EINVAL; + } + + assert(module->base_bdevs_min != 0); + if (raid_bdev->num_base_bdevs < module->base_bdevs_min) { + SPDK_ERRLOG("At least %u base devices required for %s\n", + module->base_bdevs_min, + raid_bdev_level_to_str(raid_bdev->level)); + return -EINVAL; + } + + switch (module->base_bdevs_constraint.type) { + case CONSTRAINT_MAX_BASE_BDEVS_REMOVED: + min_operational = raid_bdev->num_base_bdevs - module->base_bdevs_constraint.value; + break; + case CONSTRAINT_MIN_BASE_BDEVS_OPERATIONAL: + min_operational = module->base_bdevs_constraint.value; + break; + case CONSTRAINT_UNSET: + if (module->base_bdevs_constraint.value != 0) { + SPDK_ERRLOG("Unexpected constraint value '%u' provided for raid_bdev bdev '%s'.\n", + (uint8_t)module->base_bdevs_constraint.value, raid_bdev->bdev.name); + return -EINVAL; + } + min_operational = raid_bdev->num_base_bdevs; + break; + default: + SPDK_ERRLOG("Unrecognised constraint type '%u' in module for raid_bdev level '%s'.\n", + (uint8_t)module->base_bdevs_constraint.type, + raid_bdev_level_to_str(module->level)); + return -EINVAL; + } + + if (min_operational == 0 || min_operational > raid_bdev->num_base_bdevs) { + SPDK_ERRLOG("Wrong constraint value for raid_bdev level '%s'.\n", + raid_bdev_level_to_str(module->level)); + return -EINVAL; + } + + raid_bdev->min_base_bdevs_operational = min_operational; + raid_bdev->module = module; + + return 0; } /* @@ -1069,12 +1069,12 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs, raid_bdev->state = RAID_BDEV_STATE_CONFIGURING; raid_bdev->level = level; raid_bdev->superblock_enabled = superblock_enabled; - raid_bdev->is_new = true; + raid_bdev->is_new = true; - if (raid_bdev_module_init(raid_bdev)) { - raid_bdev_free(raid_bdev); - return -EINVAL; - } + if (raid_bdev_module_init(raid_bdev)) { + raid_bdev_free(raid_bdev); + return -EINVAL; + } raid_bdev_gen = &raid_bdev->bdev; @@ -1144,254 +1144,254 @@ raid_bdev_configure_md(struct raid_bdev *raid_bdev) static void raid_bdev_base_bdev_write_sb_compete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { - struct raid_base_bdev_info *base_info = cb_arg; + struct raid_base_bdev_info *base_info = cb_arg; - if (success) { - SPDK_DEBUGLOG(bdev_raid, "Write superblock to base bdev : '%s'\n", base_info->name); - } else { - SPDK_ERRLOG("Base bdev '%s' superblock io write has failed\n", base_info->name); - } + if (success) { + SPDK_DEBUGLOG(bdev_raid, "Write superblock to base bdev : '%s'\n", base_info->name); + } else { + SPDK_ERRLOG("Base bdev '%s' superblock io write has failed\n", base_info->name); + } - spdk_bdev_free_io(bdev_io); + spdk_bdev_free_io(bdev_io); } static int raid_bdev_base_bdev_write_sb(struct raid_base_bdev_info *base_info) { - int rc = 0; - struct spdk_io_channel *ch; + int rc = 0; + struct spdk_io_channel *ch; - ch = spdk_bdev_get_io_channel(base_info->desc); + ch = spdk_bdev_get_io_channel(base_info->desc); - if (!ch) { - SPDK_ERRLOG("Unable to create io channel for bdev '%s'\n", base_info->name); - return -ENOMEM; - } + if (!ch) { + SPDK_ERRLOG("Unable to create io channel for bdev '%s'\n", base_info->name); + return -ENOMEM; + } - rc = spdk_bdev_write_blocks(base_info->desc, ch, base_info->raid_sb, 0, - RAID_SB_BLOCKS(spdk_bdev_desc_get_bdev(base_info->desc)->blocklen), - (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_write_sb_compete, - base_info); + rc = spdk_bdev_write_blocks(base_info->desc, ch, base_info->raid_sb, 0, + RAID_SB_BLOCKS(spdk_bdev_desc_get_bdev(base_info->desc)->blocklen), + (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_write_sb_compete, + base_info); - spdk_put_io_channel(ch); - return rc; + spdk_put_io_channel(ch); + return rc; } static void raid_bdev_base_bdev_read_sb_compete(struct spdk_bdev_io *bdev_io, bool success, void *cb_arg) { - struct raid_base_bdev_info *base_info = cb_arg; + struct raid_base_bdev_info *base_info = cb_arg; - if (success) { - SPDK_DEBUGLOG(bdev_raid, "Read superblock from base bdev : '%s'\n", base_info->name); - } else { - SPDK_ERRLOG("base bdev '%s' superblock io read has failed\n", base_info->name); - } + if (success) { + SPDK_DEBUGLOG(bdev_raid, "Read superblock from base bdev : '%s'\n", base_info->name); + } else { + SPDK_ERRLOG("base bdev '%s' superblock io read has failed\n", base_info->name); + } - spdk_bdev_free_io(bdev_io); + spdk_bdev_free_io(bdev_io); } static int raid_bdev_base_bdev_read_sb(struct raid_base_bdev_info *base_info, uint32_t sb_size) { - int rc = 0; - struct spdk_io_channel *ch; - - ch = spdk_bdev_get_io_channel(base_info->desc); + int rc = 0; + struct spdk_io_channel *ch; - if (!ch) { - SPDK_ERRLOG("Unable to create io channel for bdev '%s'\n", base_info->name); - return -ENOMEM; - } + ch = spdk_bdev_get_io_channel(base_info->desc); - rc = spdk_bdev_read_blocks(base_info->desc, ch, base_info->raid_sb, 0, - sb_size, (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_read_sb_compete, - base_info); + if (!ch) { + SPDK_ERRLOG("Unable to create io channel for bdev '%s'\n", base_info->name); + return -ENOMEM; + } - spdk_put_io_channel(ch); - return rc; + rc = spdk_bdev_read_blocks(base_info->desc, ch, base_info->raid_sb, 0, + sb_size, (spdk_bdev_io_completion_cb) raid_bdev_base_bdev_read_sb_compete, + base_info); +// wait(NULL); + spdk_put_io_channel(ch); + return rc; } static int raid_bdev_base_bdev_super_load(struct raid_base_bdev_info *base_info, struct raid_base_bdev_info **freshest) { - int rc = 0; - struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); + int rc = 0; + struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); - base_info->raid_sb = spdk_dma_malloc(RAID_SB_BLOCKS(base_bdev->blocklen) * base_bdev->blocklen, 0, NULL); - if (!base_info->raid_sb) { - SPDK_ERRLOG("Failed to allocate memory for raid_sb\n"); - return -ENOMEM; - } + base_info->raid_sb = spdk_dma_malloc(RAID_SB_BLOCKS(base_bdev->blocklen) * base_bdev->blocklen, 0, NULL); + if (!base_info->raid_sb) { + SPDK_ERRLOG("Failed to allocate memory for raid_sb\n"); + return -ENOMEM; + } - rc = raid_bdev_base_bdev_read_sb(base_info, RAID_SB_BLOCKS(base_bdev->blocklen)); - if (rc) { - SPDK_ERRLOG("Failed while read superblock from base bdev '%s'\n", base_info->name); - return rc; - } + rc = raid_bdev_base_bdev_read_sb(base_info, RAID_SB_BLOCKS(base_bdev->blocklen)); + if (rc) { + SPDK_ERRLOG("Failed while read superblock from base bdev '%s'\n", base_info->name); + return rc; + } - struct raid_superblock *sb = base_info->raid_sb; + struct raid_superblock *sb = base_info->raid_sb; - if (sb->magic != RAID_SUPERBLOCK_MAGIC) - base_info->is_new = true; + if (sb->magic != RAID_SUPERBLOCK_MAGIC) + base_info->is_new = true; - if (base_info->is_new) { - *freshest = *freshest ? : base_info; - return 0; - } + if (base_info->is_new) { + *freshest = *freshest ? : base_info; + return 0; + } - if (!*freshest) { - *freshest = base_info; - return 0; - } + if (!*freshest) { + *freshest = base_info; + return 0; + } - if ((*freshest)->raid_sb->timestamp.tv_sec <= sb->timestamp.tv_sec && - (*freshest)->raid_sb->timestamp.tv_nsec < sb->timestamp.tv_nsec) - *freshest = base_info; + if ((*freshest)->raid_sb->timestamp.tv_sec <= sb->timestamp.tv_sec && + (*freshest)->raid_sb->timestamp.tv_nsec < sb->timestamp.tv_nsec) + *freshest = base_info; - return 0; + return 0; } static int raid_bdev_super_init_validation(struct raid_bdev *raid, struct raid_base_bdev_info *base_info) { - struct raid_superblock *sb = base_info->raid_sb; + struct raid_superblock *sb = base_info->raid_sb; - assert(raid->num_base_bdevs == raid->num_base_bdevs_discovered); + assert(raid->num_base_bdevs == raid->num_base_bdevs_discovered); - if (base_info->is_new) { - SPDK_ERRLOG("There isn't any metadata on base bdevs for raid '%s'\n", raid->bdev.name); - return -EINVAL; - } + if (base_info->is_new) { + SPDK_ERRLOG("There isn't any metadata on base bdevs for raid '%s'\n", raid->bdev.name); + return -EINVAL; + } - if (sb->version != RAID_METADATA_VERSION_01) { - SPDK_ERRLOG("Unsupported version of raid metadata has been found in base '%s' bdev's superblock\n", - base_info->name); - return -EINVAL; - } + if (sb->version != RAID_METADATA_VERSION_01) { + SPDK_ERRLOG("Unsupported version of raid metadata has been found in base '%s' bdev's superblock\n", + base_info->name); + return -EINVAL; + } - if (sb->num_base_bdevs != raid->num_base_bdevs) { - SPDK_ERRLOG("The '%s' bdev's number of devices isn't equal to RAID's\n", - base_info->name); - return -EINVAL; - } + if (sb->num_base_bdevs != raid->num_base_bdevs) { + SPDK_ERRLOG("The '%s' bdev's number of devices isn't equal to RAID's\n", + base_info->name); + return -EINVAL; + } - if (!raid_bdev_is_raid_level(sb->level)) { - SPDK_ERRLOG("Invalid RAID level in base '%s' bdev's metadata. Retrieving of array isn't possible\n", - base_info->name); - return -EINVAL; - } + if (!raid_bdev_is_raid_level(sb->level)) { + SPDK_ERRLOG("Invalid RAID level in base '%s' bdev's metadata. Retrieving of array isn't possible\n", + base_info->name); + return -EINVAL; + } - raid->num_base_bdevs = sb->num_base_bdevs; - raid->num_base_bdevs_discovered = sb->num_base_bdevs; - raid->level = sb->level; - raid->bdev.blocklen = sb->blocklen; - raid->bdev.blockcnt = sb->raid_blockcnt; - spdk_uuid_copy(&raid->bdev.uuid, &sb->uuid); - raid->strip_size = sb->strip_size; - raid->strip_size_kb = sb->strip_size * sb->blocklen; - raid->strip_size_shift = spdk_u32log2(sb->strip_size); - raid->blocklen_shift = spdk_u32log2(sb->blocklen); + raid->num_base_bdevs = sb->num_base_bdevs; + raid->num_base_bdevs_discovered = sb->num_base_bdevs; + raid->level = sb->level; + raid->bdev.blocklen = sb->blocklen; + raid->bdev.blockcnt = sb->raid_blockcnt; + spdk_uuid_copy(&raid->bdev.uuid, &sb->uuid); + raid->strip_size = sb->strip_size; + raid->strip_size_kb = sb->strip_size * sb->blocklen; + raid->strip_size_shift = spdk_u32log2(sb->strip_size); + raid->blocklen_shift = spdk_u32log2(sb->blocklen); - raid->is_new = false; + raid->is_new = false; - return 0; + return 0; } static int raid_bdev_base_bdev_super_sync(struct raid_base_bdev_info *base_info) { - struct raid_superblock *sb = base_info->raid_sb; - struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); - struct raid_bdev *raid = base_info->raid_bdev; - - sb->magic = RAID_SUPERBLOCK_MAGIC; - sb->version = RAID_METADATA_VERSION_01; - sb->blocklen = raid->bdev.blocklen; - sb->num_base_bdevs = raid->num_base_bdevs; - sb->level = raid->level; - sb->array_position = base_info->position; - sb->strip_size = raid->strip_size; - sb->raid_blockcnt = raid->bdev.blockcnt; - spdk_uuid_copy(&sb->uuid, &raid->bdev.uuid); + struct raid_superblock *sb = base_info->raid_sb; + struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); + struct raid_bdev *raid = base_info->raid_bdev; + + sb->magic = RAID_SUPERBLOCK_MAGIC; + sb->version = RAID_METADATA_VERSION_01; + sb->blocklen = raid->bdev.blocklen; + sb->num_base_bdevs = raid->num_base_bdevs; + sb->level = raid->level; + sb->array_position = base_info->position; + sb->strip_size = raid->strip_size; + sb->raid_blockcnt = raid->bdev.blockcnt; + spdk_uuid_copy(&sb->uuid, &raid->bdev.uuid); + + memset(sb+1, 0, RAID_SB_BLOCKS(base_bdev->blocklen)); - memset(sb+1, 0, RAID_SB_BLOCKS(base_bdev->blocklen)); - - return 0; + return 0; } static int raid_bdev_base_bdev_capture_super_validate(struct raid_base_bdev_info *base_info) { - struct raid_superblock *sb = base_info->raid_sb; - struct raid_bdev *raid = base_info->raid_bdev; - - /* The absence of a rebuild does not allow to capture not SPDK RAID disks or disks from the another RAID now. - * Redo when the rebuild will be done. - */ - if (base_info->is_new) { - SPDK_ERRLOG("The bdev '%s' haven't been in a SPDK RAID\n", - base_info->name); - return -EINVAL; - } else if (spdk_uuid_compare(&sb->uuid, &raid->bdev.uuid)) { - SPDK_ERRLOG("The bdev '%s' have been in another RAID bdev\n", - base_info->name); - return -EINVAL; - } - - if (sb->array_position != base_info->position) { - SPDK_WARNLOG("The base bdev '%s' has changed position in the raid from '%n' to '%n'\n", base_info->name, - &sb->array_position, &base_info->position); - } - - if (sb->blocklen != raid->bdev.blocklen) { - SPDK_ERRLOG("The logical block length stored in metadata of base '%s' bdev differ from RAID block length\n", - base_info->name); - return -EINVAL; - } - - if (sb->strip_size != raid->strip_size) { - SPDK_WARNLOG("The strip size stored in metadata of base '%s' bdev differ from RAID strip size and" - "it will be overwritten\n", base_info->name); - } - - return 0; + struct raid_superblock *sb = base_info->raid_sb; + struct raid_bdev *raid = base_info->raid_bdev; + + /* The absence of a rebuild does not allow to capture not SPDK RAID disks or disks from the another RAID now. + * Redo when the rebuild will be done. + */ + if (base_info->is_new) { + SPDK_ERRLOG("The bdev '%s' haven't been in a SPDK RAID\n", + base_info->name); + return -EINVAL; + } else if (spdk_uuid_compare(&sb->uuid, &raid->bdev.uuid)) { + SPDK_ERRLOG("The bdev '%s' have been in another RAID bdev\n", + base_info->name); + return -EINVAL; + } + + if (sb->array_position != base_info->position) { + SPDK_WARNLOG("The base bdev '%s' has changed position in the raid from '%n' to '%n'\n", base_info->name, + &sb->array_position, &base_info->position); + } + + if (sb->blocklen != raid->bdev.blocklen) { + SPDK_ERRLOG("The logical block length stored in metadata of base '%s' bdev differ from RAID block length\n", + base_info->name); + return -EINVAL; + } + + if (sb->strip_size != raid->strip_size) { + SPDK_WARNLOG("The strip size stored in metadata of base '%s' bdev differ from RAID strip size and" + "it will be overwritten\n", base_info->name); + } + + return 0; } static int raid_bdev_base_bdev_super_init(struct raid_base_bdev_info *base_info) { - struct raid_superblock *sb = base_info->raid_sb; - struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); - struct timespec time_0 = {.tv_nsec = 0, .tv_sec = 0}; + struct raid_superblock *sb = base_info->raid_sb; + struct spdk_bdev *base_bdev = spdk_bdev_desc_get_bdev(base_info->desc); + struct timespec time_0 = {.tv_nsec = 0, .tv_sec = 0}; - sb->blockcnt = base_bdev->blockcnt; - sb->timestamp = time_0; + sb->blockcnt = base_bdev->blockcnt; + sb->timestamp = time_0; - return 0; + return 0; } static int raid_bdev_base_bdev_super_validate(struct raid_base_bdev_info *base_info, bool recreate) { - int rc = 0; - struct raid_superblock *sb = base_info->raid_sb; - struct raid_bdev *raid = base_info->raid_bdev; + int rc = 0; + struct raid_superblock *sb = base_info->raid_sb; + struct raid_bdev *raid = base_info->raid_bdev; - if (raid->num_base_bdevs <= 0 || !sb) - return 0; + if (raid->num_base_bdevs <= 0 || !sb) + return 0; - if (!recreate && raid->is_new && raid_bdev_super_init_validation(raid, base_info)) - return -EINVAL; + if (!recreate && raid->is_new && raid_bdev_super_init_validation(raid, base_info)) + return -EINVAL; - if (!raid->is_new || recreate) { - if (!recreate && raid_bdev_base_bdev_capture_super_validate(base_info)) - return -EINVAL; - - raid_bdev_base_bdev_super_sync(base_info); + if (!raid->is_new || recreate) { + if (!recreate && raid_bdev_base_bdev_capture_super_validate(base_info)) + return -EINVAL; - if (recreate) - raid_bdev_base_bdev_super_init(base_info); - } + raid_bdev_base_bdev_super_sync(base_info); - return rc; + if (recreate) + raid_bdev_base_bdev_super_init(base_info); + } + + return rc; } /* @@ -1412,8 +1412,8 @@ raid_bdev_configure(struct raid_bdev *raid_bdev) struct spdk_bdev *raid_bdev_gen; struct raid_base_bdev_info *base_info; struct spdk_bdev *base_bdev; - bool sb_recreate = raid_bdev->is_new; - struct raid_base_bdev_info *freshest = NULL; + bool sb_recreate = raid_bdev->is_new; + struct raid_base_bdev_info *freshest = NULL; int rc = 0; assert(raid_bdev->state == RAID_BDEV_STATE_CONFIGURING); @@ -1456,34 +1456,34 @@ raid_bdev_configure(struct raid_bdev *raid_bdev) return rc; } - if (raid_bdev->superblock_enabled) { - RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { - rc = raid_bdev_base_bdev_super_load(base_info, &freshest); - if (rc) { - SPDK_DEBUGLOG(bdev_raid, "super_load for base bdev has failed'%s'\n", base_info->name); - return rc; - } - } + if (raid_bdev->superblock_enabled) { + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + rc = raid_bdev_base_bdev_super_load(base_info, &freshest); + if (rc) { + SPDK_DEBUGLOG(bdev_raid, "super_load for base bdev has failed'%s'\n", base_info->name); + return rc; + } + } - SPDK_DEBUGLOG(bdev_raid, "The freshest bdev is '%s'\n", freshest->name); + SPDK_DEBUGLOG(bdev_raid, "The freshest bdev is '%s'\n", freshest->name); - if (!freshest) { - SPDK_ERRLOG("There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); - return -EINVAL; - } + if (!freshest) { + SPDK_ERRLOG("There aren't base bdevs in raid bdev '%s'\n", raid_bdev->bdev.name); + return -EINVAL; + } - raid_bdev->is_new = true; + raid_bdev->is_new = true; - rc = raid_bdev_base_bdev_super_validate(freshest, sb_recreate); - if (rc) { - SPDK_DEBUGLOG(bdev_raid, "super_validate for freshest '%s' bdev has failed\n", base_info->name); - return rc; - } + rc = raid_bdev_base_bdev_super_validate(freshest, sb_recreate); + if (rc) { + SPDK_DEBUGLOG(bdev_raid, "super_validate for freshest '%s' bdev has failed\n", base_info->name); + return rc; + } - if (!sb_recreate && raid_bdev_module_init(raid_bdev)) { - return -EINVAL; - } - } + if (!sb_recreate && raid_bdev_module_init(raid_bdev)) { + return -EINVAL; + } + } rc = raid_bdev->module->start(raid_bdev); if (rc != 0) { @@ -1511,22 +1511,22 @@ raid_bdev_configure(struct raid_bdev *raid_bdev) SPDK_DEBUGLOG(bdev_raid, "raid bdev is created with name %s, raid_bdev %p\n", raid_bdev_gen->name, raid_bdev); - if (raid_bdev->superblock_enabled) { - RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { - raid_bdev_base_bdev_super_validate(base_info, sb_recreate); - if (rc) { - SPDK_DEBUGLOG(bdev_raid, "super_validate for '%s' bdev has failed\n", base_info->name); - return rc; - } - } - - RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { - rc = raid_bdev_base_bdev_write_sb(base_info); - if (rc) { - SPDK_DEBUGLOG(bdev_raid, "Write superblock to '%s' bdev has failed\n", base_info->name); - } - } - } + if (raid_bdev->superblock_enabled) { + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + raid_bdev_base_bdev_super_validate(base_info, sb_recreate); + if (rc) { + SPDK_DEBUGLOG(bdev_raid, "super_validate for '%s' bdev has failed\n", base_info->name); + return rc; + } + } + + RAID_FOR_EACH_BASE_BDEV(raid_bdev, base_info) { + rc = raid_bdev_base_bdev_write_sb(base_info); + if (rc) { + SPDK_DEBUGLOG(bdev_raid, "Write superblock to '%s' bdev has failed\n", base_info->name); + } + } + } return 0; } @@ -1891,7 +1891,7 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info) if (raid_bdev->superblock_enabled) { assert((RAID_BDEV_MIN_DATA_OFFSET_SIZE % bdev->blocklen) == 0); base_info->data_offset = spdk_max(RAID_BDEV_MIN_DATA_OFFSET_SIZE/bdev->blocklen, - RAID_SB_BLOCKS(bdev->blocklen)); + RAID_SB_BLOCKS(bdev->blocklen)); if (bdev->optimal_io_boundary) { base_info->data_offset = spdk_divide_round_up(base_info->data_offset, @@ -1900,7 +1900,7 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info) if (base_info->data_offset >= bdev->blockcnt) { SPDK_ERRLOG("Data offset %lu exceeds base bdev capacity %lu on bdev '%s'\n", - base_info->data_offset, bdev->blockcnt, base_info->name); + base_info->data_offset, bdev->blockcnt, base_info->name); return -EINVAL; } diff --git a/module/bdev/raid/bdev_raid.h b/module/bdev/raid/bdev_raid.h index c7b8e881f30..f905db03f02 100644 --- a/module/bdev/raid/bdev_raid.h +++ b/module/bdev/raid/bdev_raid.h @@ -50,7 +50,7 @@ enum raid_bdev_state { typedef void (*raid_bdev_remove_base_bdev_cb)(void *ctx, int status); enum metadata_version { - RAID_METADATA_VERSION_01 = 01 + RAID_METADATA_VERSION_01 = 01 }; #pragma pack(push, 1) @@ -59,38 +59,38 @@ enum metadata_version { * It stores some metadata of the base bdev and the raid */ struct raid_superblock { - /* SPDK raid magic number "SKRD" */ - uint32_t magic; + /* SPDK raid magic number "SKRD" */ + uint32_t magic; - /* The version of metadata. Currently, only 01 version exists */ - uint32_t version; + /* The version of metadata. Currently, only 01 version exists */ + uint32_t version; - /* Logical block size of base bdev */ - uint32_t blocklen; + /* Logical block size of base bdev */ + uint32_t blocklen; - /* Number of base bdevs */ - uint8_t num_base_bdevs; + /* Number of base bdevs */ + uint8_t num_base_bdevs; - /* Raid Level of device's raid */ - int32_t level; + /* Raid Level of device's raid */ + int32_t level; - /* Position of device in raid */ - uint32_t array_position; + /* Position of device in raid */ + uint32_t array_position; - /* strip size of device's raid in blocks */ - uint32_t strip_size; + /* strip size of device's raid in blocks */ + uint32_t strip_size; - /* Number of blocks held by this base bdev */ - uint64_t blockcnt; + /* Number of blocks held by this base bdev */ + uint64_t blockcnt; - /* Number of blocks held by device's raid */ - uint64_t raid_blockcnt; + /* Number of blocks held by device's raid */ + uint64_t raid_blockcnt; - /* Timestamp to know the freshest device in raid */ - struct timespec timestamp; + /* Timestamp to know the freshest device in raid */ + struct timespec timestamp; - /* UUID of raid bdev */ - struct spdk_uuid uuid; + /* UUID of raid bdev */ + struct spdk_uuid uuid; }; #pragma pack(pop) @@ -109,14 +109,14 @@ struct raid_base_bdev_info { /* pointer to base bdev descriptor opened by raid bdev */ struct spdk_bdev_desc *desc; - /* position of the base bdev in raid_bdev's base_bdev_info (slot) */ - uint32_t position; + /* position of the base bdev in raid_bdev's base_bdev_info (slot) */ + uint32_t position; - /* indicate if the metadata was on disk or not */ - bool is_new; + /* indicate if the metadata was on disk or not */ + bool is_new; - /* base bdev's superblock */ - struct raid_superblock *raid_sb; + /* base bdev's superblock */ + struct raid_superblock *raid_sb; /* offset in blocks from the start of the base bdev to the start of the data region */ uint64_t data_offset; @@ -197,8 +197,8 @@ struct raid_bdev { /* state of raid bdev */ enum raid_bdev_state state; - /* indicate if raid is new for */ - bool is_new; + /* indicate if raid is new for */ + bool is_new; /* number of base bdevs comprising raid bdev */ uint8_t num_base_bdevs; diff --git a/module/bdev/raid/bdev_raid_rpc.c b/module/bdev/raid/bdev_raid_rpc.c index 67a65a9d93d..2421c7fe5f9 100644 --- a/module/bdev/raid/bdev_raid_rpc.c +++ b/module/bdev/raid/bdev_raid_rpc.c @@ -134,8 +134,8 @@ struct rpc_bdev_raid_create { /* If set, information about raid bdev will be stored in superblock on each base bdev */ bool superblock_enabled; - /* If set, it tries to retrieve raid array from the base bdevs and fail if it can't */ - bool retrieve; + /* If set, it tries to retrieve raid array from the base bdevs and fail if it can't */ + bool retrieve; }; /* @@ -202,7 +202,7 @@ static const struct spdk_json_object_decoder rpc_bdev_raid_create_decoders[] = { {"base_bdevs", offsetof(struct rpc_bdev_raid_create, base_bdevs), decode_base_bdevs}, {"uuid", offsetof(struct rpc_bdev_raid_create, uuid), spdk_json_decode_uuid, true}, {"superblock", offsetof(struct rpc_bdev_raid_create, superblock_enabled), spdk_json_decode_bool, true}, - {"retrieve", offsetof(struct rpc_bdev_raid_create, retrieve), spdk_json_decode_bool, true} + {"retrieve", offsetof(struct rpc_bdev_raid_create, retrieve), spdk_json_decode_bool, true} }; /* @@ -232,12 +232,12 @@ rpc_bdev_raid_create(struct spdk_jsonrpc_request *request, goto cleanup; } - if (!req.superblock_enabled && req.retrieve) { - spdk_jsonrpc_send_error_response_fmt(request, -EINVAL, - "Retrieve option can be only if superblock is specified: %s", - spdk_strerror(-EINVAL)); - goto cleanup; - } + if (!req.superblock_enabled && req.retrieve) { + spdk_jsonrpc_send_error_response_fmt(request, -EINVAL, + "Retrieve option can be only if superblock is specified: %s", + spdk_strerror(-EINVAL)); + goto cleanup; + } rc = raid_bdev_create(req.name, req.strip_size_kb, req.base_bdevs.num_base_bdevs, req.level, req.superblock_enabled, &req.uuid, &raid_bdev); @@ -248,9 +248,9 @@ rpc_bdev_raid_create(struct spdk_jsonrpc_request *request, goto cleanup; } - if (req.retrieve) { - raid_bdev->is_new = false; - } + if (req.retrieve) { + raid_bdev->is_new = false; + } for (i = 0; i < req.base_bdevs.num_base_bdevs; i++) { const char *base_bdev_name = req.base_bdevs.base_bdevs[i];