Rebuild process

include/spdk/util.h

@@ -42,6 +42,14 @@ extern "C" {
 /* Ceiling division of unsigned integers */
 #define SPDK_CEIL_DIV(x,y) (((x)+(y)-1)/(y))
 
+#define SPDK_TEST_BIT(number_ptr, shift_size) (*(number_ptr) & (1UL << shift_size))
+
+#define SPDK_SET_BIT(number_ptr, shift_size) (*(number_ptr) |= 1UL << shift_size)
+
+#define SPDK_REMOVE_BIT(number_ptr, shift_size) (*(number_ptr) &= ~(1UL << shift_size))
+
+#define SPDK_KB_TO_B(number) (number << 10)
+
 /**
  * Macro to align a value to a given power-of-two. The resultant value
  * will be of the same type as the first parameter, and will be no

module/bdev/raid/Makefile

@@ -6,11 +6,14 @@
 SPDK_ROOT_DIR := $(abspath $(CURDIR)/../../..)
 include $(SPDK_ROOT_DIR)/mk/spdk.common.mk
 
-SO_VER := 5
+SO_VER := 6
 SO_MINOR := 0
-
-CFLAGS += -I$(SPDK_ROOT_DIR)/lib/bdev/
-C_SRCS = bdev_raid.c bdev_raid_rpc.c raid0.c raid1.c concat.c
+# ->
+# CFLAGS += $(ENV_CFLAGS)
+# <-
+CFLAGS += -I$(SPDK_ROOT_DIR)/lib/bdev/ 
+# -I$(SPDK_ROOT_DIR)/lib/env_dpdk/
+C_SRCS = service.c bdev_raid.c bdev_raid_rpc.c raid0.c raid1.c concat.c
 
 ifeq ($(CONFIG_RAID5F),y)
 C_SRCS += raid5f.c
@@ -21,3 +24,24 @@ LIBNAME = bdev_raid
 SPDK_MAP_FILE = $(SPDK_ROOT_DIR)/mk/spdk_blank.map
 
 include $(SPDK_ROOT_DIR)/mk/spdk.lib.mk
+
+# ->
+# LIBDPDK_PKGCONFIG = $(call pkgconfig_filename,spdk_dpdklibs)
+
+# $(LIBDPDK_PKGCONFIG): $(PKGCONFIG) $(PKGCONFIG_INST)
+# 	$(Q)$(SPDK_ROOT_DIR)/scripts/pc_libs.sh \
+# 		"-L$(DPDK_LIB_DIR) $(DPDK_LIB_LIST:%=-l%)" "" DPDK spdk_dpdklibs > $@
+# 	$(Q)sed -i.bak '5s,.*,Requires: $(DEPDIRS-$(LIBNAME):%=spdk_%) spdk_dpdklibs,' $(PKGCONFIG) ; rm $(PKGCONFIG).bak
+# 	$(Q)sed -i.bak '5s,.*,Requires: $(DEPDIRS-$(LIBNAME):%=spdk_%) spdk_dpdklibs,' $(PKGCONFIG_INST) ; rm $(PKGCONFIG_INST).bak
+
+# _install_dpdklibs: $(LIBDPDK_PKGCONFIG)
+# 	@$(call pkgconfig_install,$(LIBDPDK_PKGCONFIG))
+
+# _uninstall_dpdklibs: $(LIBDPDK_PKGCONFIG)
+# 	@$(call pkgconfig_uninstall,$(LIBDPDK_PKGCONFIG))
+
+# all: $(LIBDPDK_PKGCONFIG)
+# install: _install_dpdklibs
+# uninstall: _uninstall_dpdklibs
+# <-
+

module/bdev/raid/atomic_raid.h

@@ -0,0 +1,117 @@
+/*   SPDX-License-Identifier: BSD-3-Clause
+ *   Copyright (C) 2018 Intel Corporation.
+ *   All rights reserved.
+ */
+
+#ifndef SPDK_ATOMIC_RAID_INTERNAL_H
+#define SPDK_ATOMIC_RAID_INTERNAL_H
+
+#include "spdk/util.h"
+
+//typedef int raid_atomic; //реализовать можно позже, но пока не вижу смысла
+
+typedef uint64_t raid_atomic64;
+
+#define atomic_read(ptr)       (*(__typeof__(*ptr) *volatile) (ptr))
+#define atomic_set(ptr, i)     ((*(__typeof__(*ptr) *volatile) (ptr)) = (i))
+#define atomic_inc(ptr)        ((void) __sync_fetch_and_add(ptr, 1))
+#define atomic_dec(ptr)        ((void) __sync_fetch_and_add(ptr, -1))
+#define atomic_add(ptr, n)     ((void) __sync_fetch_and_add(ptr, n))
+#define atomic_sub(ptr, n)     ((void) __sync_fetch_and_sub(ptr, n))
+
+#define atomic_cmpxchg         __sync_bool_compare_and_swap
+
+
+static inline uint64_t
+raid_atomic64_read(const raid_atomic64 *a)
+{
+	return atomic_read(a);
+}
+
+static inline void
+raid_atomic64_set(raid_atomic64 *a, uint64_t i)
+{
+	atomic_set(a, i);
+}
+
+static inline void
+raid_atomic64_add(uint64_t i, raid_atomic64 *a)
+{
+	atomic_add(a, i);
+}
+
+static inline void
+raid_atomic64_sub(uint64_t i, raid_atomic64 *a)
+{
+	atomic_sub(a, i);
+}
+
+static inline void
+raid_atomic64_inc(raid_atomic64 *a)
+{
+	atomic_inc(a);
+}
+
+static inline void
+raid_atomic64_dec(raid_atomic64 *a)
+{
+	atomic_dec(a);
+}
+
+static inline uint64_t
+raid_atomic64_add_return(uint64_t i, raid_atomic64 *a)
+{
+	return __sync_add_and_fetch(a, i);
+}
+
+static inline uint64_t
+raid_atomic64_sub_return(uint64_t i, raid_atomic64 *a)
+{
+	return __sync_sub_and_fetch(a, i);
+}
+
+static inline uint64_t
+raid_atomic64_inc_return(raid_atomic64 *a)
+{
+	return raid_atomic64_add_return(1, a);
+}
+
+static inline uint64_t
+raid_atomic64_dec_return(raid_atomic64 *a)
+{
+	return raid_atomic64_sub_return(1, a);
+}
+
+static inline uint64_t
+raid_atomic64_cmpxchg(raid_atomic64 *a, uint64_t old_val, uint64_t new_val)
+{
+	return atomic_cmpxchg(a, old_val, new_val);
+}
+
+static inline void 
+raid_atomic64_set_bit(raid_atomic64 *atomic_ptr, uint64_t shift_size)
+{
+	uint64_t old_val;
+	uint64_t new_val;
+	do
+	{
+		old_val = raid_atomic64_read(atomic_ptr);
+		new_val = old_val;
+		SPDK_SET_BIT(&new_val, shift_size);
+	} while (raid_atomic64_cmpxchg(atomic_ptr, old_val, new_val));
+}
+
+static inline void 
+raid_atomic64_remove_bit(raid_atomic64 *atomic_ptr, uint64_t shift_size)
+{
+	uint64_t old_val;
+	uint64_t new_val;
+	do
+	{
+		old_val = raid_atomic64_read(atomic_ptr);
+		new_val = old_val;
+		SPDK_REMOVE_BIT(&new_val, shift_size);
+	} while (raid_atomic64_cmpxchg(atomic_ptr, old_val, new_val));
+}
+
+#endif /* SPDK_ATOMIC_RAID_INTERNAL_H */

module/bdev/raid/bdev_raid.c

@@ -5,6 +5,7 @@
  */
 
 #include "bdev_raid.h"
+#include "service.h"
 #include "spdk/env.h"
 #include "spdk/thread.h"
 #include "spdk/log.h"
@@ -185,6 +186,7 @@ raid_bdev_cleanup(struct raid_bdev *raid_bdev)
 
 	TAILQ_REMOVE(&g_raid_bdev_list, raid_bdev, global_link);
 	free(raid_bdev->base_bdev_info);
+	spdk_poller_unregister(&(raid_bdev->rebuild_poller));
 }
 
 static void
@@ -951,12 +953,7 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
 		return -EEXIST;
 	}
 
-	if (level == RAID1) {
-		if (strip_size != 0) {
-			SPDK_ERRLOG("Strip size is not supported by raid1\n");
-			return -EINVAL;
-		}
-	} else if (spdk_u32_is_pow2(strip_size) == false) {
+	if (spdk_u32_is_pow2(strip_size) == false) {
 		SPDK_ERRLOG("Invalid strip size %" PRIu32 "\n", strip_size);
 		return -EINVAL;
 	}
@@ -1009,6 +1006,20 @@ raid_bdev_create(const char *name, uint32_t strip_size, uint8_t num_base_bdevs,
 		return -ENOMEM;
 	}
 
+	/* allocate rebuild struct  */
+	switch(level) {
+		case RAID1:
+			raid_bdev->rebuild = calloc(1, sizeof(struct raid_rebuild));
+			if (!raid_bdev->rebuild) {
+				SPDK_ERRLOG("Unable to allocate memory for raid rebuild struct\n");
+				return -ENOMEM;
+			}
+			raid_bdev->rebuild_poller = SPDK_POLLER_REGISTER(run_rebuild_poller, raid_bdev, 200000);
+			break;
+		default:
+			raid_bdev->rebuild = NULL;
+			raid_bdev->rebuild_poller = NULL;
+	}
 	raid_bdev->module = module;
 	raid_bdev->num_base_bdevs = num_base_bdevs;
 	raid_bdev->base_bdev_info = calloc(raid_bdev->num_base_bdevs,
@@ -1539,19 +1550,28 @@ raid_bdev_configure_base_bdev(struct raid_base_bdev_info *base_info)
 
 	SPDK_DEBUGLOG(bdev_raid, "bdev %s is claimed\n", bdev->name);
 
-	assert(raid_bdev->state != RAID_BDEV_STATE_ONLINE);
-
 	base_info->desc = desc;
-	base_info->blockcnt = bdev->blockcnt;
 	raid_bdev->num_base_bdevs_discovered++;
 	assert(raid_bdev->num_base_bdevs_discovered <= raid_bdev->num_base_bdevs);
 
-	if (raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs) {
-		rc = raid_bdev_configure(raid_bdev);
-		if (rc != 0) {
-			SPDK_ERRLOG("Failed to configure raid bdev\n");
-			return rc;
-		}
+	switch (raid_bdev->state) {
+		case RAID_BDEV_STATE_ONLINE:
+			bdev->blockcnt = base_info->blockcnt;
+			break;
+		case RAID_BDEV_STATE_CONFIGURING:
+			base_info->blockcnt = bdev->blockcnt;
+			if (raid_bdev->num_base_bdevs_discovered == raid_bdev->num_base_bdevs) {
+				rc = raid_bdev_configure(raid_bdev);
+				if (rc != 0) {
+					SPDK_ERRLOG("Failed to configure raid bdev\n");
+					return rc;
+				}
+			}
+			break;
+		case RAID_BDEV_STATE_OFFLINE:
+			/* TODO when OFFLINE state is completely implemented */
+		default:
+			SPDK_ERRLOG("unexpected bdev raid state when adding '%s' base bdev", base_info->name);
 	}
 
 	return 0;
@@ -1604,6 +1624,42 @@ raid_bdev_add_base_device(struct raid_bdev *raid_bdev, const char *name, uint8_t
 	return 0;
 }
 
+static int
+fill_matrix(void) {
+	SPDK_DEBUGLOG("Fill matrix's stub\n");
+	return 0;
+};
+
+int
+raid_bdev_add_base_bdev(struct raid_bdev *raid_bdev, char *base_bdev_name, uint8_t slot) {
+	int rc;
+	struct spdk_bdev *bdev = spdk_bdev_get_by_name(base_bdev_name);
+
+	if (bdev == NULL) {
+		SPDK_ERRLOG("Currently unable to find bdev with name: %s\n", base_bdev_name);
+		return -ENXIO;
+	}
+
+	if (bdev->blocklen != raid_bdev->bdev.blocklen) {
+		SPDK_ERRLOG("Blocklen of the bdev %s not matching with other base bdevs\n", base_bdev_name);
+		return -EINVAL;
+	}
+
+	if (bdev->blockcnt < raid_bdev->bdev.blockcnt) {
+		SPDK_ERRLOG("The bdev %s size is too small\n", base_bdev_name);
+		return -EINVAL;
+	}
+
+	rc = raid_bdev_add_base_device(raid_bdev, base_bdev_name, slot);
+	if (rc)
+		return rc;
+
+	rc = fill_matrix();
+	if (rc)
+		SPDK_ERRLOG("Failed to copy data to adding base bdev\n");
+	return rc;
+};
+
 /*
  * brief:
  * raid_bdev_examine function is the examine function call by the below layers