diff --git a/src/backend.h b/src/backend.h index 75f2feb1..5a6f65d1 100644 --- a/src/backend.h +++ b/src/backend.h @@ -51,5 +51,5 @@ class backend virtual bool exists(std::string name) = 0; }; -extern std::shared_ptr make_file_backend(const char *prefix); extern std::shared_ptr make_rados_backend(rados_ioctx_t io); +rados_ioctx_t connect_to_pool(str pool_name); diff --git a/src/rados_backend.cc b/src/rados_backend.cc index b63e2904..74ab05a0 100644 --- a/src/rados_backend.cc +++ b/src/rados_backend.cc @@ -205,3 +205,22 @@ std::shared_ptr make_rados_backend(rados_ioctx_t io) { return std::make_shared(io); } + +rados_ioctx_t connect_to_pool(str pool_name) +{ + rados_t cluster; + int err = rados_create2(&cluster, "ceph", "client.admin", 0); + check_ret_neg(err, "Failed to create cluster handle"); + + err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf"); + check_ret_neg(err, "Failed to read config file"); + + err = rados_connect(cluster); + check_ret_neg(err, "Failed to connect to cluster"); + + rados_ioctx_t io_ctx; + err = rados_ioctx_create(cluster, pool_name.c_str(), &io_ctx); + check_ret_neg(err, "Failed to connect to pool {}", pool_name); + + return io_ctx; +} diff --git a/src/spdk_frontend.cc b/src/spdk_frontend.cc index 111d35de..7ccd95a2 100644 --- a/src/spdk_frontend.cc +++ b/src/spdk_frontend.cc @@ -1,101 +1,176 @@ #include "spdk/event.h" +#include "spdk/nvme.h" #include "spdk/nvmf.h" +#include #include #include #include +#include "backend.h" #include "bdev_lsvd.h" #include "spdk/nvmf_spec.h" #include "utils.h" +const char *NVME_SS_NQN = "nqn.2019-05.io.lsvd:cnode1"; +const char *HOSTNAME = "127.0.0.1"; +const char *PORT = "4420"; + +using IntCallbackFn = std::function; +IntCallbackFn *alloc_cb(std::function cb) +{ + return new IntCallbackFn(cb); +} + +void invoke_and_free_cb(void *ctx, int status) +{ + auto cb = static_cast *>(ctx); + (*cb)(status); + delete cb; +} + struct start_lsvd_args { const char *pool_name; const char *image_name; }; -static void start_lsvd(void *arg) +spdk_nvmf_tgt *create_target() { - log_info("Starting LSVD SPDK program ..."); - auto args = (start_lsvd_args *)arg; - - rados_t cluster; - int err = rados_create2(&cluster, "ceph", "client.admin", 0); - check_ret_neg(err, "Failed to create cluster handle"); - - err = rados_conf_read_file(cluster, "/etc/ceph/ceph.conf"); - check_ret_neg(err, "Failed to read config file"); - - err = rados_connect(cluster); - check_ret_neg(err, "Failed to connect to cluster"); - - rados_ioctx_t io_ctx; - err = rados_ioctx_create(cluster, args->pool_name, &io_ctx); - check_ret_neg(err, "Failed to connect to pool {}", args->pool_name); - - lsvd_config cfg; // TODO get this from somewhere reasonable - cfg.cache_size = 160 * 1024 * 1024; - err = bdev_lsvd_create(args->image_name, io_ctx, cfg); - if (err) { - log_error("Failed to create bdev"); - spdk_app_stop(err); - } - - // TODO setup nvmf subsystems and all that nonsense - // we can worry about refactoring it into functions later - - // Step 1: create nvmf target log_info("Creating NVMF target"); - auto nvmf_opts = (spdk_nvmf_target_opts){ + spdk_nvmf_target_opts opts = { .name = "lsvd_nvmf_tgt", }; - auto tgt = spdk_nvmf_tgt_create(&nvmf_opts); + auto tgt = spdk_nvmf_tgt_create(&opts); assert(tgt != nullptr); + return tgt; +} - // Step 1.5: add discovery subsystem so we can probe for it +spdk_nvmf_subsystem *add_discovery_ss(spdk_nvmf_tgt *tgt) +{ log_info("Creating NVMF discovery subsystem"); - auto disc_ss = spdk_nvmf_subsystem_create( + auto ss = spdk_nvmf_subsystem_create( tgt, SPDK_NVMF_DISCOVERY_NQN, SPDK_NVMF_SUBTYPE_DISCOVERY_CURRENT, 0); - assert(disc_ss != nullptr); - spdk_nvmf_subsystem_set_allow_any_host(disc_ss, true); + assert(ss != nullptr); + spdk_nvmf_subsystem_set_allow_any_host(ss, true); + return ss; +} - // Step 2: create TCP transport +spdk_nvmf_subsystem *add_nvme_ss(spdk_nvmf_tgt *tgt) +{ + log_info("Creating SPDK controller subsystem"); + auto ss = + spdk_nvmf_subsystem_create(tgt, NVME_SS_NQN, SPDK_NVMF_SUBTYPE_NVME, 1); + assert(ss != nullptr); + spdk_nvmf_subsystem_set_allow_any_host(ss, true); + spdk_nvmf_subsystem_set_sn(ss, "SPDK_000001"); + spdk_nvmf_subsystem_set_mn(ss, "LSVD NVMe controller"); + spdk_nvmf_subsystem_set_ana_reporting(ss, true); + return ss; +} + +using TranspCb = std::function; +void create_tcp_transport(TranspCb *cb) +{ + log_info("Creating TCP transport"); spdk_nvmf_transport_opts opts; auto succ = spdk_nvmf_transport_opts_init("TCP", &opts, sizeof(opts)); assert(succ == true); - // opts.io_unit_size = 131072; - // opts.max_qpairs_per_ctrlr = 8; + opts.io_unit_size = 131072; + opts.max_qpairs_per_ctrlr = 8; opts.in_capsule_data_size = 8192; debug("TCP transport opts: io_unit_size={}, max_qpairs_per_ctrlr={}, " "in_capsule_data_size={}", opts.io_unit_size, opts.max_qpairs_per_ctrlr, opts.in_capsule_data_size); - log_info("Creating TCP transport"); - auto transport_p = std::promise(); - spdk_nvmf_transport_create_async( + auto rc = spdk_nvmf_transport_create_async( "TCP", &opts, - [](auto p, auto b) { - auto pr = (std::promise *)p; - pr->set_value(b); - }, - &transport_p); - auto transport = transport_p.get_future().get(); - assert(transport != nullptr); - - log_info("Adding TCP transport to target"); - auto stat_p = std::promise(); - spdk_nvmf_tgt_add_transport( - tgt, transport, - [](auto p, auto stat) { - auto pr = (std::promise *)p; - pr->set_value(stat); + [](auto ctx, auto r) { + auto cb = static_cast(ctx); + (*cb)(r); + delete cb; }, - nullptr); - auto status = stat_p.get_future().get(); - assert(status == 0); + cb); + assert(rc == 0); +} + +void add_transport(spdk_nvmf_tgt *tgt, spdk_nvmf_transport *tr, + std::function *cb) +{ + log_info("Adding transport to target"); + spdk_nvmf_tgt_add_transport(tgt, tr, invoke_and_free_cb, cb); +} + +void add_ss_listener(spdk_nvmf_tgt *tgt, spdk_nvmf_subsystem *ss, str host, + str port, std::function *cb) +{ + log_info("Adding listener to subsystem"); + + spdk_nvme_transport_id trid; + // They're fixed-size char[] bufs in the struct, so make sure we have space + assert(host.size() < sizeof(trid.traddr)); + assert(port.size() < sizeof(trid.trsvcid)); + trid.trtype = SPDK_NVME_TRANSPORT_TCP; + trid.adrfam = SPDK_NVMF_ADRFAM_IPV4; + std::copy(host.begin(), host.end(), trid.traddr); + std::copy(port.begin(), port.end(), trid.trsvcid); + + spdk_nvmf_listen_opts lopts1; + spdk_nvmf_listen_opts_init(&lopts1, sizeof(lopts1)); + auto rc = spdk_nvmf_tgt_listen_ext(tgt, &trid, &lopts1); + assert(rc == 0); + + spdk_nvmf_listener_opts lopts; + spdk_nvmf_subsystem_listener_opts_init(&lopts, sizeof(lopts)); + lopts.secure_channel = false; + + spdk_nvmf_subsystem_add_listener_ext(ss, &trid, invoke_and_free_cb, cb, + &lopts); +} + +void add_bdev_ns(spdk_nvmf_subsystem *ss, str bdev_name) +{ + log_info("Adding bdev namespace to subsystem"); + spdk_nvmf_ns_opts nopts; + spdk_nvmf_ns_opts_get_defaults(&nopts, sizeof(nopts)); + auto err = spdk_nvmf_subsystem_add_ns_ext(ss, bdev_name.c_str(), &nopts, + sizeof(nopts), nullptr); + assert(err == 0); +} + +static void start_lsvd(void *arg) +{ + log_info("Starting LSVD SPDK program ..."); + auto args = (start_lsvd_args *)arg; - // Step 3: create subsystem for our bdev - log_info("Creating SPDK controller"); + auto io_ctx = connect_to_pool(args->pool_name); + + // Setup spdk nvmf + auto tgt = create_target(); + auto disc_ss = add_discovery_ss(tgt); + auto nvme_ss = add_nvme_ss(tgt); + + // Add lsvd bdev + lsvd_config cfg; // TODO read this in from a config file + cfg.cache_size = 160 * 1024 * 1024; // small 160mb cache for testing + auto err = bdev_lsvd_create(args->image_name, io_ctx, cfg); + assert(err == 0); + add_bdev_ns(nvme_ss, args->image_name); + + // some stupid formatting decisions up ahead due to tower-of-callback + // clang-format off + create_tcp_transport(new TranspCb([=](auto *tr) { + assert(tr != nullptr); + add_transport(tgt, tr, alloc_cb([=](int status) { + assert(status == 0); + add_ss_listener(tgt, nvme_ss, HOSTNAME, PORT, alloc_cb([=](int status) { + assert(status == 0); + // Start both subsystems + spdk_nvmf_subsystem_start(nvme_ss, nullptr, nullptr); + spdk_nvmf_subsystem_start(disc_ss, nullptr, nullptr); + })); + })); + })); + // clang-format on } int main(int argc, const char **argv)