Merge branch 'spdk-backend'

.github/workflows/test.yaml

@@ -8,21 +8,22 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
       - uses: actions/checkout@v3
 
-      - name: Setup Clang 17
+      - name: Setup Clang 18
         run: |
           wget https://apt.llvm.org/llvm.sh
           chmod +x llvm.sh
-          sudo ./llvm.sh 17
+          sudo ./llvm.sh 18
 
       - name: Install dependencies
         run: |
           sudo apt update
           make install-deps
+          sudo pip3 install --upgrade meson
 
       - name: Build
         run: |
@@ -37,12 +38,13 @@ jobs:
         run: |
           mkdir -p /tmp/lsvd-read
           mkdir -p /tmp/lsvd-write
+          mkdir -p /tmp/lsvd
           
           cd build-dbg
           sudo meson test
-          sudo ./imgtool --create --size 1g --pool pone test-img
+          sudo ./imgtool create --size 1g --pool pone --img test-img
 
       - name: Logs
         if: always()
         run: |
-          cat build-dbg/meson-logs/testlog.txt
+          cat build-dbg/meson-logs/*.txt

.gitignore

@@ -5,10 +5,11 @@ __pycache__/
 venv/
 .ipynb_checkpoints/
 
-subprojects/
-
 builddir
+build
 qemu/*.img
 qemu/*.iso
 qemu/bzImage
 
+subprojects/liburing-*
+subprojects/packagecache

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "spdk"]
-	path = spdk
-	url = https://github.com/spdk/spdk
 [submodule "lsvd-atc24"]
 	path = atc2024
 	url = git@github.com:CCI-MOC/lsvd-atc24.git

Makefile

@@ -2,11 +2,12 @@
 .PHONY: setup setup-debug release debug paper clean
 
 setup:
-	meson setup --native-file meson.ini build-rel --buildtype=release
-	meson setup --native-file meson.ini build-dbg --buildtype=debug
+	meson setup --native-file meson.ini build-rel --buildtype=release -Db_sanitize=none
+	meson setup --native-file meson.ini build-dbg --buildtype=debug 
 	ln -s build-dbg builddir
 
-debug: setup
+debug:
+	meson setup --native-file meson.ini build-dbg --buildtype=debug 
 	cd build-dbg; meson compile
 
 paper:
@@ -17,6 +18,14 @@ clean:
 	cd build-dbg; meson compile --clean
 
 install-deps:
-	sudo apt install -y meson libfmt-dev libaio-dev librados-dev mold \
-    	libtcmalloc-minimal4 libboost-dev libradospp-dev \
-    	liburing-dev pkg-config uuid-dev
+	# Folly deps
+	sudo apt install -y libboost-all-dev libdouble-conversion-dev libevent-dev \
+		libgflags-dev libgmock-dev libgoogle-glog-dev libgtest-dev \
+		liblz4-dev liblzma-dev libsnappy-dev libsodium-dev libunwind-dev \
+		libzstd-dev ninja-build zlib1g-dev
+	# SPDK deps
+	sudo apt install -y libnuma-dev libarchive-dev libibverbs-dev librdmacm-dev \
+		python3-pyelftools libcunit1-dev libaio-dev nasm
+	# LSVD deps
+	sudo apt install -y meson mold libfmt-dev librados-dev \
+    	libjemalloc-dev libradospp-dev pkg-config uuid-dev

README.md

@@ -52,7 +52,7 @@ Parameters are:
 - `batch_size`, `LSVD_BATCH_SIZE`: size of objects written to the backend, in bytes (K/M recognized as 1024, 1024\*1024). Default: 8MiB
 - `wcache_batch`: write cache batching (see below)
 - `wcache_chunk': maximum size of atomic write, in bytes - larger writes will be split and may be non-atomic.
-- `rcache_dir` - directory used for read cache file and GC temporary files. Note that `lsvd_imgtool` can format a partition for cache and symlink it into this directory, although the performance improvement seems limited.
+- `rcache_dir` - directory used for read cache file and GC temporary files. Note that `imgtool` can format a partition for cache and symlink it into this directory, although the performance improvement seems limited.
 - `wcache_dir` - directory used for write cache file
 - `xlate_window`: max writes (i.e. objects) in flight to the backend. Note that this value is coupled to the size of the write cache, which must be big enough to hold all outstanding writes in case of a crash.
 - `hard_sync` (untested): "flush" forces all batched writes to the backend.
@@ -70,7 +70,7 @@ figure out how to optimize at runtime instead of bothering the user for a value.
 
 First create a volume:
 ```
-build$ sudo bin/lsvd_imgtool --create --rados --size=20g pool/imgname
+build$ sudo imgtool create poolname imgname --size=20g 
 ```
 
 Then you can start a SPDK NVMe-oF gateway:
@@ -158,21 +158,17 @@ The read cache typically fetches 64K blocks, so there may be a bit of extra load
 Most of the testing to date has been with an 8,3 code with 64K stripe size.
 
 ## Tools
-`lsvd_imgtool` mostly just calls the LSVD versions of `rbd_create` and `rbd_remove`, although it can also format a cache file (e.g. if you're using a raw partition) 
-```
-build$ bin/lsvd_imgtool --help
-Usage: lsvd_imgtool [OPTION...]
-IMAGE
 
-  -C, --create               create image
-  -d, --cache-dir=DIR        cache directory
-  -D, --delete               delete image
-  -I, --info                 show image information
-  -k, --mkcache=DEV          use DEV as cache
-  -O, --rados                use RADOS
-  -z, --size=SIZE            size in bytes (M/G=2^20,2^30)
-  -?, --help                 Give this help list
-      --usage                Give a short usage message
+```
+build$ ./imgtool --help
+❯ ./imgtool --help
+Allowed options:
+  --help                produce help message
+  --cmd arg             subcommand: create, clone, delete, info
+  --img arg             name of the iname
+  --pool arg            pool where the image resides
+  --size arg (=1G)      size in bytes (M=2^20,G=2^30)
+  --dest arg            destination (for clone)
 ```
 
 Other tools live in the `tools` subdirectory - see the README there for more details.

docs/qemu-launch.md

@@ -5,7 +5,7 @@ install config, run QEMU with
 `-drive format=raw,file=seed.iso,cache=none,if=virtio`.
 
 1. Create a lsvd image if you don't already have one: 
-   `./imgtool --create --rados --size 10g $pool_name/$img_name`
+   `./imgtool create --size 10g $pool_name $img_name`
 2. Launch LSVD as a NVMF target `qemu-gateway.bash $pool_name $img_name`
 3. Lanuch QEMU with the NVMF target `qemu-client.bash`. This does the following:
    - `nvme connect` to the nvmf target on the gateway

experiments/multigw/gateway-1.bash

@@ -42,7 +42,7 @@ scripts/rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:lsvd-gw1 -a -s SPDKMULT
 scripts/rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:lsvd-gw1 -t tcp -a $gw_ip -s 9922
 
 function add_rbd_img {
-  cd $lsvd_dir/spdk
+  cd $lsvd_dir/subprojects/spdk
   local pool=$1
   local img=$2
   local bdev="bdev_$img"

experiments/multigw/gateway-2.bash

@@ -42,7 +42,7 @@ scripts/rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:lsvd-gw2 -a -s SPDKMULT
 scripts/rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:lsvd-gw2 -t tcp -a $gw_ip -s 9922
 
 function add_rbd_img {
-  cd $lsvd_dir/spdk
+  cd $lsvd_dir/subprojects/spdk
   local pool=$1
   local img=$2
   local bdev="bdev_$img"

experiments/multigw/rbd-gateway-1.bash

@@ -44,7 +44,7 @@ scripts/rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:lsvd-gw1 -a -s SPDKMULT
 scripts/rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:lsvd-gw1 -t tcp -a $gw_ip -s 9922
 
 function add_rbd_img {
-  cd $lsvd_dir/spdk
+  cd $lsvd_dir/subprojects/spdk
   local pool=$1
   local img=$2
   local bdev="bdev_$img"

experiments/multigw/rbd-gateway-2.bash

@@ -44,7 +44,7 @@ scripts/rpc.py nvmf_create_subsystem nqn.2016-06.io.spdk:lsvd-gw2 -a -s SPDKMULT
 scripts/rpc.py nvmf_subsystem_add_listener nqn.2016-06.io.spdk:lsvd-gw2 -t tcp -a $gw_ip -s 9922
 
 function add_rbd_img {
-  cd $lsvd_dir/spdk
+  cd $lsvd_dir/subprojects/spdk
   local pool=$1
   local img=$2
   local bdev="bdev_$img"

meson.build

@@ -1,11 +1,14 @@
 project(
 	'lsvd-rbd',
-	'cpp',
+	['c', 'cpp'],
 	version: '0.1',
 	default_options: [
-		'cpp_std=c++20',
+		'cpp_std=c++23',
 		'warning_level=2',
 		'b_colorout=always',
+		'b_sanitize=address,undefined',
+		'b_lto=true',
+		'b_thinlto_cache=true',
 	],
 )
 
@@ -14,34 +17,29 @@ add_project_arguments('-Wno-unused-parameter', language: 'cpp')
 
 if get_option('buildtype') == 'debug'
 	add_project_arguments('-fno-inline', language: 'cpp')
-	add_project_arguments('-DLOGLV=1', language: 'cpp')
-
-	# add sanitizers for debug builds
-	add_project_arguments(
-		'-fsanitize=address,undefined,nullability,implicit-conversion',
-		language: 'cpp',
-	)
-	add_project_link_arguments(
-		'-fsanitize=address,undefined',
-		'-Wl,--unresolved-symbols=ignore-in-object-files',
-		language: 'cpp',
-	)
+	add_project_arguments('-DLOGLV=0', language: 'cpp')
 endif
 
 subdir('src')
-liblsvd = library('lsvd', lsvd_src, dependencies: lsvd_deps, install: false)
+lsvd_ar = static_library('lsvd', lsvd_src, dependencies: lsvd_deps)
+
+executable(
+	'lsvd',
+	spdk_fe,
+	dependencies: lsvd_deps + [dependency('_spdk')],
+)
 
 executable(
 	'imgtool',
-	'src/imgtool.cc',
-	link_with: liblsvd,
+	['src/imgtool.cc'],
+	link_whole: lsvd_ar,
 	dependencies: lsvd_deps,
 )
 executable(
 	'thick-image',
-	'src/thick-image.cc',
-	link_with: liblsvd,
+	['src/thick-image.cc'],
+	link_whole: lsvd_ar,
 	dependencies: lsvd_deps,
 )
 
-subdir('test')
+subdir('test')

meson.ini

@@ -1,5 +1,5 @@
 [binaries]
-c = 'clang-17'
-c_ld = 'lld-17'
-cpp = 'clang++-17'
-cpp_ld = 'lld-17'
+c = 'clang-18'
+c_ld = 'lld-18'
+cpp = 'clang++-18'
+cpp_ld = 'lld-18'

src/backend.h

@@ -4,9 +4,9 @@
 #include <rados/librados.h>
 #include <sys/uio.h>
 
-#include "config.h"
 #include "request.h"
 #include "smartiov.h"
+#include "utils.h"
 
 class backend
 {
@@ -45,16 +45,11 @@ class backend
         smartiov iov((char *)buf, len);
         return aio_read(name, offset, iov);
     }
+
+    virtual opt<u64> get_size(std::string name) = 0;
+    virtual opt<vec<byte>> read_whole_obj(std::string name) = 0;
+    virtual bool exists(std::string name) = 0;
 };
 
-extern std::shared_ptr<backend> make_file_backend(const char *prefix);
 extern std::shared_ptr<backend> make_rados_backend(rados_ioctx_t io);
-
-inline std::shared_ptr<backend> get_backend(lsvd_config *cfg, rados_ioctx_t io,
-                                            const char *name)
-{
-    if (cfg->backend == BACKEND_RADOS)
-        return make_rados_backend(io);
-
-    throw std::runtime_error("Unknown backend");
-}
+rados_ioctx_t connect_to_pool(str pool_name);