From d4b66a45062baedeadf927780ce2b13539bf5632 Mon Sep 17 00:00:00 2001 From: Deyoung Hong Date: Thu, 28 Dec 2017 18:09:21 -0800 Subject: [PATCH] Revised Python wrapper and misc changes. --- .gitignore | 1 - Makefile | 7 +-- ioengine/Makefile | 4 +- ioengine/unvme_fio.c | 48 ++++++++------- src/Makefile | 12 ++-- src/rdtsc.h | 13 ++-- src/unvme_nvme.c | 34 ++++++----- src/unvme_nvme.h | 32 +++++++++- test/nvme/Makefile | 4 +- test/python/unvme.py | 98 +++++++++++++++++-------------- test/python/unvme_get_features.py | 1 + test/python/unvme_info.py | 1 + test/python/unvme_wr_ex.py | 52 +++++++--------- test/unvme-benchmark | 15 +++-- test/unvme/Makefile | 4 +- test/unvme/unvme_wrc.c | 4 +- 16 files changed, 185 insertions(+), 145 deletions(-) mode change 100644 => 100755 test/python/unvme_get_features.py mode change 100644 => 100755 test/python/unvme_info.py mode change 100644 => 100755 test/python/unvme_wr_ex.py diff --git a/.gitignore b/.gitignore index eb23f6b..9402a30 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,6 @@ *.so *.pyc *.*~ -src/.model src/unvmed test/unvme/*_test test/unvme/unvme_info diff --git a/Makefile b/Makefile index 69359ef..ce11d28 100644 --- a/Makefile +++ b/Makefile @@ -50,11 +50,8 @@ uninstall: $(INSTALLDIR)/lib/libunvme* \ $(INSTALLDIR)/bin/unvme* -lint: - @(for d in $(SUBDIRS); do $(MAKE) -C $$d lint; done) - -clean: - @(for d in $(SUBDIRS); do $(MAKE) -C $$d clean; done) +clean lint: + @(for d in $(SUBDIRS); do $(MAKE) -C $$d $@; done) .PHONY: all install uninstall lint clean $(SUBDIRS) diff --git a/ioengine/Makefile b/ioengine/Makefile index fc908e0..d8d1357 100644 --- a/ioengine/Makefile +++ b/ioengine/Makefile @@ -42,7 +42,7 @@ UNVME_SRC = ../src CPPFLAGS += -I$(UNVME_SRC) -I$(FIODIR) LDFLAGS += -shared -rdynamic -LDLIBS += -lpthread -lrt +LDLIBS += -lrt OBJS = $(addsuffix .o, $(TARGET)) @@ -50,7 +50,7 @@ all: $(TARGET) $(TARGET): $(UNVME_SRC)/libunvme.a -lint: CFLAGS = -Wall -D_FORTIFY_SOURCE=2 -DUNVME_DEBUG -O3 +lint: CFLAGS = -Wall -O3 -D_FORTIFY_SOURCE=2 -DUNVME_DEBUG lint: clean $(OBJS) @$(RM) *.o diff --git a/ioengine/unvme_fio.c b/ioengine/unvme_fio.c index 809485a..7fddd3b 100644 --- a/ioengine/unvme_fio.c +++ b/ioengine/unvme_fio.c @@ -12,9 +12,11 @@ #include "fio.h" #include "optgroup.h" // since fio 2.4 + #define TDEBUG(fmt, arg...) //fprintf(stderr, "#%s.%d " fmt "\n", __func__, td->thread_number, ##arg) #define FATAL(fmt, arg...) do { warnx(fmt, ##arg); abort(); } while (0) + /// Context used for thread initialization typedef struct { pthread_mutex_t mutex; @@ -42,7 +44,7 @@ static void do_unvme_cleanup(void) /* * Initialize UNVMe once. */ -static int do_unvme_init(struct thread_data *td) +static void do_unvme_init(struct thread_data *td) { TDEBUG("device=%s numjobs=%d iodepth=%d", td->o.filename, td->o.numjobs, td->o.iodepth); @@ -74,25 +76,6 @@ static int do_unvme_init(struct thread_data *td) } pthread_mutex_unlock(&unvme.mutex); - return 0; -} - -/* - * The ->get_file_size() is called once for every job (i.e. numjobs) - * before all other functions. This is called after ->setup() but - * is simpler to initialize here since we only care about the device name - * (given as file_name) and just have to specify the device size. - */ -static int fio_unvme_get_file_size(struct thread_data *td, struct fio_file *f) -{ - TDEBUG("file=%s", f->file_name); - if (!fio_file_size_known(f)) { - do_unvme_init(td); - f->filetype = FIO_TYPE_CHAR; - f->real_file_size = unvme.ns->blockcount * unvme.ns->blocksize; - fio_file_set_size_known(f); - } - return 0; } /* @@ -208,6 +191,9 @@ static int fio_unvme_getevents(struct thread_data *td, unsigned int min, return 0; } + +#ifndef _UNVME_MMCC_H + /* * The ->queue() hook is responsible for initiating io on the io_u * being passed in. If the io engine is a synchronous one, io may complete @@ -220,10 +206,10 @@ static int fio_unvme_getevents(struct thread_data *td, unsigned int min, */ static int fio_unvme_queue(struct thread_data *td, struct io_u *io_u) { + int q = td->thread_number - 1; void* buf = io_u->buf; u64 slba = io_u->offset >> unvme.ns->blockshift; int nlb = io_u->xfer_buflen >> unvme.ns->blockshift; - int q = td->thread_number - 1; switch (io_u->ddir) { case DDIR_READ: @@ -247,6 +233,24 @@ static int fio_unvme_queue(struct thread_data *td, struct io_u *io_u) return FIO_Q_COMPLETED; } +/* + * The ->get_file_size() is called once for every job (i.e. numjobs) + * before all other functions. This is called after ->setup() but + * is simpler to initialize here since we only care about the device name + * (given as file_name) and just have to specify the device size. + */ +static int fio_unvme_get_file_size(struct thread_data *td, struct fio_file *f) +{ + TDEBUG("file=%s", f->file_name); + if (!fio_file_size_known(f)) { + do_unvme_init(td); + f->filetype = FIO_TYPE_CHAR; + f->real_file_size = unvme.ns->blockcount * unvme.ns->blocksize; + fio_file_set_size_known(f); + } + return 0; +} + // Note that the structure is exported, so that fio can get it via // dlsym(..., "ioengine"); @@ -266,3 +270,5 @@ struct ioengine_ops ioengine = { .flags = FIO_NOEXTEND | FIO_RAWIO, }; +#endif // _UNVME_MMCC_H + diff --git a/src/Makefile b/src/Makefile index 739d3f3..f76d2b0 100644 --- a/src/Makefile +++ b/src/Makefile @@ -31,28 +31,28 @@ include ../Makefile.def -TARGET = libunvme.a -TARGET_SO = libunvme.so +TARGET_LIB = libunvme.a +TARGET_LIBSO = libunvme.so INCS = $(wildcard *.h) SRCS = $(wildcard *.c) OBJS = $(SRCS:.c=.o) LDLIBS = -lrt -all: $(TARGET) $(TARGET_SO) +all: $(TARGET_LIB) $(TARGET_LIBSO) $(OBJS): $(INCS) -$(TARGET): $(OBJS) +$(TARGET_LIB): $(OBJS) $(AR) crs $@ $^ -$(TARGET_SO): $(OBJS) +$(TARGET_LIBSO): $(OBJS) $(CC) -shared -rdynamic -o $@ $^ -lrt %.i: %.c $(CPP) $(CPPFLAGS) -o $@ $< -lint: CFLAGS = -Wall -D_FORTIFY_SOURCE=2 -DUNVME_DEBUG -O3 +lint: CFLAGS = -Wall -O3 -D_FORTIFY_SOURCE=2 -DUNVME_DEBUG lint: clean $(OBJS) @$(RM) *.o diff --git a/src/rdtsc.h b/src/rdtsc.h index 5a1e47e..7e10276 100644 --- a/src/rdtsc.h +++ b/src/rdtsc.h @@ -65,7 +65,6 @@ static inline uint64_t rdtsc(void) */ static inline uint64_t rdtsc_elapse(uint64_t tsc) { int64_t et; - do { et = rdtsc() - tsc; } while (et <= 0); @@ -79,9 +78,15 @@ static inline uint64_t rdtsc_second() { static uint64_t tsc_ps = 0; if (!tsc_ps) { - uint64_t tsc = rdtsc(); - usleep(10000); - tsc_ps = (rdtsc() - tsc) * 100; + uint64_t t0 = rdtsc(); + usleep(1000); + uint64_t t1 = rdtsc(); + usleep(1000); + uint64_t t2 = rdtsc(); + t2 -= t1; + t1 -= t0; + if (t2 > t1) t2 = t1; + tsc_ps = t2 * 1000; } return tsc_ps; } diff --git a/src/unvme_nvme.c b/src/unvme_nvme.c index 78d4bb2..3987d75 100644 --- a/src/unvme_nvme.c +++ b/src/unvme_nvme.c @@ -174,10 +174,10 @@ int nvme_check_completion(nvme_queue_t* q, int* stat, u32* cqe_cs) #endif if (*stat == 0) { - DEBUG_FN("q=%d h=%d cid=%#x (C)", q->id, q->cq_head, cqe->cid); + DEBUG_FN("q=%d cq=%d sq=%d-%d cid=%#x (C)", q->id, q->cq_head, q->sq_head, q->sq_tail, cqe->cid); } else { - ERROR("q=%d cid=%#x stat=%#x (dnr=%d m=%d sct=%d sc=%#x) (C)", - q->id, cqe->cid, *stat, cqe->dnr, cqe->m, cqe->sct, cqe->sc); + ERROR("q=%d cq=%d sq=%d-%d cid=%#x stat=%#x (dnr=%d m=%d sct=%d sc=%#x) (C)", + q->id, q->cq_head, q->sq_head, q->sq_tail, cqe->cid, *stat, cqe->dnr, cqe->m, cqe->sct, cqe->sc); } return cqe->cid; @@ -236,7 +236,7 @@ int nvme_acmd_identify(nvme_device_t* dev, int nsid, u64 prp1, u64 prp2) cmd->common.prp2 = prp2; cmd->cns = nsid == 0 ? 1 : 0; - DEBUG_FN("t=%d h=%d cid=%#x nsid=%d", adminq->sq_tail, adminq->sq_head, cid, nsid); + DEBUG_FN("sq=%d-%d cid=%#x nsid=%d", adminq->sq_head, adminq->sq_tail, cid, nsid); int err = nvme_submit_cmd(adminq); if (!err) err = nvme_wait_completion(adminq, cid, 30); return err; @@ -269,7 +269,7 @@ int nvme_acmd_get_log_page(nvme_device_t* dev, int nsid, cmd->lid = lid; cmd->numd = numd; - DEBUG_FN("t=%d h=%d cid=%#x lid=%d", adminq->sq_tail, adminq->sq_head, cid, lid); + DEBUG_FN("sq=%d-%d cid=%#x lid=%d", adminq->sq_head, adminq->sq_tail, cid, lid); int err = nvme_submit_cmd(adminq); if (!err) err = nvme_wait_completion(adminq, cid, 30); return err; @@ -302,7 +302,7 @@ int nvme_acmd_get_features(nvme_device_t* dev, int nsid, cmd->fid = fid; *res = -1; - DEBUG_FN("t=%d h=%d cid=%#x fid=%d", adminq->sq_tail, adminq->sq_head, cid, fid); + DEBUG_FN("sq=%d-%d cid=%#x fid=%d", adminq->sq_head, adminq->sq_tail, cid, fid); int err = nvme_submit_cmd(adminq); if (!err) err = nvme_wait_completion(adminq, cid, 30); if (!err) *res = adminq->cq[cid].cs; @@ -365,7 +365,7 @@ int nvme_acmd_create_cq(nvme_queue_t* ioq, u64 prp) cmd->qid = ioq->id; cmd->qsize = ioq->size - 1; - DEBUG_FN("t=%d h=%d cid=%#x cq=%d qs=%d", adminq->sq_tail, adminq->sq_head, cid, ioq->id, ioq->size); + DEBUG_FN("sq=%d-%d cid=%#x cq=%d qs=%d", adminq->sq_head, adminq->sq_tail, cid, ioq->id, ioq->size); int err = nvme_submit_cmd(adminq); if (!err) err = nvme_wait_completion(adminq, cid, 30); return err; @@ -394,7 +394,7 @@ int nvme_acmd_create_sq(nvme_queue_t* ioq, u64 prp) cmd->cqid = ioq->id; cmd->qsize = ioq->size - 1; - DEBUG_FN("t=%d h=%d cid=%#x sq=%d qs=%d", adminq->sq_tail, adminq->sq_head, cid, ioq->id, ioq->size); + DEBUG_FN("sq=%d-%d cid=%#x cq=%d qs=%d", adminq->sq_head, adminq->sq_tail, cid, ioq->id, ioq->size); int err = nvme_submit_cmd(adminq); if (!err) err = nvme_wait_completion(adminq, cid, 30); return err; @@ -418,7 +418,7 @@ static inline int nvme_acmd_delete_ioq(nvme_queue_t* ioq, int opc) cmd->common.cid = cid; cmd->qid = ioq->id; - DEBUG_FN("t=%d h=%d cid=%#x %cq=%d", adminq->sq_tail, adminq->sq_head, cid, + DEBUG_FN("sq=%d-%d cid=%#x %cq=%d", adminq->sq_head, adminq->sq_tail, cid, opc == NVME_ACMD_DELETE_CQ ? 'c' : 's', ioq->id); int err = nvme_submit_cmd(adminq); if (!err) err = nvme_wait_completion(adminq, cid, 30); @@ -470,8 +470,8 @@ int nvme_cmd_vs(nvme_queue_t* q, int opc, u16 cid, int nsid, cmd->common.prp1 = prp1; cmd->common.prp2 = prp2; if (cdw10_15) memcpy(cmd->cdw10_15, cdw10_15, sizeof(cmd->cdw10_15)); - DEBUG_FN("q=%d t=%d h=%d cid=%#x nsid=%d opc=%#x", - q->id, q->sq_tail, q->sq_head, cid, nsid, opc); + DEBUG_FN("q=%d sq=%d-%d cid=%#x nsid=%d opc=%#x", + q->id, q->sq_head, q->sq_tail, cid, nsid, opc); return nvme_submit_cmd(q); } @@ -500,8 +500,8 @@ int nvme_cmd_rw(nvme_queue_t* ioq, int opc, u16 cid, int nsid, cmd->common.prp2 = prp2; cmd->slba = slba; cmd->nlb = nlb - 1; - DEBUG_FN("q=%d t=%d h=%d cid=%#x nsid=%d lba=%#lx nb=%#x prp=%#lx.%#lx (%c)", - ioq->id, ioq->sq_tail, ioq->sq_head, cid, nsid, slba, nlb, prp1, prp2, + DEBUG_FN("q=%d sq=%d-%d cid=%#x nsid=%d lba=%#lx nb=%#x prp=%#lx.%#lx (%c)", + ioq->id, ioq->sq_head, ioq->sq_tail, cid, nsid, slba, nlb, prp1, prp2, opc == NVME_CMD_READ? 'R' : 'W'); return nvme_submit_cmd(ioq); } @@ -662,8 +662,12 @@ nvme_device_t* nvme_create(nvme_device_t* dev, int mapfd) dev->maxqsize = cap.mqes + 1; dev->dbstride = 1 << cap.dstrd; // in u32 size offset - DEBUG_FN("cap=%#lx mps=%u-%u to=%u maxqs=%u dbs=%u", cap.val, - cap.mpsmin, cap.mpsmax, cap.to, dev->maxqsize, dev->dbstride); + dev->cmbloc.val = r32(dev, &dev->reg->cmbloc.val); + dev->cmbsz.val = r32(dev, &dev->reg->cmbsz.val); + + DEBUG_FN("cap=%#lx mps=%u-%u to=%u maxqs=%u dbs=%u cmbloc=%#x cmbsz=%#x", + cap.val, cap.mpsmin, cap.mpsmax, cap.to, dev->maxqsize, + dev->dbstride, dev->cmbloc.val, dev->cmbsz.val); return dev; } diff --git a/src/unvme_nvme.h b/src/unvme_nvme.h index d6335c4..38a0a6f 100644 --- a/src/unvme_nvme.h +++ b/src/unvme_nvme.h @@ -164,6 +164,31 @@ typedef union _nvme_controller_status { }; } nvme_controller_status_t; +/// Controller memory buffer location register +typedef union _nvme_cmbloc { + u32 val; ///< whole value + struct { + u32 bir : 3; ///< base indicator register + u32 rsvd : 9; ///< reserved + u32 ofst : 20; ///< offset (in cmbsz units) + }; +} nvme_cmbloc_t; + +/// Controller memory buffer size register +typedef union _nvme_cmbsz { + u32 val; ///< whole value + struct { + u32 sqs : 1; ///< submission queue support + u32 cqs : 1; ///< completion queue support + u32 lists : 1; ///< PRP SGL list support + u32 rds : 1; ///< read data support + u32 wds : 1; ///< write data support + u32 rsvd : 3; ///< reserved + u32 szu : 4; ///< size units (0=4K,1=64K,2=1M,3=16M,4=256M,5=4G,6=64G) + u32 sz : 20; ///< size (in cmbsz units) + }; +} nvme_cmbsz_t; + /// Controller register (bar 0) typedef struct _nvme_controller_reg { nvme_controller_cap_t cap; ///< controller capabilities @@ -177,8 +202,8 @@ typedef struct _nvme_controller_reg { nvme_adminq_attr_t aqa; ///< admin queue attributes u64 asq; ///< admin submission queue base address u64 acq; ///< admin completion queue base address - u32 cmbloc; ///< controller memory buffer location - u32 cmbsz; ///< controller memory buffer size + nvme_cmbloc_t cmbloc; ///< controller memory buffer location + nvme_cmbsz_t cmbsz; ///< controller memory buffer size u32 rcss[1008]; ///< reserved and command set specific u32 sq0tdbl[1024]; ///< sq0 tail doorbell at 0x1000 } nvme_controller_reg_t; @@ -545,6 +570,8 @@ typedef struct _nvme_queue { typedef struct _nvme_device { nvme_controller_reg_t* reg; ///< register address map nvme_queue_t adminq; ///< admin queue reference + nvme_cmbloc_t cmbloc; ///< CMB location + nvme_cmbsz_t cmbsz; ///< CMB size u64 rdtsec; ///< rdtsc per second u32 maxqcount; ///< max queue count u32 maxqsize; ///< max queue size @@ -583,3 +610,4 @@ int nvme_check_completion(nvme_queue_t* q, int* stat, u32* cqe_cs); int nvme_wait_completion(nvme_queue_t* q, int cid, int timeout); #endif // _UNVME_NVME_H + diff --git a/test/nvme/Makefile b/test/nvme/Makefile index 4c071cc..abfe3a7 100644 --- a/test/nvme/Makefile +++ b/test/nvme/Makefile @@ -36,7 +36,7 @@ TARGETS = nvme_identify nvme_get_log_page nvme_get_features nvme_set_features UNVME_SRC = ../../src CPPFLAGS += -I$(UNVME_SRC) -LDLIBS += $(UNVME_SRC)/libunvme.a -lrt -lpthread +LDLIBS += -lrt OBJS = $(addsuffix .o, $(TARGETS)) @@ -44,7 +44,7 @@ all: $(TARGETS) $(TARGETS): $(UNVME_SRC)/libunvme.a -lint: CFLAGS = -Wall -D_FORTIFY_SOURCE=2 -DUNVME_DEBUG -O3 +lint: CFLAGS = -Wall -O3 -D_FORTIFY_SOURCE=2 -DUNVME_DEBUG lint: clean $(OBJS) @$(RM) *.o diff --git a/test/python/unvme.py b/test/python/unvme.py index 91e8580..9422c31 100644 --- a/test/python/unvme.py +++ b/test/python/unvme.py @@ -4,7 +4,7 @@ from ctypes import * # Namespace attributes structure -class unvme_ns(ctypes.Structure): +class unvme_ns_t(ctypes.Structure): _fields_ = [ ("pci", c_uint32), # PCI device id ("id", c_uint16), # namespace id @@ -29,18 +29,18 @@ class unvme_ns(ctypes.Structure): ("maxqcount", c_uint32), # max number of queues supported ("qsize", c_uint32), # I/O queue size ("maxqsize", c_uint32), # max queue size supported - ("ses", POINTER(c_uint64)) # associated session + ("ses", c_void_p) # associated session ] # I/O descriptor structure -class unvme_iod(ctypes.Structure): - _fields = [ - ("buf", POINTER(c_uint32)), # data buffer (submitted) +class unvme_iod_t(ctypes.Structure): + _fields_ = [ + ("buf", c_void_p), # data buffer (submitted) ("slba", c_uint64), # starting lba (submitted) ("nlb", c_uint32), # number of blocks (submitted) ("qid", c_uint32), # queue id (submitted) ("opc", c_uint32), # op code - ("id", c_uint32), # descriptor id + ("id", c_uint32) # descriptor id ] # Load libunvme @@ -52,67 +52,75 @@ class unvme_iod(ctypes.Structure): # UNVMe library functions def unvme_open(pci): - open = libunvme.unvme_open - open.restype = POINTER(unvme_ns) - return open(pci) + fn = libunvme.unvme_open + fn.restype = POINTER(unvme_ns_t) + return fn(pci) def unvme_close(ns): - close = libunvme.unvme_close - close.argtypes = [POINTER(unvme_ns)] - return close(ns) + fn = libunvme.unvme_close + fn.argtypes = [POINTER(unvme_ns_t)] + fn.restype = c_int + return fn(ns) def unvme_alloc(ns, size): - alloc = libunvme.unvme_alloc - alloc.argtypes = [POINTER(unvme_ns), c_uint64] - alloc.restype = POINTER(c_uint64) - return alloc(ns, size) + fn = libunvme.unvme_alloc + fn.argtypes = [POINTER(unvme_ns_t), c_uint64] + fn.restype = c_void_p + return fn(ns, size) def unvme_free(ns, buf): - free = libunvme.unvme_free - free.argtypes = [POINTER(unvme_ns), POINTER(c_uint64)] - return free(ns, buf) + fn = libunvme.unvme_free + fn.argtypes = [POINTER(unvme_ns_t), c_void_p] + fn.restype = c_int + return fn(ns, buf) def unvme_write(ns, qid, buf, slba, nlb): - write = libunvme.unvme_write - write.argtypes = [POINTER(unvme_ns), c_int, POINTER(c_uint64), c_uint64, c_uint32] - return write(ns, qid, buf, slba, nlb) + fn = libunvme.unvme_write + fn.argtypes = [POINTER(unvme_ns_t), c_int, c_void_p, c_uint64, c_uint32] + fn.restype = c_int + return fn(ns, qid, buf, slba, nlb) def unvme_read(ns, qid, buf, slba, nlb): - read = libunvme.unvme_read - read.argtypes = [POINTER(unvme_ns), c_int, POINTER(c_uint64), c_uint64, c_uint32] - return read(ns, qid, buf, slba, nlb) + fn = libunvme.unvme_read + fn.argtypes = [POINTER(unvme_ns_t), c_int, c_void_p, c_uint64, c_uint32] + fn.restype = c_int + return fn(ns, qid, buf, slba, nlb) def unvme_cmd(ns, qid, opc, nsid, buf, bufsz, cdw10_15, cqe_cs): - cmd = libunvme.unvme_cmd - cmd.argtypes = [POINTER(unvme_ns), c_int, c_int, c_int, POINTER(c_uint64), c_uint64, POINTER(c_uint32), POINTER(c_uint32)] - return cmd(ns, qid, opc, nsid, buf, bufsz, cdw10_15, cqe_cs) + fn = libunvme.unvme_cmd + fn.argtypes = [POINTER(unvme_ns_t), c_int, c_int, c_int, c_void_p, c_uint64, POINTER(c_uint32), POINTER(c_uint32)] + fn.restype = c_int + return fn(ns, qid, opc, nsid, buf, bufsz, cdw10_15, cqe_cs) def unvme_awrite(ns, qid, buf, slba, nlb): - awrite = libunvme.unvme_awrite - awrite.argtypes = [POINTER(unvme_ns), c_int, POINTER(c_uint64), c_uint64, c_uint32] - awrite.restype = POINTER(unvme_iod) - return awrite(ns, qid, buf, slba, nlb) + fn = libunvme.unvme_awrite + fn.argtypes = [POINTER(unvme_ns_t), c_int, c_void_p, c_uint64, c_uint32] + fn.restype = POINTER(unvme_iod_t) + return fn(ns, qid, buf, slba, nlb) def unvme_aread(ns, qid, buf, slba, nlb): - aread = libunvme.unvme_aread - aread.argtypes = [POINTER(unvme_ns), c_int, POINTER(c_uint64), c_uint64, c_uint32] - aread.restype = POINTER(unvme_iod) - return aread(ns, qid, buf, slba, nlb) + fn = libunvme.unvme_aread + fn.argtypes = [POINTER(unvme_ns_t), c_int, c_void_p, c_uint64, c_uint32] + fn.restype = POINTER(unvme_iod_t) + return fn(ns, qid, buf, slba, nlb) def unvme_acmd(ns, qid, opc, nsid, buf, bufsz, cdw10_15): - acmd = libunvme.unvme_acmd - acmd.argtypes = [POINTER(unvme_ns), c_int, c_int, c_int, POINTER(c_uint64), c_uint64, POINTER(c_uint32)] - return acmd(ns, qid, opc, nsid, buf, bufsz, cdw10_15) + fn = libunvme.unvme_acmd + fn.argtypes = [POINTER(unvme_ns_t), c_int, c_int, c_int, c_void_p, c_uint64, POINTER(c_uint32)] + fn.restype = POINTER(unvme_iod_t) + return fn(ns, qid, opc, nsid, buf, bufsz, cdw10_15) def unvme_apoll(iod, timeout): - apoll = libunvme.unvme_apoll - apoll.argtypes = [POINTER(unvme_iod), c_int] - return apoll(iod, timeout) + fn = libunvme.unvme_apoll + fn.argtypes = [POINTER(unvme_iod_t), c_int] + fn.restype = c_int + return fn(iod, timeout) def unvme_apoll_cs(iod, timeout, cqe_cs): - apoll = libunvme.unvme_apoll_cs - apoll.argtypes = [POINTER(unvme_iod), c_int, POINTER(c_uint32)] - return apoll(iod, timeout, cqe_cs) + fn = libunvme.unvme_apoll_cs + fn.argtypes = [POINTER(unvme_iod_t), c_int, POINTER(c_uint32)] + fn.restype = c_int + return fn(iod, timeout, cqe_cs) # UNVMe class diff --git a/test/python/unvme_get_features.py b/test/python/unvme_get_features.py old mode 100644 new mode 100755 index 3facfab..ff44737 --- a/test/python/unvme_get_features.py +++ b/test/python/unvme_get_features.py @@ -1,3 +1,4 @@ +#!/bin/python # Example to invoke admin get features command. # diff --git a/test/python/unvme_info.py b/test/python/unvme_info.py old mode 100644 new mode 100755 index ae912d6..8da4dff --- a/test/python/unvme_info.py +++ b/test/python/unvme_info.py @@ -1,3 +1,4 @@ +#!/bin/python # Example to open a device and print its namespace info. # # This program shows 2 ways to invoke Python UNVMe interfaces: diff --git a/test/python/unvme_wr_ex.py b/test/python/unvme_wr_ex.py old mode 100644 new mode 100755 index 97a0122..4d6b65b --- a/test/python/unvme_wr_ex.py +++ b/test/python/unvme_wr_ex.py @@ -1,20 +1,10 @@ +#!/bin/python # Example to write-read-verify synchronously and asynchronously. # from __future__ import print_function -import sys -import argparse -import random -import time -import unvme - - -# IO buffer class. -#class ExBuf(object): -# def __init__(self, id, buf): -# self.id = id -# self.buf = buf - +import os, sys, argparse, random, time, ctypes, unvme +from ctypes import * # Write-Read-Verify example test class. class WRTest: @@ -28,8 +18,10 @@ def __init__(self, pci, ioc, maxnlb): self.wbuf = [] self.rbuf = [] for i in range(ioc): - self.wbuf.append(self.unvme.alloc(maxnlb * self.ns.blocksize)) - self.rbuf.append(self.unvme.alloc(maxnlb * self.ns.blocksize)) + buf = cast(self.unvme.alloc(maxnlb * self.ns.blocksize), POINTER(c_uint64)) + self.wbuf.append(buf) + buf = cast(self.unvme.alloc(maxnlb * self.ns.blocksize), POINTER(c_uint64)) + self.rbuf.append(buf) def __del__(self): for i in range(0, self.ioc): @@ -55,7 +47,7 @@ def verifyBuffer(self, buf, lba, nlb): pat = (lba << 24) | b for w in range(self.wpb): if buf[i] != pat: - self.error("ERROR: data miscompare at lba %#lx" % lba) + self.error('ERROR: data miscompare at lba %#lx' % lba) i = i + 1 lba = lba + 1 @@ -67,25 +59,25 @@ def pickqbc(self, seed): return q, lba, nlb def syncTest(self): - print("Sync test"); + print('Sync test'); for i in range(self.ioc): q, lba, nlb = self.pickqbc(i) - print(" write q=%-3d lba=0x%-9lx nlb=%#x" % (q, lba, nlb)) + print(' write q=%-3d lba=0x%-9lx nlb=%#x' % (q, lba, nlb)) self.fillBuffer(self.wbuf[i], lba, nlb) self.unvme.write(q, self.wbuf[i], lba, nlb) for i in range(self.ioc): q, lba, nlb = self.pickqbc(i) - print(" read q=%-3d lba=0x%-9lx nlb=%#x" % (q, lba, nlb)) + print(' read q=%-3d lba=0x%-9lx nlb=%#x' % (q, lba, nlb)) self.unvme.read(q, self.rbuf[i], lba, nlb) self.verifyBuffer(self.rbuf[i], lba, nlb) def asyncTest(self): - print("Async test"); + print('Async test'); wiod = [] for i in range(self.ioc): q, lba, nlb = self.pickqbc(i) - print(" awrite q=%-3d lba=0x%-9lx nlb=%#x" % (q, lba, nlb)) + print(' awrite q=%-3d lba=0x%-9lx nlb=%#x' % (q, lba, nlb)) self.fillBuffer(self.wbuf[i], lba, nlb) iod = self.unvme.awrite(q, self.wbuf[i], lba, nlb) wiod.append(iod) @@ -93,31 +85,31 @@ def asyncTest(self): riod = [] for i in range(self.ioc): q, lba, nlb = self.pickqbc(i) - print(" aread q=%-3d lba=0x%-9lx nlb=%#x" % (q, lba, nlb)) + print(' aread q=%-3d lba=0x%-9lx nlb=%#x' % (q, lba, nlb)) iod = self.unvme.aread(q, self.rbuf[i], lba, nlb) riod.append(iod) for i in range(self.ioc): q, lba, nlb = self.pickqbc(i) - print(" apoll q=%-3d lba=0x%-9lx nlb=%#x" % (q, lba, nlb)) + print(' apoll q=%-3d lba=0x%-9lx nlb=%#x' % (q, lba, nlb)) if self.unvme.apoll(wiod[i], 30) or self.unvme.apoll(riod[i], 30): - self.error("ERROR: apoll") + self.error('ERROR: apoll') self.verifyBuffer(self.rbuf[i], lba, nlb) # Main program. -parser = argparse.ArgumentParser() -parser.add_argument('--ioc', help="number of IOs per test", type=int, default=8) -parser.add_argument('--nlb', help="max number of blocks per IO", type=int, default=65536) -parser.add_argument('pci', help="PCI device name (as 0a:00.0[/1] format)") +parser = argparse.ArgumentParser(description='RANDOM WRITE-READ TEST EXAMPLE', add_help=False) +parser.add_argument('--ioc', help='number of IOs per test', type=int, default=8) +parser.add_argument('--nlb', help='max number of blocks per IO', type=int, default=65536) +parser.add_argument('pci', help='PCI device name (as 0a:00.0[/1] format)') if len(sys.argv) == 1: parser.print_help() sys.exit(1) opts = parser.parse_args() -print("PYTHON RANDOM WRITE-READ EXAMPLE TEST BEGIN") +print('PYTHON RANDOM WRITE-READ TEST EXAMPLE BEGIN') ex = WRTest(opts.pci, opts.ioc, opts.nlb) ex.syncTest() ex.asyncTest() -print("PYTHON RANDOM WRITE-READ EXAMPLE TEST COMPLETE") +print('PYTHON RANDOM WRITE-READ TEST EXAMPLE COMPLETE') diff --git a/test/unvme-benchmark b/test/unvme-benchmark index c811123..36a6d02 100755 --- a/test/unvme-benchmark +++ b/test/unvme-benchmark @@ -32,19 +32,18 @@ iodepth=IODEPTH" : ${FIODIR=$(grep '^FIODIR' ../Makefile.def | sed 's/.*=\s*//')} : ${RW="randwrite randread"} -: ${NUMJOBS="01 04 08"} -: ${IODEPTH="01 04 08 16 32"} +: ${NUMJOBS="01 08 16"} +: ${IODEPTH="01 08 16 32"} : ${RAMPTIME=60} : ${RUNTIME=120} : ${BLOCKSIZE=4096} : ${IOENGINE=""} -[ $# -lt 1 ] && echo ${USAGE} && exit 1 +FIO=${FIODIR}/fio +[ $# -lt 1 ] && echo ${USAGE} && exit 1 [ ${EUID} -ne 0 ] && echo "ERROR: ${PROG} must be run as root" && exit 1 - -FIO=fio -[ -n "${FIODIR}" ] && FIO=${FIODIR}/fio +[ ! -x "${FIO}" ] && echo "ERROR: Missing ${FIO}" && exit 1 for i in $@; do case $i in @@ -53,8 +52,8 @@ for i in $@; do echo "Unknown NVMe device: $i" exit 1 fi - [ -z "${IOENGINE}" ] && IOENGINE=libaio FILENAME=$i + [ -z "${IOENGINE}" ] && IOENGINE=libaio DRIVER="nvme" ;; @@ -68,8 +67,8 @@ for i in $@; do echo "Device ${pcidev} is not uNVMe enabled" exit 1 fi - [ -z "${IOENGINE}" ] && IOENGINE=$(readlink -f ../ioengine/unvme_fio) FILENAME=$(echo $i | tr :/ .) + [ -z "${IOENGINE}" ] && IOENGINE=$(readlink -f ../ioengine/unvme_fio) DRIVER="unvme" [ -n "$(pgrep unvmed)" ] && DRIVER="unvmed" ;; diff --git a/test/unvme/Makefile b/test/unvme/Makefile index 92cbe7d..ad71a59 100644 --- a/test/unvme/Makefile +++ b/test/unvme/Makefile @@ -38,7 +38,7 @@ TARGETS = unvme_sim_test unvme_api_test unvme_mts_test unvme_lat_test \ UNVME_SRC = ../../src CPPFLAGS += -I$(UNVME_SRC) -LDLIBS += $(UNVME_SRC)/libunvme.a -lrt -lpthread +LDLIBS += -lrt -lpthread OBJS = $(addsuffix .o, $(TARGETS)) @@ -46,7 +46,7 @@ all: $(TARGETS) $(TARGETS): $(UNVME_SRC)/libunvme.a -lint: CFLAGS = -Wall -D_FORTIFY_SOURCE=2 -DUNVME_DEBUG -O3 +lint: CFLAGS = -Wall -O3 -D_FORTIFY_SOURCE=2 -DUNVME_DEBUG lint: clean $(OBJS) @$(RM) *.o diff --git a/test/unvme/unvme_wrc.c b/test/unvme/unvme_wrc.c index 57fdab0..439bb81 100644 --- a/test/unvme/unvme_wrc.c +++ b/test/unvme/unvme_wrc.c @@ -265,9 +265,9 @@ int main(int argc, char** argv) if (stat) { // terminate on error if (stat != -1) - errx(1, "unvme_apoll error %#x", stat); + errx(1, "unvme_apoll error=%#x slba=%#lx nlb=%#x", stat, iod->slba, iod->nlb); else if ((time(0) - tio) > UNVME_TIMEOUT) - errx(1, "unvme_apoll timeout"); + errx(1, "unvme_apoll timeout slba=%#lx nlb=%#x", iod->slba, iod->nlb); // if no completion go on to next queue if (++q >= qcount) q = 0; continue;