Rearranged codes and added unvme_info.

Author: Deyoung Hong

.gitignore

@@ -4,6 +4,7 @@ src/.model
 src/libunvme.a
 src/unvmed
 test/unvme/*_test
+test/unvme/unvme_info
 test/nvme/nvme_identify
 test/nvme/nvme_get_log_page
 test/nvme/nvme_get_features

Makefile.def

@@ -29,16 +29,16 @@
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 #
 
-SHELL := /bin/sh
+SHELL:=/bin/bash
 
 # Default compilation flags
-CFLAGS ?= -O2
-CFLAGS += -Wall -fPIC
-CPPFLAGS += -D_GNU_SOURCE
+CFLAGS?=-O2
+CFLAGS+=-Wall -fPIC
+CPPFLAGS+=-D_GNU_SOURCE
 
 # To turn on debug message logging
-#CPPFLAGS += -DUNVME_DEBUG
+#CPPFLAGS+=-DUNVME_DEBUG
 
 # To build the ioengine modules, specify fio directory where the header files
-#FIODIR := /opt/fio
+#FIODIR:=/opt/fio
 

README.md

@@ -68,18 +68,19 @@ UNVMe requires root privilege to access a device.
 Build, Run, and Test
 ====================
 
-To install the driver and library, run:
+To download and install the driver library, run:
 
+    $ git clone https://github.com/MicronSSD/unvme.git
     $ make install
 
 
-To set up for UNVMe driver usage (once before running applications), run:
+To setup a device for UNVMe usage (do once before running applications), run:
 
     $ unvme-setup bind
 
     By default, all NVMe devices found in the system will be bound to the
     VFIO driver enabling them for UNVMe usage.  Specific PCI device(s)
-    may also be specified for binding.
+    may also be specified for binding, e.g. unvme-setup bind 07:00.0.
 
 
 To reset device(s) to the NVMe kernel space driver, run:
@@ -112,7 +113,8 @@ To run fio benchmark tests against UNVMe:
     1) Download and compile the fio source code (available on https://github.com/axboe/fio).
 
 
-    2) Edit Makefile.def and set FIODIR to the compiled fio source directory.
+    2) Edit unvme/Makefile.def and set FIODIR to the compiled fio source
+       directory (or alternatively export the FIODIR variable).
 
 
     3) Rerun make to include building the fio engine, since setting FIODIR
@@ -189,6 +191,5 @@ As defined in unvme.h, the following functions are supported:
 
 Note that a user space filesystem, namely UNFS, has also been developed
 at Micron to work with the UNVMe driver.  Such available filesystem enables
-major applications like MongoDB to be ported to run on UNVMe driver.
+major applications like MongoDB to work with UNVMe driver.
 See https://github.com/MicronSSD/unfs.git for details.
-

src/unvme.c

@@ -111,18 +111,6 @@ int unvme_free(const unvme_ns_t* ns, void* buf)
     return unvme_do_free(ns, buf);
 }
 
-/**
- * Poll for completion status of a previous IO submission.
- * If there's no error, the descriptor will be released.
- * @param   iod         IO descriptor
- * @param   timeout     in seconds
- * @return  0 if ok else error status.
- */
-int unvme_apoll(unvme_iod_t iod, int timeout)
-{
-    return unvme_do_poll(iod, timeout);
-}
-
 /**
  * Read data from specified logical blocks on device.
  * @param   ns          namespace handle
@@ -152,6 +140,18 @@ unvme_iod_t unvme_awrite(const unvme_ns_t* ns, int qid,
     return unvme_rw(ns, qid, NVME_CMD_WRITE, (void*)buf, slba, nlb);
 }
 
+/**
+ * Poll for completion status of a previous IO submission.
+ * If there's no error, the descriptor will be released.
+ * @param   iod         IO descriptor
+ * @param   timeout     in seconds
+ * @return  0 if ok else error status.
+ */
+int unvme_apoll(unvme_iod_t iod, int timeout)
+{
+    return unvme_do_poll(iod, timeout);
+}
+
 /**
  * Read data from specified logical blocks on device.
  * @param   ns          namespace handle
@@ -164,7 +164,10 @@ unvme_iod_t unvme_awrite(const unvme_ns_t* ns, int qid,
 int unvme_read(const unvme_ns_t* ns, int qid, void* buf, u64 slba, u32 nlb)
 {
     unvme_desc_t* desc = unvme_rw(ns, qid, NVME_CMD_READ, buf, slba, nlb);
-    if (desc) return unvme_do_poll(desc, UNVME_TIMEOUT);
+    if (desc) {
+        sched_yield();
+        return unvme_do_poll(desc, UNVME_TIMEOUT);
+    }
     return -1;
 }
 
@@ -181,7 +184,10 @@ int unvme_write(const unvme_ns_t* ns, int qid,
                 const void* buf, u64 slba, u32 nlb)
 {
     unvme_desc_t* desc = unvme_rw(ns, qid, NVME_CMD_WRITE, (void*)buf, slba, nlb);
-    if (desc) return unvme_do_poll(desc, UNVME_TIMEOUT);
+    if (desc) {
+        sched_yield();
+        return unvme_do_poll(desc, UNVME_TIMEOUT);
+    }
     return -1;
 }
 

src/unvme.h

@@ -51,8 +51,8 @@ typedef uint32_t        u32;        ///< 32-bit unsigned
 typedef uint64_t        u64;        ///< 64-bit unsigned
 #endif // _U_TYPE
 
-#define UNVME_TIMEOUT   60          ///< default I/O timeout in seconds
-#define UNVME_QSIZE     65          ///< default I/O queue size
+#define UNVME_TIMEOUT   30          ///< default I/O timeout in seconds
+#define UNVME_QSIZE     256         ///< default I/O queue size
 
 /// Namespace attributes structure
 typedef struct _unvme_ns {

src/unvme_core.c

@@ -70,66 +70,13 @@
 /// Log and print an unrecoverable error message and exit
 #define FATAL(fmt, arg...)  do { ERROR(fmt, ##arg); abort(); } while (0)
 
-/// Fast lock write bit
-#define UNVME_LOCKW     0x10000000
-
 
 // Global static variables
 static const char*      unvme_log = "/dev/shm/unvme.log";   ///< Log filename
 static unvme_session_t* unvme_ses = NULL;                   ///< session list
 static unvme_lock_t     unvme_lock = 0;                     ///< session lock
 
 
-/**
- * Increment read lock and wait if pending write.
- * @param   lock    lock variable
- */
-static inline void unvme_lockr(unvme_lock_t* lock)
-{
-    for (;;) {
-        if (__sync_fetch_and_add(lock, 1) < UNVME_LOCKW) return;
-        __sync_fetch_and_sub(lock, 1);
-        sched_yield();
-    }
-}
-
-/**
- * Decrement read lock.
- * @param   lock    lock variable
- */
-static inline void unvme_unlockr(unvme_lock_t* lock)
-{
-    __sync_fetch_and_sub(lock, 1);
-}
-
-/**
- * Acquire write lock and wait for all pending read/write.
- * @param   lock    lock variable
- */
-static inline void unvme_lockw(unvme_lock_t* lock)
-{
-    for (;;) {
-        int val = __sync_fetch_and_or(lock, UNVME_LOCKW);
-        if (val == 0) return;
-        sched_yield();
-
-        // if not pending write then just wait for all reads to clear
-        if (val < UNVME_LOCKW) {
-            while (*lock != UNVME_LOCKW) sched_yield();
-            return;
-        }
-    }
-}
-
-/**
- * Clear the write lock.
- * @param   lock    lock variable
- */
-static inline void unvme_unlockw(unvme_lock_t* lock)
-{
-    __sync_fetch_and_and(lock, ~UNVME_LOCKW);
-}
-
 /**
  * Get a descriptor entry by moving from the free to the use list.
  * @param   ioq     IO queue context
@@ -223,7 +170,7 @@ static int unvme_complete_io(unvme_ioq_t* ioq, int timeout)
     if (ioq->cidcount) {
         while (ioq->descnext->cidcount == 0) ioq->descnext = ioq->descnext->next;
     }
-    PDEBUG("# c q%d={%d %d %#lx} @%d={%d %#lx} @%d",
+    PDEBUG("# c q%d={%d %d %#lx} d={%d %d %#lx} @%d",
            ioq->nvmeq.id, cid, ioq->cidcount, *ioq->cidmask,
            desc->id, desc->cidcount, *desc->cidmask, ioq->descnext->id);
     return err;
@@ -326,7 +273,7 @@ static int unvme_submit_io(const unvme_ns_t* ns, unvme_desc_t* desc,
         ioq->cidcount++;
         desc->cidmask[b] |= mask;
         desc->cidcount++;
-        PDEBUG("# %c %#lx %#x q%d={%d %d %#lx} @%d={%d %#lx}",
+        PDEBUG("# %c %#lx %#x q%d={%d %d %#lx} d={%d %d %#lx}",
                desc->opc == NVME_CMD_READ ? 'r' : 'w', slba, nlb,
                ioq->nvmeq.id, cid, ioq->cidcount, *ioq->cidmask,
                desc->id, desc->cidcount, *desc->cidmask);
@@ -493,7 +440,7 @@ unvme_ns_t* unvme_do_open(int pci, int nsid, int qcount, int qsize)
     while (xses) {
         if (xses->ns.pci == pci) {
             if (nsid > xses->dev->nscount) {
-                ERROR("invalid %06x nsid %d", pci, nsid);
+                ERROR("invalid %06x nsid %d (max %d)", pci, nsid, xses->dev->nscount);
                 return NULL;
             }
             if (xses->ns.id == nsid) {
@@ -523,7 +470,7 @@ unvme_ns_t* unvme_do_open(int pci, int nsid, int qcount, int qsize)
         nvme_identify_ctlr_t* idc = (nvme_identify_ctlr_t*)dma->buf;
         dev->nscount = idc->nn;
         if (nsid > dev->nscount) {
-            ERROR("invalid %06x nsid %d", pci, nsid);
+            ERROR("invalid %06x nsid %d (max %d)", pci, nsid, dev->nscount);
             return NULL;
         }
 
@@ -661,7 +608,7 @@ int unvme_do_free(const unvme_ns_t* ns, void* buf)
  */
 int unvme_do_poll(unvme_desc_t* desc, int timeout)
 {
-    PDEBUG("# POLL @%d={%d %#lx}", desc->id, desc->cidcount, *desc->cidmask);
+    PDEBUG("# POLL d={%d %d %#lx}", desc->id, desc->cidcount, *desc->cidmask);
     int err = 0;
     while (desc->cidcount) {
         if ((err = unvme_complete_io(desc->ioq, timeout)) != 0) break;
@@ -680,7 +627,7 @@ int unvme_do_poll(unvme_desc_t* desc, int timeout)
  * @param   buf         data buffer
  * @param   slba        starting lba
  * @param   nlb         number of logical blocks
- * @return  0 if ok else error status.
+ * @return  I/O descriptor or NULL if error.
  */
 unvme_desc_t* unvme_rw(const unvme_ns_t* ns, int qid, int opc,
                        void* buf, u64 slba, u32 nlb)

src/unvme_core.h

@@ -42,12 +42,10 @@
 #include "unvme_log.h"
 #include "unvme_vfio.h"
 #include "unvme_nvme.h"
+#include "unvme_lock.h"
 #include "unvme.h"
 
 
-/// Simple read write lock
-typedef int unvme_lock_t;
-
 /// IO memory allocation tracking info
 typedef struct _unvme_iomem {
     vfio_dma_t**            map;        ///< dynamic array of allocated memory

src/unvme_lock.h

@@ -0,0 +1,95 @@
+/**
+ * Copyright (c) 2015-2016, Micron Technology, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *   1. Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *
+ *   2. Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *
+ *   3. Neither the name of the copyright holder nor the names of its
+ *      contributors may be used to endorse or promote products derived
+ *      from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/**
+ * @file
+ * @brief UNVMe fast read lock with occasional writes.
+ */
+
+#include <sched.h>
+
+/// Lock write bit
+#define UNVME_LOCKWBIT      0x80000000
+
+/// Simple read write lock
+typedef unsigned unvme_lock_t;
+
+
+/**
+ * Increment read lock and wait if pending write.
+ * @param   lock    lock variable
+ */
+static inline void unvme_lockr(unvme_lock_t* lock)
+{
+    for (;;) {
+        if (!(__sync_fetch_and_add(lock, 1) & UNVME_LOCKWBIT)) return;
+        __sync_fetch_and_sub(lock, 1);
+        sched_yield();
+    }
+}
+
+/**
+ * Decrement read lock.
+ * @param   lock    lock variable
+ */
+static inline void unvme_unlockr(unvme_lock_t* lock)
+{
+    __sync_fetch_and_sub(lock, 1);
+}
+
+/**
+ * Acquire write lock and wait for all pending read/write.
+ * @param   lock    lock variable
+ */
+static inline void unvme_lockw(unvme_lock_t* lock)
+{
+    for (;;) {
+        int val = __sync_fetch_and_or(lock, UNVME_LOCKWBIT);
+        if (val == 0) return;
+        sched_yield();
+
+        // if not pending write then just wait for all reads to clear
+        if (!(val & UNVME_LOCKWBIT)) {
+            while (*lock != UNVME_LOCKWBIT) sched_yield();
+            return;
+        }
+    }
+}
+
+/**
+ * Clear the write lock.
+ * @param   lock    lock variable
+ */
+static inline void unvme_unlockw(unvme_lock_t* lock)
+{
+    __sync_fetch_and_and(lock, ~UNVME_LOCKWBIT);
+}
+

src/unvme_nvme.c

@@ -466,8 +466,9 @@ int nvme_cmd_rw(nvme_queue_t* ioq, int opc, u16 cid, int nsid,
     cmd->common.prp2 = prp2;
     cmd->slba = slba;
     cmd->nlb = nlb - 1;
-    DEBUG_FN("q=%d t=%d cid=%#x nsid=%d lba=%#lx nb=%d (%c)", ioq->id,
-             ioq->sq_tail, cid, nsid, slba, nlb, opc == NVME_CMD_READ? 'R' : 'W');
+    DEBUG_FN("q=%d t=%d h=%d cid=%#x nsid=%d lba=%#lx nb=%d (%c)",
+             ioq->id, ioq->sq_tail, ioq->sq_head, cid, nsid, slba, nlb,
+             opc == NVME_CMD_READ? 'R' : 'W');
     return nvme_submit_cmd(ioq);
 }
 

test/unvme/Makefile

@@ -32,7 +32,7 @@
 include ../../Makefile.def
 
 TARGETS = unvme_sim_test unvme_api_test unvme_mts_test unvme_lat_test \
-          unvme_mcd_test
+          unvme_mcd_test unvme_info
 
 UNVME_SRC = ../../src