Support both dmabuf and p2p in device tests

NETCASSINI-6991

Signed-off-by: Chuck Fossen <charles.fossen@hpe.com>

tests/libcxi_gpu_cuda.c

@@ -97,6 +97,18 @@ static int c_memcpy(void *dst, const void *src, size_t count,
 	return 0;
 }
 
+static int c_gpu_props(struct mem_window *win, void **base, size_t *size)
+{
+	win->hints.dmabuf_valid = false;
+
+	return 0;
+}
+
+static int c_gpu_close_fd(int fd)
+{
+	return 0;
+}
+
 int cuda_lib_init(void)
 {
 	cudaError_t ret;
@@ -146,6 +158,8 @@ int cuda_lib_init(void)
 	gpu_host_free = c_host_free;
 	gpu_memset = c_memset;
 	gpu_memcpy = c_memcpy;
+	gpu_props = c_gpu_props;
+	gpu_close_fd = c_gpu_close_fd;
 
 	printf("Found NVIDIA GPU\n");
 
@@ -163,6 +177,8 @@ void cuda_lib_fini(void)
 	gpu_host_free = NULL;
 	gpu_memset = NULL;
 	gpu_memcpy = NULL;
+	gpu_props = NULL;
+	gpu_close_fd = NULL;
 	dlclose(libcuda_handle);
 	dlclose(libcudart_handle);
 

tests/libcxi_gpu_hip.c

@@ -97,17 +97,6 @@ static int h_memcpy(void *dst, const void *src, size_t count,
 	return 0;
 }
 
-int h_get_dmabuf_fd(const void *addr, size_t size, int *dmabuf_fd,
-		    uint64_t *offset)
-{
-	hsa_status_t rc;
-
-	rc = hsa_get_dmabuf(addr, size, dmabuf_fd, offset);
-	cr_assert_eq(rc, HSA_STATUS_SUCCESS, "hsa_get_dmabuf() failed %d", rc);
-
-	return 0;
-}
-
 int h_put_dmabuf_fd(int dmabuf_fd)
 {
 	hsa_status_t rc;
@@ -118,24 +107,31 @@ int h_put_dmabuf_fd(int dmabuf_fd)
 	return 0;
 }
 
-__attribute__((unused))
-static int h_mem_props(const void *addr, void **base, size_t *size,
-		       int *dmabuf_fd, uint64_t *offset)
+static int h_mem_props(struct mem_window *win, void **base, size_t *size)
 {
-	hsa_status_t ret;
+	hsa_status_t rc;
 	hsa_amd_pointer_info_t info = {
 		.size = sizeof(info),
 	};
 
-	ret = hsa_ptr_info((void *)addr, &info, NULL, NULL, NULL);
-	cr_assert_eq(ret, HSA_STATUS_SUCCESS, "hsa_amd_ptr_info() failed %d",
-		     ret);
+	if (!win->use_dmabuf) {
+		win->hints.dmabuf_valid = false;
+		return 0;
+	}
+
+	rc = hsa_ptr_info(win->buffer, &info, NULL, NULL, NULL);
+	cr_assert_eq(rc, HSA_STATUS_SUCCESS, "hsa_amd_ptr_info() error %d", rc);
 	cr_assert_eq(info.type, HSA_EXT_POINTER_TYPE_HSA,
 		      "hsa_amd_ptr_info() not HSA");
+
 	*size = info.sizeInBytes;
 	*base = info.agentBaseAddress;
 
-	h_get_dmabuf_fd(addr, *size, dmabuf_fd, offset);
+	rc = hsa_get_dmabuf(win->buffer, *size, &win->hints.dmabuf_fd,
+			    &win->hints.dmabuf_offset);
+	cr_assert_eq(rc, HSA_STATUS_SUCCESS, "hsa_get_dmabuf() failed %d", rc);
+
+	win->hints.dmabuf_valid = true;
 
 	return 0;
 }
@@ -194,6 +190,8 @@ int hip_lib_init(void)
 	gpu_host_free = h_host_free;
 	gpu_memset = h_memset;
 	gpu_memcpy = h_memcpy;
+	gpu_props = h_mem_props;
+	gpu_close_fd = h_put_dmabuf_fd;
 
 	printf("Found AMD GPU\n");
 
@@ -210,6 +208,8 @@ void hip_lib_fini(void)
 	gpu_host_alloc = NULL;
 	gpu_memset = NULL;
 	gpu_memcpy = NULL;
+	gpu_props = NULL;
+	gpu_close_fd = NULL;
 
 	dlclose(hsa_handle);
 	hsa_handle = NULL;

tests/libcxi_gpu_ze.c

@@ -401,37 +401,44 @@ static const char *const alloc_types[] = {
 	[ZE_MEMORY_TYPE_SHARED] = "shared",
 };
 
-static int ze_mem_props(const void *addr, void **base, size_t *size,
-			int *dma_buf_fd)
+static int ze_mem_props(struct mem_window *win, void **base, size_t *size)
 {
 	void *handle;
 	ze_result_t ze_ret;
 	ze_device_handle_t device_ptr;
 	ze_memory_allocation_properties_t alloc_props = {};
 
-	ze_ret = cxi_zeMemGetIpcHandle(context, addr,
+	ze_ret = cxi_zeMemGetIpcHandle(context, win->buffer,
 				       (ze_ipc_mem_handle_t *)&handle);
 	if (ze_ret != ZE_RESULT_SUCCESS) {
 		ZE_ERR("Failed to get IPC handle: %d\n", ze_ret);
 		return -EIO;
 	}
 
-	*dma_buf_fd = (int)(uintptr_t)handle;
+	win->hints.dmabuf_fd = (int)(uintptr_t)handle;
 
-	ze_ret = cxi_zeMemGetAddressRange(context, addr, base, size);
+	ze_ret = cxi_zeMemGetAddressRange(context, win->buffer, base, size);
 	if (ze_ret != ZE_RESULT_SUCCESS) {
 		ZE_ERR("Failed to get address range: %d\n", ze_ret);
 		return -EIO;
 	}
 
-	ze_ret = cxi_zeMemGetAllocProperties(context, addr, &alloc_props,
+	ze_ret = cxi_zeMemGetAllocProperties(context, win->buffer, &alloc_props,
 					     &device_ptr);
 	if (ze_ret != ZE_RESULT_SUCCESS)
 		ZE_ERR("Failed to get AllocProperties:%d\n", ze_ret);
 	else if (0)
 		printf("alloc type:%d (%s)\n", alloc_props.type,
 		       alloc_types[alloc_props.type]);
 
+	win->hints.dmabuf_offset = (uintptr_t)win->buffer - (uintptr_t)*base;
+	win->hints.dmabuf_valid = true;
+
+	return 0;
+}
+
+static int ze_close_fd(int fd)
+{
 	return 0;
 }
 
@@ -498,6 +505,7 @@ int ze_init(void)
 	gpu_memset = ze_memset;
 	gpu_memcpy = ze_memcpy;
 	gpu_props = ze_mem_props;
+	gpu_close_fd = ze_close_fd;
 
 	return 0;
 

tests/libcxi_test_common.c

@@ -53,7 +53,8 @@ int (*gpu_host_free)(void *p);
 int (*gpu_memset)(void *devPtr, int value, size_t count);
 int (*gpu_memcpy)(void *dst, const void *src, size_t count,
 		  enum gpu_copy_dir dir);
-int (*gpu_props)(const void *addr, void **base, size_t *size, int *dma_buf_fd);
+int (*gpu_props)(struct mem_window *win, void **base, size_t *size);
+int (*gpu_close_fd)(int dma_buf_fd);
 
 int s_page_size;
 
@@ -691,37 +692,27 @@ void memset_device(struct mem_window *win, int value, size_t count)
 	cr_assert_eq(rc, 0, "gpu_memset() failed %d", rc);
 }
 
-void map_devicebuf(struct mem_window *win, uint32_t prot)
+static void map_devicebuf(struct mem_window *win, uint32_t prot)
 {
 	int rc;
-	struct cxi_md_hints hints = {};
 	void *base_addr;
 	size_t size;
-	int dma_buf_fd = 0;
 
-	hints.dmabuf_fd = dma_buf_fd;
 	prot &= (CXI_MAP_WRITE | CXI_MAP_READ);
 
 	memset(&win->md, 0, sizeof(win->md));
 	base_addr = win->buffer;
 	size = win->length;
 
-	if (gpu_props) {
-		rc = gpu_props(win->buffer, &base_addr, &size, &dma_buf_fd);
-		cr_assert(rc == 0 || rc == -ENOSYS);
-	}
-
-	if (dma_buf_fd) {
-		hints.dmabuf_fd = dma_buf_fd;
-		hints.dmabuf_valid = true;
-	}
+	rc = gpu_props(win, &base_addr, &size);
+	cr_assert_eq(rc, 0, "gpu_props failed\n");
 
 	if (win->is_device)
 		prot |= CXI_MAP_DEVICE;
 
 	rc = cxil_map(lni, base_addr, size,
 		      CXI_MAP_PIN | prot,
-		      &hints, &win->md);
+		      &win->hints, &win->md);
 	cr_assert_eq(rc, 0, "cxil_map() failed %d", rc);
 }
 
@@ -753,6 +744,9 @@ void free_unmap_devicebuf(struct mem_window *win)
 
 	cr_expect_eq(rc, 0, "cxil_unmap() failed %d", rc);
 
+	if (win->hints.dmabuf_valid)
+		gpu_close_fd(win->hints.dmabuf_fd);
+
 	if (win->loc == on_device)
 		gpu_free(win->buffer);
 	else
@@ -934,6 +928,18 @@ bool is_vm(void) {
 	return hypervisor_count > 0;
 }
 
+static int no_gpu_props(struct mem_window *win, void **base, size_t *size)
+{
+	win->hints.dmabuf_valid = false;
+
+	return 0;
+}
+
+static int no_gpu_close_fd(int fd)
+{
+	return 0;
+}
+
 int gpu_lib_init(void)
 {
 	int rc = -1;
@@ -964,6 +970,8 @@ int gpu_lib_init(void)
 	gpu_host_free = NULL;
 	gpu_memset = NULL;
 	gpu_memcpy = NULL;
+	gpu_props = no_gpu_props;
+	gpu_close_fd = no_gpu_close_fd;
 
 	return rc;
 }

tests/libcxi_test_common.h

@@ -47,8 +47,10 @@ struct mem_window {
 	size_t length;
 	uint8_t *buffer;
 	struct cxi_md *md;
+	struct cxi_md_hints hints;
 	enum buf_location loc;
 	bool is_device;
+	bool use_dmabuf;
 };
 
 extern uint32_t dev_id;
@@ -193,8 +195,8 @@ extern int (*gpu_host_free)(void *p);
 extern int (*gpu_memset)(void *devPtr, int value, size_t count);
 extern int (*gpu_memcpy)(void *dst, const void *src, size_t count,
 			 enum gpu_copy_dir dir);
-extern int (*gpu_props)(const void *addr, void **base, size_t *size,
-			int *dma_buf_fd);
+extern int (*gpu_props)(struct mem_window *win, void **base, size_t *size);
+extern int (*gpu_close_fd)(int dma_buf_fd);
 
 #define ONE_GB (1024*1024*1024)
 #define TWO_MB (2*1024*1024)