iiod: Support zero-copy to USB

Add support for passing sample data as DMABUF objects between IIO
devices, and the USB stack.

This mechanism has the benefit that the CPU will never access the data
to copy it from one hardware buffer to another, and therefore results in
a much higher throughput at a much lower CPU usage.

For this new mechanism to work, the DMABUF-based IIO kernel API must be
available and used by the local backend, and the FunctionFS stack must
also support importing DMABUFs. If those conditions are not met, the
standard way of transferring the data will be used.

Signed-off-by: Paul Cercueil <paul@crapouillou.net>

iio-config.h.cmakein

@@ -31,6 +31,7 @@
 #cmakedefine01 WITH_IIOD_NETWORK
 #cmakedefine01 WITH_IIOD_USBD
 #cmakedefine01 WITH_IIOD_SERIAL
+#cmakedefine01 WITH_IIOD_USB_DMABUF
 #cmakedefine01 WITH_IIOD_V0_COMPAT
 #cmakedefine01 WITH_LOCAL_CONFIG
 #cmakedefine01 WITH_LOCAL_DMABUF_API

iiod/CMakeLists.txt

@@ -68,6 +68,12 @@ if (WITH_AIO)
 		endif()
 
 		target_sources(iiod PRIVATE usbd.c)
+
+
+		option(WITH_IIOD_USB_DMABUF "Enable DMABUF support on the USB stack" OFF)
+		if (WITH_IIOD_USB_DMABUF)
+			target_sources(iiod PRIVATE usb-dmabuf.c)
+		endif()
 	endif()
 
 	target_include_directories(iiod PRIVATE ${LIBAIO_INCLUDE_DIR})

iiod/ops.h

@@ -47,6 +47,7 @@
 
 struct iio_task;
 struct iiod_io;
+struct parser_pdata;
 struct thread_pool;
 extern struct thread_pool *main_thread_pool;
 struct DevEntry;
@@ -64,6 +65,7 @@ struct block_entry {
 	uint64_t bytes_used;
 	uint16_t client_id;
 	bool cyclic;
+	int dmabuf_fd;
 };
 
 struct buffer_entry {
@@ -74,6 +76,7 @@ struct buffer_entry {
 	struct iio_task *enqueue_task, *dequeue_task;
 	uint32_t *words;
 	uint16_t idx;
+	bool is_tx;
 
 	SLIST_HEAD(BlockList, block_entry) blocklist;
 	pthread_mutex_t lock;
@@ -147,6 +150,10 @@ int start_network_daemon(struct iio_context *ctx,
 			 struct thread_pool *pool, const void *xml_zstd,
 			 size_t xml_zstd_len, uint16_t port);
 
+int usb_attach_dmabuf(int ep_fd, int fd);
+int usb_detach_dmabuf(int ep_fd, int fd);
+int usb_transfer_dmabuf(int ep_fd, int fd, uint64_t size);
+
 int binary_parse(struct parser_pdata *pdata);
 
 void enable_binary(struct parser_pdata *pdata);

iiod/responder.c

@@ -15,6 +15,7 @@
 #include <fcntl.h>
 #include <iio/iio-lock.h>
 #include <pthread.h>
+#include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
 #include <unistd.h>
@@ -319,7 +320,17 @@ static int buffer_dequeue_block(void *priv, void *d)
 	if (ret < 0)
 		goto out_send_response;
 
-	if (!iio_buffer_is_tx(buffer->buf)) {
+	if (!buffer->is_tx) {
+		if (WITH_IIOD_USB_DMABUF && entry->dmabuf_fd > 0) {
+			/* We need to send the error code before the data.
+			 * If usb_transfer_dmabuf() fails, we're screwed... */
+			iiod_io_send_response_code(entry->io, entry->bytes_used);
+
+			return usb_transfer_dmabuf(buffer->pdata->fd_out,
+						   entry->dmabuf_fd,
+						   entry->bytes_used);
+		}
+
 		data.ptr = iio_block_start(entry->block);
 		data.size = iio_block_end(entry->block) - data.ptr;
 		nb_data++;
@@ -358,6 +369,8 @@ static void handle_create_buffer(struct parser_pdata *pdata,
 		goto err_send_response;
 	}
 
+	entry->pdata = pdata;
+
 	nb_channels = iio_device_get_channels_count(dev);
 	nb_words = (nb_channels + 31) / 32;
 
@@ -423,6 +436,8 @@ static void handle_create_buffer(struct parser_pdata *pdata,
 			entry->words[BIT_WORD(i)] &= ~BIT_MASK(i);
 	}
 
+	entry->is_tx = iio_buffer_is_tx(buf);
+
 	/* Success, destroy the temporary mask object */
 	iio_channels_mask_destroy(mask);
 
@@ -605,7 +620,7 @@ static void handle_create_block(struct parser_pdata *pdata,
 	struct iiod_buf data;
 	uint64_t block_size;
 	struct iiod_io *io;
-	int ret;
+	int ret, ep_fd;
 
 	io = iiod_command_create_io(cmd, cmd_data);
 	ret = iio_err(io);
@@ -651,6 +666,24 @@ static void handle_create_block(struct parser_pdata *pdata,
 	entry->io = io;
 	entry->client_id = cmd->client_id;
 
+	if (WITH_IIOD_USB_DMABUF && pdata->is_usb) {
+		entry->dmabuf_fd = iio_block_get_dmabuf_fd(block);
+		if (entry->dmabuf_fd > 0) {
+			ep_fd = buf_entry->is_tx ? pdata->fd_in : pdata->fd_out;
+
+			ret = usb_attach_dmabuf(ep_fd, entry->dmabuf_fd);
+			if (!ret) {
+				/* We could attach to functionfs. Disable CPU
+				 * access to the block as we won't need it. */
+				iio_block_disable_cpu_access(block, true);
+			} else {
+				/* If we can't attach - no problem. The
+				 * data will be transferred the regular way. */
+				entry->dmabuf_fd = -ENOSYS;
+			}
+		}
+	}
+
 	/* Keep a reference to the iiod_io until the block is freed. */
 	iiod_io_ref(io);
 
@@ -672,7 +705,7 @@ static void handle_free_block(struct parser_pdata *pdata,
 	struct iio_buffer *buf;
 	struct iio_block *block;
 	struct iiod_io *io;
-	int ret;
+	int ret, ep_fd;
 
 	buf = get_iio_buffer(pdata, cmd, &buf_entry);
 	ret = iio_err(buf);
@@ -692,6 +725,10 @@ static void handle_free_block(struct parser_pdata *pdata,
 		if (entry->block != block)
 			continue;
 
+		ep_fd = buf_entry->is_tx ? pdata->fd_in : pdata->fd_out;
+		if (WITH_IIOD_USB_DMABUF && entry->dmabuf_fd > 0)
+			usb_detach_dmabuf(ep_fd, entry->dmabuf_fd);
+
 		SLIST_REMOVE(&buf_entry->blocklist, entry, block_entry, entry);
 
 		free_block_entry(entry);
@@ -756,13 +793,21 @@ static void handle_transfer_block(struct parser_pdata *pdata,
 	}
 
 	/* Read the data into the block if we are dealing with a TX buffer */
-	if (iio_buffer_is_tx(buf)) {
-		readbuf.ptr = iio_block_start(block);
-		readbuf.size = iio_block_end(block) - readbuf.ptr;
-
-		ret = iiod_command_data_read(cmd_data, &readbuf);
-		if (ret < 0)
-			goto out_send_response;
+	if (entry->is_tx) {
+		if (WITH_IIOD_USB_DMABUF && block_entry->dmabuf_fd > 0) {
+			ret = usb_transfer_dmabuf(pdata->fd_in,
+						  block_entry->dmabuf_fd,
+						  bytes_used);
+			if (ret)
+				goto out_send_response;
+		} else {
+			readbuf.ptr = iio_block_start(block);
+			readbuf.size = iio_block_end(block) - readbuf.ptr;
+
+			ret = iiod_command_data_read(cmd_data, &readbuf);
+			if (ret < 0)
+				goto out_send_response;
+		}
 	}
 
 	block_entry->bytes_used = bytes_used;

iiod/usb-dmabuf.c

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: LGPL-2.1-or-later
+/*
+ * libiio - Library for interfacing industrial I/O (IIO) devices
+ *
+ * Copyright (C) 2023 Analog Devices, Inc.
+ * Author: Paul Cercueil <paul.cercueil@analog.com>
+ */
+
+#include <errno.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <sys/ioctl.h>
+
+#define IIO_FFS_DMABUF_ATTACH	_IOW('g', 131, int)
+#define IIO_FFS_DMABUF_DETACH	_IOW('g', 132, int)
+#define IIO_FFS_DMABUF_TRANSFER	_IOW('g', 133, struct iio_ffs_dmabuf_transfer)
+
+struct iio_ffs_dmabuf_transfer {
+	int fd;
+	uint32_t flags;
+	uint64_t length;
+};
+
+int usb_attach_dmabuf(int ep_fd, int fd)
+{
+	int ret;
+
+	ret = ioctl(ep_fd, IIO_FFS_DMABUF_ATTACH, &fd);
+	if (ret == -1)
+		return -errno;
+
+	return 0;
+}
+
+int usb_detach_dmabuf(int ep_fd, int fd)
+{
+	int ret;
+
+	ret = ioctl(ep_fd, IIO_FFS_DMABUF_DETACH, &fd);
+	if (ret == -1)
+		return -errno;
+
+	return 0;
+}
+
+int usb_transfer_dmabuf(int ep_fd, int fd, uint64_t size)
+{
+	struct iio_ffs_dmabuf_transfer req;
+	int ret;
+
+	req.fd = fd;
+	req.length = size;
+	req.flags = 0;
+
+	ret = ioctl(ep_fd, IIO_FFS_DMABUF_TRANSFER, &req);
+	if (ret == -1)
+		return -errno;
+
+	return 0;
+}