From 2707d58c9ae77dd341b1f57bcfab329d44724e35 Mon Sep 17 00:00:00 2001 From: Paul Cercueil Date: Fri, 25 Nov 2022 16:21:38 +0000 Subject: [PATCH] iiod: Support zero-copy to USB Add support for passing sample data as DMABUF objects between IIO devices, and the USB stack. This mechanism has the benefit that the CPU will never access the data to copy it from one hardware buffer to another, and therefore results in a much higher throughput at a much lower CPU usage. For this new mechanism to work, the DMABUF-based IIO kernel API must be available and used by the local backend, and the FunctionFS stack must also support importing DMABUFs. If those conditions are not met, the standard way of transferring the data will be used. Signed-off-by: Paul Cercueil --- iio-config.h.cmakein | 1 + iiod/CMakeLists.txt | 6 ++++ iiod/ops.h | 7 +++++ iiod/responder.c | 65 +++++++++++++++++++++++++++++++++++++------- iiod/usb-dmabuf.c | 60 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 10 deletions(-) create mode 100644 iiod/usb-dmabuf.c diff --git a/iio-config.h.cmakein b/iio-config.h.cmakein index 2af5377ec..bc8b72d56 100644 --- a/iio-config.h.cmakein +++ b/iio-config.h.cmakein @@ -31,6 +31,7 @@ #cmakedefine01 WITH_IIOD_NETWORK #cmakedefine01 WITH_IIOD_USBD #cmakedefine01 WITH_IIOD_SERIAL +#cmakedefine01 WITH_IIOD_USB_DMABUF #cmakedefine01 WITH_IIOD_V0_COMPAT #cmakedefine01 WITH_LOCAL_CONFIG #cmakedefine01 WITH_LOCAL_DMABUF_API diff --git a/iiod/CMakeLists.txt b/iiod/CMakeLists.txt index 7eb148573..8ceaf1b40 100644 --- a/iiod/CMakeLists.txt +++ b/iiod/CMakeLists.txt @@ -68,6 +68,12 @@ if (WITH_AIO) endif() target_sources(iiod PRIVATE usbd.c) + + + option(WITH_IIOD_USB_DMABUF "Enable DMABUF support on the USB stack" OFF) + if (WITH_IIOD_USB_DMABUF) + target_sources(iiod PRIVATE usb-dmabuf.c) + endif() endif() target_include_directories(iiod PRIVATE ${LIBAIO_INCLUDE_DIR}) diff --git a/iiod/ops.h b/iiod/ops.h index 11011e6e2..f0733ca22 100644 --- a/iiod/ops.h +++ b/iiod/ops.h @@ -47,6 +47,7 @@ struct iio_task; struct iiod_io; +struct parser_pdata; struct thread_pool; extern struct thread_pool *main_thread_pool; struct DevEntry; @@ -64,6 +65,7 @@ struct block_entry { uint64_t bytes_used; uint16_t client_id; bool cyclic; + int dmabuf_fd; }; struct buffer_entry { @@ -74,6 +76,7 @@ struct buffer_entry { struct iio_task *enqueue_task, *dequeue_task; uint32_t *words; uint16_t idx; + bool is_tx; SLIST_HEAD(BlockList, block_entry) blocklist; pthread_mutex_t lock; @@ -147,6 +150,10 @@ int start_network_daemon(struct iio_context *ctx, struct thread_pool *pool, const void *xml_zstd, size_t xml_zstd_len, uint16_t port); +int usb_attach_dmabuf(int ep_fd, int fd); +int usb_detach_dmabuf(int ep_fd, int fd); +int usb_transfer_dmabuf(int ep_fd, int fd, uint64_t size); + int binary_parse(struct parser_pdata *pdata); void enable_binary(struct parser_pdata *pdata); diff --git a/iiod/responder.c b/iiod/responder.c index 621d08ce1..7c06c5ea3 100644 --- a/iiod/responder.c +++ b/iiod/responder.c @@ -15,6 +15,7 @@ #include #include #include +#include #include #include #include @@ -319,7 +320,17 @@ static int buffer_dequeue_block(void *priv, void *d) if (ret < 0) goto out_send_response; - if (!iio_buffer_is_tx(buffer->buf)) { + if (!buffer->is_tx) { + if (WITH_IIOD_USB_DMABUF && entry->dmabuf_fd > 0) { + /* We need to send the error code before the data. + * If usb_transfer_dmabuf() fails, we're screwed... */ + iiod_io_send_response_code(entry->io, entry->bytes_used); + + return usb_transfer_dmabuf(buffer->pdata->fd_out, + entry->dmabuf_fd, + entry->bytes_used); + } + data.ptr = iio_block_start(entry->block); data.size = iio_block_end(entry->block) - data.ptr; nb_data++; @@ -358,6 +369,8 @@ static void handle_create_buffer(struct parser_pdata *pdata, goto err_send_response; } + entry->pdata = pdata; + nb_channels = iio_device_get_channels_count(dev); nb_words = (nb_channels + 31) / 32; @@ -423,6 +436,8 @@ static void handle_create_buffer(struct parser_pdata *pdata, entry->words[BIT_WORD(i)] &= ~BIT_MASK(i); } + entry->is_tx = iio_buffer_is_tx(buf); + /* Success, destroy the temporary mask object */ iio_channels_mask_destroy(mask); @@ -605,7 +620,7 @@ static void handle_create_block(struct parser_pdata *pdata, struct iiod_buf data; uint64_t block_size; struct iiod_io *io; - int ret; + int ret, ep_fd; io = iiod_command_create_io(cmd, cmd_data); ret = iio_err(io); @@ -651,6 +666,24 @@ static void handle_create_block(struct parser_pdata *pdata, entry->io = io; entry->client_id = cmd->client_id; + if (WITH_IIOD_USB_DMABUF && pdata->is_usb) { + entry->dmabuf_fd = iio_block_get_dmabuf_fd(block); + if (entry->dmabuf_fd > 0) { + ep_fd = buf_entry->is_tx ? pdata->fd_in : pdata->fd_out; + + ret = usb_attach_dmabuf(ep_fd, entry->dmabuf_fd); + if (!ret) { + /* We could attach to functionfs. Disable CPU + * access to the block as we won't need it. */ + iio_block_disable_cpu_access(block, true); + } else { + /* If we can't attach - no problem. The + * data will be transferred the regular way. */ + entry->dmabuf_fd = -ENOSYS; + } + } + } + /* Keep a reference to the iiod_io until the block is freed. */ iiod_io_ref(io); @@ -672,7 +705,7 @@ static void handle_free_block(struct parser_pdata *pdata, struct iio_buffer *buf; struct iio_block *block; struct iiod_io *io; - int ret; + int ret, ep_fd; buf = get_iio_buffer(pdata, cmd, &buf_entry); ret = iio_err(buf); @@ -692,6 +725,10 @@ static void handle_free_block(struct parser_pdata *pdata, if (entry->block != block) continue; + ep_fd = buf_entry->is_tx ? pdata->fd_in : pdata->fd_out; + if (WITH_IIOD_USB_DMABUF && entry->dmabuf_fd > 0) + usb_detach_dmabuf(ep_fd, entry->dmabuf_fd); + SLIST_REMOVE(&buf_entry->blocklist, entry, block_entry, entry); free_block_entry(entry); @@ -756,13 +793,21 @@ static void handle_transfer_block(struct parser_pdata *pdata, } /* Read the data into the block if we are dealing with a TX buffer */ - if (iio_buffer_is_tx(buf)) { - readbuf.ptr = iio_block_start(block); - readbuf.size = iio_block_end(block) - readbuf.ptr; - - ret = iiod_command_data_read(cmd_data, &readbuf); - if (ret < 0) - goto out_send_response; + if (entry->is_tx) { + if (WITH_IIOD_USB_DMABUF && block_entry->dmabuf_fd > 0) { + ret = usb_transfer_dmabuf(pdata->fd_in, + block_entry->dmabuf_fd, + bytes_used); + if (ret) + goto out_send_response; + } else { + readbuf.ptr = iio_block_start(block); + readbuf.size = iio_block_end(block) - readbuf.ptr; + + ret = iiod_command_data_read(cmd_data, &readbuf); + if (ret < 0) + goto out_send_response; + } } block_entry->bytes_used = bytes_used; diff --git a/iiod/usb-dmabuf.c b/iiod/usb-dmabuf.c new file mode 100644 index 000000000..36dea651d --- /dev/null +++ b/iiod/usb-dmabuf.c @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: LGPL-2.1-or-later +/* + * libiio - Library for interfacing industrial I/O (IIO) devices + * + * Copyright (C) 2023 Analog Devices, Inc. + * Author: Paul Cercueil + */ + +#include +#include +#include +#include + +#define IIO_FFS_DMABUF_ATTACH _IOW('g', 131, int) +#define IIO_FFS_DMABUF_DETACH _IOW('g', 132, int) +#define IIO_FFS_DMABUF_TRANSFER _IOW('g', 133, struct iio_ffs_dmabuf_transfer) + +struct iio_ffs_dmabuf_transfer { + int fd; + uint32_t flags; + uint64_t length; +}; + +int usb_attach_dmabuf(int ep_fd, int fd) +{ + int ret; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_ATTACH, &fd); + if (ret == -1) + return -errno; + + return 0; +} + +int usb_detach_dmabuf(int ep_fd, int fd) +{ + int ret; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_DETACH, &fd); + if (ret == -1) + return -errno; + + return 0; +} + +int usb_transfer_dmabuf(int ep_fd, int fd, uint64_t size) +{ + struct iio_ffs_dmabuf_transfer req; + int ret; + + req.fd = fd; + req.length = size; + req.flags = 0; + + ret = ioctl(ep_fd, IIO_FFS_DMABUF_TRANSFER, &req); + if (ret == -1) + return -errno; + + return 0; +}