From dd6fb1fead90a8daaab44638f2abc0e7a83e4633 Mon Sep 17 00:00:00 2001 From: Vamsi Attunuru Date: Fri, 17 Jan 2025 11:01:07 +0530 Subject: [PATCH] lib/vfio: rename vfio platform Patch renames vfio_platform into vfio to make it use with both platform and pcie devices. Signed-off-by: Vamsi Attunuru Change-Id: I233cdc4ab6e9d37600497058e1326015493052c0 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/dataplane/dpu-offload/+/143441 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Jerin Jacob --- doc/api/doxy-api-index.md | 2 +- doc/guides/prog_guide/vfio_lib.rst | 69 ++++++++++++------- lib/pem/pem.c | 28 ++++---- lib/pem/pem.h | 6 +- lib/pem/sdp.c | 18 ++--- lib/pem/sdp.h | 12 ++-- lib/vfio/{dao_vfio_platform.c => dao_vfio.c} | 20 +++--- lib/vfio/dao_vfio.h | 71 ++++++++++++++++++++ lib/vfio/dao_vfio_platform.h | 71 -------------------- lib/vfio/meson.build | 4 +- tests/bar-rw/main.c | 18 ++--- 11 files changed, 170 insertions(+), 149 deletions(-) rename lib/vfio/{dao_vfio_platform.c => dao_vfio.c} (92%) create mode 100644 lib/vfio/dao_vfio.h delete mode 100644 lib/vfio/dao_vfio_platform.h diff --git a/doc/api/doxy-api-index.md b/doc/api/doxy-api-index.md index 2798542..4f2929d 100644 --- a/doc/api/doxy-api-index.md +++ b/doc/api/doxy-api-index.md @@ -43,7 +43,7 @@ The public API headers are grouped by topics - [pem] (@ref dao_pem.h) - **vfio** - - [vfio_platform] (@ref dao_vfio_platform.h) + - [vfio] (@ref dao_vfio.h) - **virtio** - [virtio] (@ref dao_virtio.h) diff --git a/doc/guides/prog_guide/vfio_lib.rst b/doc/guides/prog_guide/vfio_lib.rst index dada47e..7b32387 100644 --- a/doc/guides/prog_guide/vfio_lib.rst +++ b/doc/guides/prog_guide/vfio_lib.rst @@ -1,16 +1,19 @@ .. SPDX-License-Identifier: Marvell-MIT Copyright (c) 2024 Marvell. -********************* -VFIO Platform Library -********************* +************ +VFIO Library +************ Platform devices in Linux refer to System-on-Chip (SoC) components that aren't situated on standard buses such as PCI or USB. You can see them in Linux at the path /sys/bus/platform/devices/. To interact with platform devices from user space, the vfio-platform driver provides a framework. This library provides DAO APIs built upon this framework, enabling access to the device resources. -Prerequisites: -~~~~~~~~~~~~~~ +Also this library can be used to access the standard PCIe devices present at /sys/bus/pci/devices/ +from the user space. + +Prerequisites for Platform Devices: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ To make use of VFIO platform framework, the ``vfio-platform`` module must be loaded first: .. code-block:: console @@ -37,47 +40,65 @@ Next ``DEV`` device must be bound to ``vfio-platform`` driver: echo DEV | sudo tee /sys/bus/platform/drivers/vfio-platform/bind +Prerequisites for PCIe Devices: +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +To make use of VFIO PCIe framework, the ``vfio-pci`` module must be loaded first: + +.. code-block:: console + + sudo modprobe vfio-pci + +The PCIe device needs to be bound to vfio-pci, following a standard two-step procedure. Initially, +the driver_override, located within the pci device directory, must be configured to vfio-pci: + +.. code-block:: console -Platform device initialization + echo vfio-pci | sudo tee /sys/bus/pci/devices//driver_override + +Next ``BDF`` of the device must be bound to ``vfio-pci`` driver: + +.. code-block:: console + + echo | sudo tee /sys/bus/pci/drivers/vfio-pci/bind + +DAO VFIO device initialization ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Invoking the `dao_vfio_platform_init()` API creates a VFIO container by opening the /dev/vfio/vfio -character device and initializes the memory used for storing the details of platform devices. This -API should be invoked only once to initiate the library. +Invoking the `dao_vfio_init()` API creates a VFIO container by opening the /dev/vfio/vfio character +device and initializes the memory used for storing the details of the devices. This API should be +invoked only once to initiate the library. .. code-block:: c - int dao_vfio_platform_init(void); + int dao_vfio_init(void); -After initializing the library, the `dao_vfio_platform_device_setup()` API can be used to initialize -a platform device. The function takes the memory for storing platform device details, specified by -the `struct dao_vfio_platform_device` argument. Upon successful execution, the resources of the -platform devices are mapped, and the device structure is populated. +After initializing the library, the `dao_vfio_device_setup()` API can be used to initialize the +device. The function takes the memory for storing the device details, specified by the +`struct dao_vfio_device` argument. Upon successful execution, the resources of the devices are +mapped, and the device structure is populated. .. code-block:: c - int dao_vfio_platform_device_setup(const char *dev_name, struct dao_vfio_platform_device *pdev); + int dao_vfio_device_setup(const char *dev_name, struct dao_vfio_device *pdev); -.. literalinclude:: ../../../lib/vfio/dao_vfio_platform.h +.. literalinclude:: ../../../lib/vfio/dao_vfio.h :language: c :start-at: struct dao_vfio_mem_resouce - :end-before: End of structure dao_vfio_platform_device. - + :end-before: End of structure dao_vfio_device. -Platform device cleanup +DAO VFIO device cleanup ~~~~~~~~~~~~~~~~~~~~~~~ -`dao_vfio_platform_device_free()` releases the VFIO platform device and frees the associated -memory. +`dao_vfio_device_free()` releases the VFIO device and frees the associated memory. .. code-block:: c - void dao_vfio_platform_device_free(struct dao_vfio_platform_device *pdev); + void dao_vfio_device_free(struct dao_vfio_device *pdev); -Upon closing all open devices, the container can be shut down by calling `dao_vfio_platform_fini()`. +Upon closing all open devices, the container can be shut down by calling `dao_vfio_fini()`. .. code-block:: c - void dao_vfio_platform_fini(void); + void dao_vfio_fini(void); diff --git a/lib/pem/pem.c b/lib/pem/pem.c index 53b4364..db795e7 100644 --- a/lib/pem/pem.c +++ b/lib/pem/pem.c @@ -6,7 +6,7 @@ #include #include "dao_pem.h" -#include "dao_vfio_platform.h" +#include "dao_vfio.h" #include "pem.h" #include "sdp.h" @@ -135,11 +135,11 @@ pem_update_bar4_info(struct pem *pem) } static void -release_vfio_platform_devices(struct pem *pem) +release_vfio_devices(struct pem *pem) { sdp_fini(&pem->sdp_pdev); - dao_vfio_platform_device_free(&pem->bar4_pdev); - dao_vfio_platform_fini(); + dao_vfio_device_free(&pem->bar4_pdev); + dao_vfio_fini(); } static int @@ -175,32 +175,32 @@ pem_bar4_pdev_name_get(struct pem *pem, char *pdev_name) } static int -setup_vfio_platform_devices(struct pem *pem) +setup_vfio_devices(struct pem *pem) { char bar4_pdev_name[VFIO_DEV_NAME_MAX_LEN]; int rc; - rc = dao_vfio_platform_init(); + rc = dao_vfio_init(); if (rc < 0) { - dao_err("Failed to initialize VFIO platform"); + dao_err("Failed to initialize DAO VFIO"); return -1; } rc = sdp_init(&pem->sdp_pdev); if (rc < 0) { - dao_err("Failed to initialize SDP platform device"); + dao_err("Failed to initialize SDP device"); return -1; } rc = pem_bar4_pdev_name_get(pem, bar4_pdev_name); if (rc < 0) { - dao_err("Failed to get PEM platform device name"); + dao_err("Failed to get PEM device name"); return -1; } - rc = dao_vfio_platform_device_setup(bar4_pdev_name, &pem->bar4_pdev); + rc = dao_vfio_device_setup(bar4_pdev_name, &pem->bar4_pdev); if (rc < 0) { - dao_err("Failed to initialize PEM BAR4 platform device"); + dao_err("Failed to initialize PEM BAR4 device"); return -1; } @@ -222,7 +222,7 @@ dao_pem_dev_init(uint16_t pem_devid, struct dao_pem_dev_conf *conf) return -1; } - rc = setup_vfio_platform_devices(pem); + rc = setup_vfio_devices(pem); if (rc < 0) return -1; @@ -267,7 +267,7 @@ dao_pem_dev_init(uint16_t pem_devid, struct dao_pem_dev_conf *conf) return 0; err: - release_vfio_platform_devices(pem); + release_vfio_devices(pem); return -EFAULT; } @@ -287,7 +287,7 @@ dao_pem_dev_fini(uint16_t pem_devid) pem->regions[i] = NULL; } - release_vfio_platform_devices(pem); + release_vfio_devices(pem); return 0; } diff --git a/lib/pem/pem.h b/lib/pem/pem.h index fc91ac6..d139f0e 100644 --- a/lib/pem/pem.h +++ b/lib/pem/pem.h @@ -8,7 +8,7 @@ #include #include -#include +#include #define PEM_BAR4_NUM_INDEX 16 #define PEM_BAR4_INDEX_START 0 @@ -36,8 +36,8 @@ struct pem { bool ctrl_done; struct pem_region *regions[DAO_PEM_CTRL_REGION_MAX]; uint64_t region_mask[DAO_PEM_CTRL_REGION_MASK_MAX]; - struct dao_vfio_platform_device bar4_pdev; - struct dao_vfio_platform_device sdp_pdev; + struct dao_vfio_device bar4_pdev; + struct dao_vfio_device sdp_pdev; }; #endif /* __INCLUDE_PEM_H__ */ diff --git a/lib/pem/sdp.c b/lib/pem/sdp.c index e1e1676..3538b17 100644 --- a/lib/pem/sdp.c +++ b/lib/pem/sdp.c @@ -14,10 +14,10 @@ #include #include -#define SDP_PLAT_DEV_NAME "86e000000000.dpi_sdp_regs" +#define SDP_PLAT_DEV_NAME "86e000000000.dpi_sdp_regs" int -sdp_reg_write(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset, uint64_t val) +sdp_reg_write(struct dao_vfio_device *sdp_pdev, uint64_t offset, uint64_t val) { if (offset > sdp_pdev->mem[0].len) return -ENOMEM; @@ -27,7 +27,7 @@ sdp_reg_write(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset, uint64 } uint64_t -sdp_reg_read(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset) +sdp_reg_read(struct dao_vfio_device *sdp_pdev, uint64_t offset) { if (offset > sdp_pdev->mem[0].len) return -ENOMEM; @@ -36,7 +36,7 @@ sdp_reg_read(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset) } uint64_t * -sdp_reg_addr(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset) +sdp_reg_addr(struct dao_vfio_device *sdp_pdev, uint64_t offset) { if (offset > sdp_pdev->mem[0].len) return NULL; @@ -45,14 +45,14 @@ sdp_reg_addr(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset) } int -sdp_init(struct dao_vfio_platform_device *sdp_pdev) +sdp_init(struct dao_vfio_device *sdp_pdev) { uint64_t reg_val; int rc; - rc = dao_vfio_platform_device_setup(SDP_PLAT_DEV_NAME, sdp_pdev); + rc = dao_vfio_device_setup(SDP_PLAT_DEV_NAME, sdp_pdev); if (rc < 0) { - dao_err("Filed to setup VFIO platform device %s", SDP_PLAT_DEV_NAME); + dao_err("Filed to setup DAO VFIO device %s", SDP_PLAT_DEV_NAME); return errno; } @@ -69,7 +69,7 @@ sdp_init(struct dao_vfio_platform_device *sdp_pdev) } void -sdp_fini(struct dao_vfio_platform_device *sdp_pdev) +sdp_fini(struct dao_vfio_device *sdp_pdev) { - dao_vfio_platform_device_free(sdp_pdev); + dao_vfio_device_free(sdp_pdev); } diff --git a/lib/pem/sdp.h b/lib/pem/sdp.h index 77365e4..b21222b 100644 --- a/lib/pem/sdp.h +++ b/lib/pem/sdp.h @@ -7,7 +7,7 @@ #include -#include +#include #define SDP_RX_OUT_ENABLE(x) (0x80010170 | (x) << 17) #define SDP_RX_OUT_CNTS(x) (0x80010100 | (x) << 17) @@ -19,10 +19,10 @@ #define SDP_EPFX_RINFO_RPVF_SHIFT 32 #define SDP_EPFX_RINFO_SRN_MASK DAO_GENMASK_ULL(6, 0) -int sdp_init(struct dao_vfio_platform_device *sdp_pdev); -uint64_t sdp_reg_read(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset); -int sdp_reg_write(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset, uint64_t val); -uint64_t *sdp_reg_addr(struct dao_vfio_platform_device *sdp_pdev, uint64_t offset); -void sdp_fini(struct dao_vfio_platform_device *sdp_pdev); +int sdp_init(struct dao_vfio_device *sdp_pdev); +uint64_t sdp_reg_read(struct dao_vfio_device *sdp_pdev, uint64_t offset); +int sdp_reg_write(struct dao_vfio_device *sdp_pdev, uint64_t offset, uint64_t val); +uint64_t *sdp_reg_addr(struct dao_vfio_device *sdp_pdev, uint64_t offset); +void sdp_fini(struct dao_vfio_device *sdp_pdev); #endif /* __INCLUDE_SDP_H__ */ diff --git a/lib/vfio/dao_vfio_platform.c b/lib/vfio/dao_vfio.c similarity index 92% rename from lib/vfio/dao_vfio_platform.c rename to lib/vfio/dao_vfio.c index f87c993..2fbfb66 100644 --- a/lib/vfio/dao_vfio_platform.c +++ b/lib/vfio/dao_vfio.c @@ -14,7 +14,7 @@ #include #include -#include +#include #define VFIO_MAX_GROUPS 8 #define VFIO_GROUP_FMT "/dev/vfio/%u" @@ -35,12 +35,12 @@ struct vfio_config { static struct vfio_config vfio_cfg = {.container_fd = -1}; int -dao_vfio_platform_init(void) +dao_vfio_init(void) { int i; if (vfio_cfg.container_fd != -1) { - dao_dbg("VFIO platform has already been initialized."); + dao_dbg("DAO VFIO has already been initialized."); return 0; } @@ -136,7 +136,7 @@ vfio_get_group_fd(const char *dev_name) } static void -vfio_platform_device_mem_free(struct dao_vfio_platform_device *pdev) +vfio_device_mem_free(struct dao_vfio_device *pdev) { unsigned int i; @@ -165,7 +165,7 @@ vfio_clear_group(int group_fd) } int -dao_vfio_platform_device_setup(const char *dev_name, struct dao_vfio_platform_device *pdev) +dao_vfio_device_setup(const char *dev_name, struct dao_vfio_device *pdev) { struct vfio_group_status group_status = {.argsz = sizeof(group_status)}; struct vfio_device_info device_info = {.argsz = sizeof(device_info)}; @@ -242,11 +242,11 @@ dao_vfio_platform_device_setup(const char *dev_name, struct dao_vfio_platform_de pdev->mem[i].len = reg.size; } - dao_dbg("%s: enabled VFIO platform device", dev_name); + dao_dbg("%s: enabled VFIO device", dev_name); return 0; device_mem_free: - vfio_platform_device_mem_free(pdev); + vfio_device_mem_free(pdev); close_device_fd: close(device_fd); clear_group: @@ -255,15 +255,15 @@ dao_vfio_platform_device_setup(const char *dev_name, struct dao_vfio_platform_de } void -dao_vfio_platform_device_free(struct dao_vfio_platform_device *pdev) +dao_vfio_device_free(struct dao_vfio_device *pdev) { - vfio_platform_device_mem_free(pdev); + vfio_device_mem_free(pdev); vfio_clear_group(pdev->group_fd); close(pdev->device_fd); } void -dao_vfio_platform_fini(void) +dao_vfio_fini(void) { int i; diff --git a/lib/vfio/dao_vfio.h b/lib/vfio/dao_vfio.h new file mode 100644 index 0000000..09d83bf --- /dev/null +++ b/lib/vfio/dao_vfio.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: Marvell-MIT + * Copyright (c) 2024 Marvell. + */ + +/** + * @file + * + * DAO VFIO library + * + * DAO VFIO APIs are used to probe VFIO devices and map the resources. + */ + +#ifndef __DAO_VFIO_H__ +#define __DAO_VFIO_H__ + +#define VFIO_DEV_NAME_MAX_LEN 64 + +/** + * VFIO device memory resource. + */ +struct dao_vfio_mem_resouce { + uint8_t *addr; /**< Mapped virtual address. */ + uint64_t len; /**< Length of the resource. */ +}; + +/** DAO VFIO device */ +struct dao_vfio_device { + char name[VFIO_DEV_NAME_MAX_LEN]; /**< Device name */ + int device_fd; /**< VFIO device fd */ + int group_fd; /**< VFIO group fd */ + unsigned int num_resource; /**< Number of device resources */ + struct dao_vfio_mem_resouce *mem; /**< Device resources */ +}; + +/* End of structure dao_vfio_device. */ + +/** + * Initialize the VFIO library by opening a container. + * + * @return + * Zero on success. + */ +int dao_vfio_init(void); + +/** + * Probe a VFIO device and map its regions. Upon a successful probe, + * the device details are set in the memory referenced by the pdev pointer. + * + * @param dev_name + * VFIO device name + * @param pdev + * Pointer to VFIO device structure. + * @return + * Zero on success. + */ +int dao_vfio_device_setup(const char *dev_name, struct dao_vfio_device *pdev); + +/** + * Release a VFIO device and free the associated memory. + * + * @param pdev + * Pointer to VFIO device structure. + */ +void dao_vfio_device_free(struct dao_vfio_device *pdev); + +/** + * Close the container. + */ +void dao_vfio_fini(void); + +#endif /* __DAO_VFIO_H__ */ diff --git a/lib/vfio/dao_vfio_platform.h b/lib/vfio/dao_vfio_platform.h deleted file mode 100644 index 5e2b831..0000000 --- a/lib/vfio/dao_vfio_platform.h +++ /dev/null @@ -1,71 +0,0 @@ -/* SPDX-License-Identifier: Marvell-MIT - * Copyright (c) 2024 Marvell. - */ - -/** - * @file - * - * DAO VFIO platform library - * - * DAO VFIO platform APIs are used to probe VFIO platform devices and map the resources. - */ - -#ifndef __DAO_VFIO_PLATFORM_H__ -#define __DAO_VFIO_PLATFORM_H__ - -#define VFIO_DEV_NAME_MAX_LEN 64 - -/** - * VFIO platform device memory resource. - */ -struct dao_vfio_mem_resouce { - uint8_t *addr; /**< Mapped virtual address. */ - uint64_t len; /**< Length of the resource. */ -}; - -/** VFIO platform device */ -struct dao_vfio_platform_device { - char name[VFIO_DEV_NAME_MAX_LEN]; /**< Device name */ - int device_fd; /**< VFIO device fd */ - int group_fd; /**< VFIO group fd */ - unsigned int num_resource; /**< Number of device resources */ - struct dao_vfio_mem_resouce *mem; /**< Device resources */ -}; - -/* End of structure dao_vfio_platform_device. */ - -/** - * Initialize the VFIO platform library by opening a container. - * - * @return - * Zero on success. - */ -int dao_vfio_platform_init(void); - -/** - * Probe a VFIO platform device and map its regions. Upon a successful probe, - * the device details are set in the memory referenced by the pdev pointer. - * - * @param dev_name - * VFIO platform device name - * @param pdev - * Pointer to VFIO platform device structure. - * @return - * Zero on success. - */ -int dao_vfio_platform_device_setup(const char *dev_name, struct dao_vfio_platform_device *pdev); - -/** - * Release a VFIO platform device and free the associated memory. - * - * @param pdev - * Pointer to VFIO platform device structure. - */ -void dao_vfio_platform_device_free(struct dao_vfio_platform_device *pdev); - -/** - * Close the container. - */ -void dao_vfio_platform_fini(void); - -#endif /* __DAO_VFIO_PLATFORM_H__ */ diff --git a/lib/vfio/meson.build b/lib/vfio/meson.build index 1dd283e..4ffb492 100644 --- a/lib/vfio/meson.build +++ b/lib/vfio/meson.build @@ -6,11 +6,11 @@ if host_build endif sources = files( - 'dao_vfio_platform.c' + 'dao_vfio.c' ) headers = files( - 'dao_vfio_platform.c' + 'dao_vfio.c' ) deps += ['common'] diff --git a/tests/bar-rw/main.c b/tests/bar-rw/main.c index 4113c88..eb1b1dc 100644 --- a/tests/bar-rw/main.c +++ b/tests/bar-rw/main.c @@ -20,7 +20,7 @@ #include "dao_log.h" #include "dao_version.h" -#include "dao_vfio_platform.h" +#include "dao_vfio.h" #define DT_SOC_PATH "/proc/device-tree/soc@0" @@ -58,7 +58,7 @@ int main(int argc, char *argv[]) { char bar4_pdev_name[VFIO_DEV_NAME_MAX_LEN]; - struct dao_vfio_platform_device bar4_pdev; + struct dao_vfio_device bar4_pdev; uint8_t *va; size_t sz; int rc; @@ -68,17 +68,17 @@ main(int argc, char *argv[]) if (rc < 0) rte_exit(EXIT_FAILURE, "Error with EAL initialization\n"); - rc = dao_vfio_platform_init(); + rc = dao_vfio_init(); if (rc < 0) - rte_exit(EXIT_FAILURE, "Failed to initialize VFIO platform\n"); + rte_exit(EXIT_FAILURE, "Failed to initialize DAO VFIO\n"); rc = pem_bar4_pdev_name_get(bar4_pdev_name); if (rc < 0) - rte_exit(EXIT_FAILURE, "Failed to find PEM platform device\n"); + rte_exit(EXIT_FAILURE, "Failed to find PEM device\n"); - rc = dao_vfio_platform_device_setup(bar4_pdev_name, &bar4_pdev); + rc = dao_vfio_device_setup(bar4_pdev_name, &bar4_pdev); if (rc < 0) - rte_exit(EXIT_FAILURE, "Failed to setup PEM BAR4 platform device\n"); + rte_exit(EXIT_FAILURE, "Failed to setup PEM BAR4 device\n"); va = bar4_pdev.mem[0].addr; sz = bar4_pdev.mem[0].len; @@ -88,8 +88,8 @@ main(int argc, char *argv[]) printf("Overwriting existing data with new data 0x%x\n", (uint8_t)getpid()); memset(va, getpid(), sz); - dao_vfio_platform_device_free(&bar4_pdev); - dao_vfio_platform_fini(); + dao_vfio_device_free(&bar4_pdev); + dao_vfio_fini(); /* clean up the EAL */ rte_eal_cleanup();