Merge pull request #319 from CESNET/dpdk-rss-fix

dpdk: fix RSS for X710 (i40e)

Author: SiskaPavel

init/config2args.py

@@ -157,6 +157,9 @@ def process_input_dpdk_plugin(settings):
     mtu = settings.get("mtu", 1518)
     if mtu is not None:
         primary_param += f"mtu={mtu};"
+    rss_offload = settings.get("rss_offload", None)
+    if rss_offload is not None:
+        primary_param += f"rss={rss_offload};"
     primary_param += f"eal={eal}\""
 
     params = []

init/link0.conf.example

@@ -35,6 +35,11 @@ input_plugin:
     eal_opts: null  # EAL options (null = default options)
     mtu: null  # Maximum Transmission Unit (defaults to RTE_ETHER_MAX_LEN)
 
+    # Our default RSS configuration is RTE_ETH_RSS_IP.
+    # Intel X710 (i40e) does not work reliably with it, so by default we use RSS provided by the NIC/driver.
+    # Set this explicitly to override the driver RSS configuration.
+    rss_offload: null
+
 # Storage configuration (storage)
 storage:
   cache:

init/schema.json

@@ -93,11 +93,11 @@
             "ndp": {
               "type": "object",
               "properties": {
-		"device": {
+                "device": {
                   "type": "array",
                   "items": {
                     "type": "string"
-                   }
+                  }
                 },
                 "queues": {
                   "type": "string"
@@ -182,6 +182,12 @@
                     "string",
                     "null"
                   ]
+                },
+                "rss_offload": {
+                  "type": [
+                    "integer",
+                    "null"
+                  ]
                 }
               },
               "required": [

src/plugins/input/dpdk/README.md

@@ -17,6 +17,7 @@ input_plugin:
     workers_cpu_list: []
     eal_opts: null
     mtu: 1518
+    rss_offload: null
 ```
 
 ## Parameters
@@ -36,6 +37,7 @@ input_plugin:
 |__workers_cpu_list__| [] (autofill) | List of CPU cores assigned to RX queues (must match number of rx_queues) |
 |__eal_opts__     | null | Extra options to be passed to the DPDK EAL (Environment Abstraction Layer). Can be used for fine-tuning DPDK behavior.|
 |__mtu__          | 1518 | Maximum Transmission Unit size for the interface. Defines the maximum packet size that can be received.|
+|__rss_offload__  | null | RSS offload configuration. Can be used to override the default RSS offload configuration.|
 
 ## How to use
 
@@ -240,9 +242,12 @@ grubby --update-kernel ALL --args "isolcpus=2-19,22-39"
 ```
 
 
-### 4. Validate with dpdk-testpmd
+### 4. Troubleshooting
 
-TODO
+⚠️ RSS on Intel X710 (i40e)
+
+We observed that RSS on Intel X710 (i40e) may not distribute packets across multiple RX queues with the default RTE_ETH_RSS_IP.
+For X710 (i40e) we use full RSS offload provided by the driver. If you experience similar issues, try to set `rss_offload` explicitly to override the default RSS offload configuration.
 
 ## FAQ
 

src/plugins/input/dpdk/src/dpdk.cpp

@@ -86,12 +86,14 @@ void DpdkCore::configure(const char* params)
 	uint16_t rxQueueCount = parser.rx_queues();
 	m_mBufsCount = parser.pkt_buffer_size();
 	uint16_t mtuSize = parser.mtu_size();
+	uint64_t rssOffload = parser.rss_offload();
 
 	configureEal(parser.eal_params());
 
 	m_dpdkDevices.reserve(parser.port_numbers().size());
 	for (auto portID : parser.port_numbers()) {
-		m_dpdkDevices.emplace_back(portID, rxQueueCount, mempoolSize, m_mBufsCount, mtuSize);
+		m_dpdkDevices
+			.emplace_back(portID, rxQueueCount, mempoolSize, m_mBufsCount, mtuSize, rssOffload);
 	}
 
 	isConfigured = true;

src/plugins/input/dpdk/src/dpdk.hpp

@@ -34,6 +34,7 @@ class DpdkOptParser : public OptionsParser {
 	uint16_t rx_queues_ = 1;
 	std::string eal_;
 	uint16_t mtu_;
+	uint64_t rss_offload_ = 0;
 
 	std::vector<uint16_t> parsePortNumbers(std::string arg)
 	{
@@ -123,6 +124,20 @@ class DpdkOptParser : public OptionsParser {
 				return true;
 			},
 			RequiredArgument);
+		register_option(
+			"r",
+			"rss",
+			"VALUE",
+			"RSS offload value. Default: 0",
+			[this](const char* arg) {
+				try {
+					rss_offload_ = str2num<decltype(rss_offload_)>(arg);
+				} catch (std::invalid_argument&) {
+					return false;
+				}
+				return true;
+			},
+			RequiredArgument);
 		register_option(
 			"e",
 			"eal",
@@ -160,6 +175,8 @@ class DpdkOptParser : public OptionsParser {
 	uint16_t rx_queues() const { return rx_queues_; }
 
 	uint16_t mtu_size() const { return mtu_; }
+
+	uint64_t rss_offload() const { return rss_offload_; }
 };
 
 class DpdkCore {

src/plugins/input/dpdk/src/dpdkDevice.cpp

@@ -42,7 +42,8 @@ DpdkDevice::DpdkDevice(
 	uint16_t rxQueueCount,
 	uint16_t memPoolSize,
 	uint16_t mbufsCount,
-	uint16_t mtuSize)
+	uint16_t mtuSize,
+	uint64_t rssOffload)
 	: m_portID(portID)
 	, m_rxQueueCount(rxQueueCount)
 	, m_txQueueCount(0)
@@ -51,13 +52,13 @@ DpdkDevice::DpdkDevice(
 	, m_supportedRSS(false)
 	, m_supportedHWTimestamp(false)
 	, m_mtuSize(mtuSize)
+	, m_rssOffload(rssOffload)
 {
 	validatePort();
 	recognizeDriver();
 	configurePort();
 	initMemPools(memPoolSize);
 	setupRxQueues(memPoolSize);
-	configureRSS();
 	enablePort();
 }
 
@@ -95,9 +96,13 @@ void DpdkDevice::recognizeDriver()
 	std::cerr << "\tflow type RSS offloads: " << rteDevInfo.flow_type_rss_offloads << std::endl;
 
 	/* Check if RSS hashing is supported in NIC */
-	m_supportedRSS = (rteDevInfo.flow_type_rss_offloads & RTE_ETH_RSS_IP) != 0;
-	std::cerr << "\tDetected RSS offload capability: " << (m_supportedRSS ? "yes" : "no")
-			  << std::endl;
+	if (m_rxQueueCount > 1) {
+		m_supportedRSS = (rteDevInfo.flow_type_rss_offloads & RTE_ETH_RSS_IP) != 0;
+		std::cerr << "\tDetected RSS offload capability: " << (m_supportedRSS ? "yes" : "no")
+				  << std::endl;
+	} else {
+		m_supportedRSS = false;
+	}
 
 	/* Check if HW timestamps are supported, we support NFB cards only */
 	if (m_isNfbDpdkDriver) {
@@ -154,7 +159,9 @@ rte_eth_conf DpdkDevice::createPortConfig()
 
 	if (m_supportedRSS) {
 		portConfig.rxmode.mq_mode = RTE_ETH_MQ_RX_RSS;
+		portConfig.rx_adv_conf.rss_conf = createRSSConfig();
 	} else {
+		std::cerr << "Skipped RSS hash setting for port " << m_portID << "." << std::endl;
 		portConfig.rxmode.mq_mode = RTE_ETH_MQ_RX_NONE;
 	}
 
@@ -220,12 +227,9 @@ void DpdkDevice::setupRxQueues(uint16_t memPoolSize)
 			  << " set up. Size of each queue: " << rxQueueSize << std::endl;
 }
 
-void DpdkDevice::configureRSS()
+rte_eth_rss_conf DpdkDevice::createRSSConfig()
 {
-	if (!m_supportedRSS) {
-		std::cerr << "Skipped RSS hash setting for port " << m_portID << "." << std::endl;
-		return;
-	}
+	struct rte_eth_rss_conf rssConfig = {};
 
 	rte_eth_dev_info rteDevInfo;
 	if (rte_eth_dev_info_get(m_portID, &rteDevInfo)) {
@@ -243,23 +247,32 @@ void DpdkDevice::configureRSS()
 			return hashKey[idx++ % sizeof(hashKey)];
 		});
 
-	const uint64_t rssOffloads = rteDevInfo.flow_type_rss_offloads & RTE_ETH_RSS_IP;
-	if (rssOffloads != RTE_ETH_RSS_IP) {
-		std::cerr << "RTE_ETH_RSS_IP is not supported by the card. Used subset: " << rssOffloads
-				  << std::endl;
+	uint64_t rssOffloads = 0;
+	if (m_rssOffload) { // user specified RSS offload
+		rssOffloads = m_rssOffload;
+	} else {
+		if (std::string(rteDevInfo.driver_name) == "net_i40e") {
+			std::cerr << "RTE_ETH_RSS_IP is not supported reliably by this driver, falling back to "
+						 "NIC-provided RSS: "
+					  << rteDevInfo.flow_type_rss_offloads << std::endl;
+			std::cerr << "You can override this behavior using the 'rss' configuration parameter."
+					  << std::endl;
+			rssOffloads = rteDevInfo.flow_type_rss_offloads;
+		} else {
+			rssOffloads = rteDevInfo.flow_type_rss_offloads & RTE_ETH_RSS_IP;
+			if (rssOffloads != RTE_ETH_RSS_IP) {
+				std::cerr << "RTE_ETH_RSS_IP is not supported by the card. Used subset: "
+						  << rssOffloads << std::endl;
+			}
+		}
 	}
 
-	struct rte_eth_rss_conf rssConfig = {};
+	std::cerr << "Using RSS offloads: " << rssOffloads << std::endl;
+
 	rssConfig.rss_key = m_hashKey.data();
 	rssConfig.rss_key_len = rssHashKeySize;
 	rssConfig.rss_hf = rssOffloads;
-
-	int ret = rte_eth_dev_rss_hash_update(m_portID, &rssConfig);
-	if (ret < 0) {
-		std::cerr << "Setting RSS {" << rssOffloads << "} for port " << m_portID
-				  << " failed. Errno:" << ret << std::endl;
-		throw PluginError("DpdkDevice::configureRSS() has failed.");
-	}
+	return rssConfig;
 }
 
 void DpdkDevice::enablePort()

src/plugins/input/dpdk/src/dpdkDevice.hpp

@@ -48,13 +48,15 @@ class DpdkDevice {
 	 * @param memPoolSize The size of the memory pool for packet buffers.
 	 * @param mbufsCount The number of mbufs (packet buffers) to be allocated.
 	 * @param mtuSize Maximum transmission unit of input interface.
+	 * @param rssOffload RSS offload value. 0 for subset of RTE_ETH_RSS_IP.
 	 */
 	DpdkDevice(
 		uint16_t portID,
 		uint16_t rxQueueCount,
 		uint16_t memPoolSize,
 		uint16_t mbufsCount,
-		uint16_t mtuSize);
+		uint16_t mtuSize,
+		uint64_t rssOffload);
 
 	/**
 	 * @brief Receives packets from the specified receive queue of the DPDK device.
@@ -84,7 +86,7 @@ class DpdkDevice {
 	rte_eth_conf createPortConfig();
 	void initMemPools(uint16_t memPoolSize);
 	void setupRxQueues(uint16_t memPoolSize);
-	void configureRSS();
+	rte_eth_rss_conf createRSSConfig();
 	void enablePort();
 	void createRteMempool(uint16_t mempoolSize);
 	void setRxTimestampDynflag();
@@ -102,6 +104,7 @@ class DpdkDevice {
 	int m_rxTimestampOffset;
 	int m_rxTimestampDynflag;
 	uint16_t m_mtuSize;
+	uint64_t m_rssOffload = 0;
 };
 
 } // namespace ipxp