From b33cef8b6af20ec249efeb75bf574f892f5cf745 Mon Sep 17 00:00:00 2001 From: Courtney Darville Date: Mon, 23 Sep 2024 16:36:21 +1000 Subject: [PATCH 1/3] Add tx_provide() to handle_irq() Signed-off-by: Courtney Darville --- drivers/network/imx/ethernet.c | 1 + drivers/network/meson/ethernet.c | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/drivers/network/imx/ethernet.c b/drivers/network/imx/ethernet.c index 90eea385..a6c77e91 100644 --- a/drivers/network/imx/ethernet.c +++ b/drivers/network/imx/ethernet.c @@ -205,6 +205,7 @@ static void handle_irq(void) while (e & IRQ_MASK) { if (e & NETIRQ_TXF) { tx_return(); + tx_provide(); } if (e & NETIRQ_RXF) { rx_return(); diff --git a/drivers/network/meson/ethernet.c b/drivers/network/meson/ethernet.c index ea745b8f..fa2f4281 100644 --- a/drivers/network/meson/ethernet.c +++ b/drivers/network/meson/ethernet.c @@ -210,18 +210,24 @@ static void tx_return(void) static void handle_irq() { uint32_t e = eth_dma->status; - if (e & DMA_INTR_RXF) { - rx_return(); - } - if (e & DMA_INTR_TXF) { - tx_return(); - } - if (e & DMA_INTR_ABNORMAL) { - if (e & DMA_INTR_FBE) { - sddf_dprintf("Ethernet device fatal bus error\n"); + eth_dma->status &= e; + + while (e & DMA_INTR_MASK) { + if (e & DMA_INTR_RXF) { + rx_return(); } + if (e & DMA_INTR_TXF) { + tx_return(); + tx_provide(); + } + if (e & DMA_INTR_ABNORMAL) { + if (e & DMA_INTR_FBE) { + sddf_dprintf("Ethernet device fatal bus error\n"); + } + } + e = eth_dma->status; + eth_dma->status &= e; } - eth_dma->status &= e; } static void eth_setup(void) From 08f05371727aa685a939733cd61aec944660552c Mon Sep 17 00:00:00 2001 From: Courtney Darville Date: Tue, 24 Sep 2024 21:17:36 +1000 Subject: [PATCH 2/3] Add tx_provide to virtio eth driver Signed-off-by: Courtney Darville --- drivers/network/virtio/ethernet.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/network/virtio/ethernet.c b/drivers/network/virtio/ethernet.c index 6a099ac7..1ec03ab3 100644 --- a/drivers/network/virtio/ethernet.c +++ b/drivers/network/virtio/ethernet.c @@ -295,6 +295,7 @@ static void handle_irq() // or TX queue, so we check both. rx_return(); tx_return(); + tx_provide(); // We have handled the used buffer notification regs->InterruptACK = VIRTIO_MMIO_IRQ_VQUEUE; } From 0e188490fbd89e4d5b083c325df23f944ad275b5 Mon Sep 17 00:00:00 2001 From: Courtney Darville Date: Fri, 20 Sep 2024 02:27:27 +1000 Subject: [PATCH 3/3] Use entire HW ring in ethernet drivers, keep head and tail overflowing Signed-off-by: Courtney Darville --- drivers/network/imx/ethernet.c | 76 ++++++++++++++++++-------------- drivers/network/meson/ethernet.c | 72 +++++++++++++++++------------- 2 files changed, 83 insertions(+), 65 deletions(-) diff --git a/drivers/network/imx/ethernet.c b/drivers/network/imx/ethernet.c index a6c77e91..45aa5b99 100644 --- a/drivers/network/imx/ethernet.c +++ b/drivers/network/imx/ethernet.c @@ -42,10 +42,11 @@ struct descriptor { /* HW ring buffer data type */ typedef struct { - unsigned int tail; /* index to insert at */ - unsigned int head; /* index to remove from */ + uint32_t tail; /* index to insert at */ + uint32_t head; /* index to remove from */ + uint32_t capacity; /* capacity of the ring */ + volatile struct descriptor *descr; /* buffer descriptor array */ net_buff_desc_t descr_mdata[MAX_COUNT]; /* associated meta data array */ - volatile struct descriptor *descr; /* buffer descripter array */ } hw_ring_t; hw_ring_t rx; /* Rx NIC ring */ @@ -58,14 +59,14 @@ net_queue_handle_t tx_queue; volatile struct enet_regs *eth; -static inline bool hw_ring_full(hw_ring_t *ring, size_t ring_capacity) +static inline bool hw_ring_full(hw_ring_t *ring) { - return !((ring->tail - ring->head + 1) % ring_capacity); + return ring->tail - ring->head == ring->capacity; } -static inline bool hw_ring_empty(hw_ring_t *ring, size_t ring_capacity) +static inline bool hw_ring_empty(hw_ring_t *ring) { - return !((ring->tail - ring->head) % ring_capacity); + return ring->tail - ring->head == 0; } static void update_ring_slot(hw_ring_t *ring, unsigned int idx, uintptr_t phys, @@ -78,7 +79,7 @@ static void update_ring_slot(hw_ring_t *ring, unsigned int idx, uintptr_t phys, /* Ensure all writes to the descriptor complete, before we set the flags * that makes hardware aware of this slot. */ - __sync_synchronize(); + THREAD_MEMORY_RELEASE(); d->stat = stat; } @@ -86,30 +87,31 @@ static void rx_provide(void) { bool reprocess = true; while (reprocess) { - while (!hw_ring_full(&rx, RX_COUNT) && !net_queue_empty_free(&rx_queue)) { + while (!hw_ring_full(&rx) && !net_queue_empty_free(&rx_queue)) { net_buff_desc_t buffer; int err = net_dequeue_free(&rx_queue, &buffer); assert(!err); + uint32_t idx = rx.tail % rx.capacity; uint16_t stat = RXD_EMPTY; - if (rx.tail + 1 == RX_COUNT) { + if (idx + 1 == rx.capacity) { stat |= WRAP; } - rx.descr_mdata[rx.tail] = buffer; - update_ring_slot(&rx, rx.tail, buffer.io_or_offset, 0, stat); - rx.tail = (rx.tail + 1) % RX_COUNT; + rx.descr_mdata[idx] = buffer; + update_ring_slot(&rx, idx, buffer.io_or_offset, 0, stat); + rx.tail++; eth->rdar = RDAR_RDAR; } /* Only request a notification from virtualiser if HW ring not full */ - if (!hw_ring_full(&rx, RX_COUNT)) { + if (!hw_ring_full(&rx)) { net_request_signal_free(&rx_queue); } else { net_cancel_signal_free(&rx_queue); } reprocess = false; - if (!net_queue_empty_free(&rx_queue) && !hw_ring_full(&rx, RX_COUNT)) { + if (!net_queue_empty_free(&rx_queue) && !hw_ring_full(&rx)) { net_cancel_signal_free(&rx_queue); reprocess = true; } @@ -119,20 +121,23 @@ static void rx_provide(void) static void rx_return(void) { bool packets_transferred = false; - while (!hw_ring_empty(&rx, RX_COUNT)) { + while (!hw_ring_empty(&rx)) { /* If buffer slot is still empty, we have processed all packets the device has filled */ - volatile struct descriptor *d = &(rx.descr[rx.head]); + uint32_t idx = rx.head % rx.capacity; + volatile struct descriptor *d = &(rx.descr[idx]); if (d->stat & RXD_EMPTY) { break; } - net_buff_desc_t buffer = rx.descr_mdata[rx.head]; + THREAD_MEMORY_ACQUIRE(); + + net_buff_desc_t buffer = rx.descr_mdata[idx]; buffer.len = d->len; int err = net_enqueue_active(&rx_queue, buffer); assert(!err); packets_transferred = true; - rx.head = (rx.head + 1) % RX_COUNT; + rx.head++; } if (packets_transferred && net_require_signal_active(&rx_queue)) { @@ -145,26 +150,26 @@ static void tx_provide(void) { bool reprocess = true; while (reprocess) { - while (!(hw_ring_full(&tx, TX_COUNT)) && !net_queue_empty_active(&tx_queue)) { + while (!(hw_ring_full(&tx)) && !net_queue_empty_active(&tx_queue)) { net_buff_desc_t buffer; int err = net_dequeue_active(&tx_queue, &buffer); assert(!err); + uint32_t idx = tx.tail % tx.capacity; uint16_t stat = TXD_READY | TXD_ADDCRC | TXD_LAST; - if (tx.tail + 1 == TX_COUNT) { + if (idx + 1 == tx.capacity) { stat |= WRAP; } - tx.descr_mdata[tx.tail] = buffer; - update_ring_slot(&tx, tx.tail, buffer.io_or_offset, buffer.len, stat); - - tx.tail = (tx.tail + 1) % TX_COUNT; + tx.descr_mdata[idx] = buffer; + update_ring_slot(&tx, idx, buffer.io_or_offset, buffer.len, stat); + tx.tail++; eth->tdar = TDAR_TDAR; } net_request_signal_active(&tx_queue); reprocess = false; - if (!hw_ring_full(&tx, TX_COUNT) && !net_queue_empty_active(&tx_queue)) { + if (!hw_ring_full(&tx) && !net_queue_empty_active(&tx_queue)) { net_cancel_signal_active(&tx_queue); reprocess = true; } @@ -174,21 +179,23 @@ static void tx_provide(void) static void tx_return(void) { bool enqueued = false; - while (!hw_ring_empty(&tx, TX_COUNT)) { + while (!hw_ring_empty(&tx)) { /* Ensure that this buffer has been sent by the device */ - volatile struct descriptor *d = &(tx.descr[tx.head]); + uint32_t idx = tx.head % tx.capacity; + volatile struct descriptor *d = &(tx.descr[idx]); if (d->stat & TXD_READY) { break; } - net_buff_desc_t buffer = tx.descr_mdata[tx.head]; - buffer.len = 0; - - tx.head = (tx.head + 1) % TX_COUNT; + THREAD_MEMORY_ACQUIRE(); + net_buff_desc_t buffer = tx.descr_mdata[idx]; + buffer.len = 0; int err = net_enqueue_free(&tx_queue, buffer); assert(!err); + enqueued = true; + tx.head++; } if (enqueued && net_require_signal_free(&tx_queue)) { @@ -225,7 +232,9 @@ static void eth_setup(void) uint32_t h = eth->paur; /* Set up HW rings */ + rx.capacity = RX_COUNT; rx.descr = (volatile struct descriptor *)hw_ring_buffer_vaddr; + tx.capacity = TX_COUNT; tx.descr = (volatile struct descriptor *)(hw_ring_buffer_vaddr + (sizeof(struct descriptor) * RX_COUNT)); /* Perform reset */ @@ -269,14 +278,13 @@ static void eth_setup(void) eth->tipg = TIPG; /* Transmit FIFO Watermark register - store and forward */ eth->tfwr = STRFWD; - /* clear rx store and forward. This must be done for hardware csums*/ + /* clear rx store and forward. This must be done for hardware csums */ eth->rsfl = 0; /* Do not forward frames with errors + check the csum */ eth->racc = RACC_LINEDIS | RACC_IPDIS | RACC_PRODIS; /* Add the checksum for known IP protocols */ eth->tacc = TACC_PROCHK | TACC_IPCHK; - /* Set RDSR */ eth->rdsr = hw_ring_buffer_paddr; eth->tdsr = hw_ring_buffer_paddr + (sizeof(struct descriptor) * RX_COUNT); diff --git a/drivers/network/meson/ethernet.c b/drivers/network/meson/ethernet.c index fa2f4281..d510ec00 100644 --- a/drivers/network/meson/ethernet.c +++ b/drivers/network/meson/ethernet.c @@ -44,10 +44,11 @@ _Static_assert((RX_COUNT + TX_COUNT) * sizeof(struct descriptor) <= NET_HW_REGIO "Expect rx+tx buffers to fit in single 2MB page"); typedef struct { - unsigned int tail; /* index to insert at */ - unsigned int head; /* index to remove from */ + uint32_t tail; /* index to insert at */ + uint32_t head; /* index to remove from */ + uint32_t capacity; /* capacity of the ring */ + volatile struct descriptor *descr; /* buffer descriptor array */ net_buff_desc_t descr_mdata[MAX_COUNT]; /* associated meta data array */ - volatile struct descriptor *descr; /* buffer descripter array */ } hw_ring_t; hw_ring_t rx; @@ -59,14 +60,14 @@ net_queue_handle_t tx_queue; volatile struct eth_mac_regs *eth_mac; volatile struct eth_dma_regs *eth_dma; -static inline bool hw_ring_full(hw_ring_t *ring, size_t ring_capacity) +static inline bool hw_ring_full(hw_ring_t *ring) { - return !((ring->tail + 2 - ring->head) % ring_capacity); + return ring->tail - ring->head == ring->capacity; } -static inline bool hw_ring_empty(hw_ring_t *ring, size_t ring_capacity) +static inline bool hw_ring_empty(hw_ring_t *ring) { - return !((ring->tail - ring->head) % ring_capacity); + return ring->tail - ring->head == 0; } static void update_ring_slot(hw_ring_t *ring, unsigned int idx, uint32_t status, @@ -87,27 +88,28 @@ static void rx_provide() { bool reprocess = true; while (reprocess) { - while (!hw_ring_full(&rx, RX_COUNT) && !net_queue_empty_free(&rx_queue)) { + while (!hw_ring_full(&rx) && !net_queue_empty_free(&rx_queue)) { net_buff_desc_t buffer; int err = net_dequeue_free(&rx_queue, &buffer); assert(!err); + uint32_t idx = rx.tail % rx.capacity; uint32_t cntl = (MAX_RX_FRAME_SZ << DESC_RXCTRL_SIZE1SHFT) & DESC_RXCTRL_SIZE1MASK; - if (rx.tail + 1 == RX_COUNT) { + if (idx + 1 == rx.capacity) { cntl |= DESC_RXCTRL_RXRINGEND; } - rx.descr_mdata[rx.tail] = buffer; - update_ring_slot(&rx, rx.tail, DESC_RXSTS_OWNBYDMA, cntl, buffer.io_or_offset, 0); + rx.descr_mdata[idx] = buffer; + update_ring_slot(&rx, idx, DESC_RXSTS_OWNBYDMA, cntl, buffer.io_or_offset, 0); eth_dma->rxpolldemand = POLL_DATA; - rx.tail = (rx.tail + 1) % RX_COUNT; + rx.tail++; } net_request_signal_free(&rx_queue); reprocess = false; - if (!net_queue_empty_free(&rx_queue) && !hw_ring_full(&rx, RX_COUNT)) { + if (!net_queue_empty_free(&rx_queue) && !hw_ring_full(&rx)) { net_cancel_signal_free(&rx_queue); reprocess = true; } @@ -117,33 +119,36 @@ static void rx_provide() static void rx_return(void) { bool packets_transferred = false; - while (!hw_ring_empty(&rx, RX_COUNT)) { + while (!hw_ring_empty(&rx)) { /* If buffer slot is still empty, we have processed all packets the device has filled */ - volatile struct descriptor *d = &(rx.descr[rx.head]); + uint32_t idx = rx.head % rx.capacity; + volatile struct descriptor *d = &(rx.descr[idx]); if (d->status & DESC_RXSTS_OWNBYDMA) { break; } - net_buff_desc_t buffer = rx.descr_mdata[rx.head]; + THREAD_MEMORY_ACQUIRE(); + net_buff_desc_t buffer = rx.descr_mdata[idx]; if (d->status & DESC_RXSTS_ERROR) { sddf_dprintf("ETH|ERROR: RX descriptor returned with error status %x\n", d->status); + idx = rx.tail % rx.capacity; uint32_t cntl = (MAX_RX_FRAME_SZ << DESC_RXCTRL_SIZE1SHFT) & DESC_RXCTRL_SIZE1MASK; - if (rx.tail + 1 == RX_COUNT) { + if (idx + 1 == rx.capacity) { cntl |= DESC_RXCTRL_RXRINGEND; } - rx.descr_mdata[rx.tail] = buffer; - update_ring_slot(&rx, rx.tail, DESC_RXSTS_OWNBYDMA, cntl, buffer.io_or_offset, 0); + rx.descr_mdata[idx] = buffer; + update_ring_slot(&rx, idx, DESC_RXSTS_OWNBYDMA, cntl, buffer.io_or_offset, 0); eth_dma->rxpolldemand = POLL_DATA; - rx.tail = (rx.tail + 1) % RX_COUNT; + rx.tail++; } else { buffer.len = (d->status & DESC_RXSTS_LENMSK) >> DESC_RXSTS_LENSHFT; int err = net_enqueue_active(&rx_queue, buffer); assert(!err); packets_transferred = true; } - rx.head = (rx.head + 1) % RX_COUNT; + rx.head++; } if (packets_transferred && net_require_signal_active(&rx_queue)) { @@ -156,26 +161,27 @@ static void tx_provide(void) { bool reprocess = true; while (reprocess) { - while (!(hw_ring_full(&tx, TX_COUNT)) && !net_queue_empty_active(&tx_queue)) { + while (!(hw_ring_full(&tx)) && !net_queue_empty_active(&tx_queue)) { net_buff_desc_t buffer; int err = net_dequeue_active(&tx_queue, &buffer); assert(!err); + uint32_t idx = tx.tail % tx.capacity; uint32_t cntl = (((uint32_t) buffer.len) << DESC_TXCTRL_SIZE1SHFT) & DESC_TXCTRL_SIZE1MASK; cntl |= DESC_TXCTRL_TXLAST | DESC_TXCTRL_TXFIRST | DESC_TXCTRL_TXINT; - if (tx.tail + 1 == TX_COUNT) { + if (idx + 1 == tx.capacity) { cntl |= DESC_TXCTRL_TXRINGEND; } - tx.descr_mdata[tx.tail] = buffer; - update_ring_slot(&tx, tx.tail, DESC_TXSTS_OWNBYDMA, cntl, buffer.io_or_offset, 0); + tx.descr_mdata[idx] = buffer; + update_ring_slot(&tx, idx, DESC_TXSTS_OWNBYDMA, cntl, buffer.io_or_offset, 0); - tx.tail = (tx.tail + 1) % TX_COUNT; + tx.tail++; } net_request_signal_active(&tx_queue); reprocess = false; - if (!hw_ring_full(&tx, TX_COUNT) && !net_queue_empty_active(&tx_queue)) { + if (!hw_ring_full(&tx) && !net_queue_empty_active(&tx_queue)) { net_cancel_signal_active(&tx_queue); reprocess = true; } @@ -186,19 +192,21 @@ static void tx_provide(void) static void tx_return(void) { bool enqueued = false; - while (!hw_ring_empty(&tx, TX_COUNT)) { + while (!hw_ring_empty(&tx)) { /* Ensure that this buffer has been sent by the device */ - volatile struct descriptor *d = &(tx.descr[tx.head]); + uint32_t idx = tx.head % tx.capacity; + volatile struct descriptor *d = &(tx.descr[idx]); if (d->status & DESC_TXSTS_OWNBYDMA) { break; } - net_buff_desc_t buffer = tx.descr_mdata[tx.head]; + THREAD_MEMORY_ACQUIRE(); + net_buff_desc_t buffer = tx.descr_mdata[idx]; int err = net_enqueue_free(&tx_queue, buffer); assert(!err); enqueued = true; - tx.head = (tx.head + 1) % TX_COUNT; + tx.head++; } if (enqueued && net_require_signal_free(&tx_queue)) { @@ -238,7 +246,9 @@ static void eth_setup(void) assert((hw_ring_buffer_paddr & 0xFFFFFFFF) == hw_ring_buffer_paddr); + rx.capacity = RX_COUNT; rx.descr = (volatile struct descriptor *)hw_ring_buffer_vaddr; + tx.capacity = TX_COUNT; tx.descr = (volatile struct descriptor *)(hw_ring_buffer_vaddr + (sizeof(struct descriptor) * RX_COUNT)); /* Perform reset */