drivers/pktmem: fix allocPktBuf and freePktBuf

fix packet buffer handling for targets with page size less than packet buffer size

JIRA: RTOS-507

drivers/bdring.c

@@ -189,7 +189,7 @@ size_t net_refillRx(net_bufdesc_ring_t *ring, size_t ethpad)
 	n = 0;
 	i = ring->tail;
 	nxt = (i + 1) & ring->last;  // NOTE: 2^n ring size verified in net_initRings
-	sz = ring->ops->pkt_buf_sz;
+	sz = net_maxDMAPbufSize;
 
 	while (nxt != atomic_load(&ring->head)) {
 		p = net_allocDMAPbuf(&pa, sz);

drivers/bdring.h

@@ -36,7 +36,6 @@ typedef struct net_bufdesc_ops_ {
 
 	size_t desc_size;
 	size_t ring_alignment;
-	size_t pkt_buf_sz;
 	size_t max_tx_frag;
 } net_bufdesc_ops_t;
 

drivers/imx-enet.c

@@ -15,6 +15,7 @@
 #include "bdring.h"
 #include "ephy.h"
 #include "physmmap.h"
+#include "pktmem.h"
 #include "res-create.h"
 #include "imx-enet-regs.h"
 
@@ -37,7 +38,6 @@
 #define ENET_DIS_RX_ON_TX             0 /* usually: 0 in half-duplex, 1 in full-duplex */
 #define ENET_RX_RING_SIZE             64
 #define ENET_TX_RING_SIZE             64
-#define ENET_BUFFER_SIZE              (2048 - 64)
 #define ENET_MDC_ALWAYS_ON            1 /* NOTE: should be always ON, otherwise unreliable*/
 #define MDIO_TIMEOUT                  (0)
 #define ENET_RESET_TIMEOUT            (100 * 1000)
@@ -139,8 +139,8 @@ static void enet_start(enet_state_t *state)
 	//	addr_t ecr_pa = (addr_t)&((struct enet_regs *)state->phys)->ECR;
 	// FIXME: last_will(ECR = ENET_ECR_MAGIC_VAL | ENET_ECR_RESET);
 
-	state->mmio->MRBR = ENET_BUFFER_SIZE;  // FIXME: coerce with net_allocPktBuf()
-	state->mmio->FTRL = BIT(14) - 1;       // FIXME: truncation to just above link MTU
+	state->mmio->MRBR = net_maxDMAPbufSize;
+	state->mmio->FTRL = BIT(14) - 1;  // FIXME: truncation to just above link MTU
 
 	state->mmio->RCR = ENET_RCR_MAX_FL_NO_VLAN_VAL << ENET_RCR_MAX_FL_SHIFT |
 		ENET_RCR_CRCFWD | ENET_RCR_PAUFWD |
@@ -428,7 +428,6 @@ static const net_bufdesc_ops_t enet_ring_ops = {
 	.fillTxDesc = enet_fillTxDesc,
 	.desc_size = sizeof(enet_buf_desc_t),
 	.ring_alignment = 64,
-	.pkt_buf_sz = ENET_BUFFER_SIZE,
 	.max_tx_frag = 0xFFFF,
 };
 

drivers/pktmem.c

@@ -17,28 +17,55 @@
 #include <string.h>
 
 
-#define CACHE_LINE_SIZE 64
-#define PAGE_SIZE       4096
+// FIXME: transfer CACHE_LINE_SIZE to appropriate file with platform-specific definitions
+#if defined(__ARM_ARCH_7EM__)
+#define CACHE_LINE_SIZE    32
+#define PKT_BUF_ALLOC_SIZE (3 * PAGE_SIZE)
+#define PKT_BUF_SIZE       (PKT_BUF_ALLOC_SIZE - 12 /* sizeof(buf_page_head_t) (without alignment) */)
+#else
+#define CACHE_LINE_SIZE    64
+#define PKT_BUF_ALLOC_SIZE PAGE_SIZE
+#define PKT_BUF_SIZE       (2048 - CACHE_LINE_SIZE)
+#endif
 
 #define TRUE_LRU 0
 
 
-#define PKT_BUF_SIZE       (2048 - CACHE_LINE_SIZE)
-#define PKT_BUF_CNT        (size_t)((PAGE_SIZE - CACHE_LINE_SIZE) / PKT_BUF_SIZE)
-#define PKT_BUF_IDX        11 /* log2(PAGE_SIZE) - ceil(log2(PKT_BUF_CNT)) */
-#define PKT_BUF_CACHE_SIZE 16
+#if PAGE_SIZE < PKT_BUF_SIZE
+#define PKT_BUF_CNT              1
+#define PKT_BUF_WHICH(p)         0
+#define PKT_BUF_PAGE_HEAD_OFFSET (PKT_BUF_ALLOC_SIZE - CACHE_LINE_SIZE)
+#define PKT_BUF_PACKET_OFFSET    0
 
+typedef struct _buf_page_head {
+	uint32_t rsvd[5]; /* NOTE: aligned access */
+	uint32_t free_mask;
+	struct _buf_page_head *next;
+	struct _buf_page_head *prev;
+} buf_page_head_t;
+#else
+#define PKT_BUF_CNT              (size_t)((PAGE_SIZE - CACHE_LINE_SIZE) / PKT_BUF_SIZE)
+#define _NEXT_POW2(x)            (((x) <= 2) ? (x) : ((1ull << 32) >> __builtin_clz((x) - 1)))
+#define PKT_BUF_IDX              (__builtin_ctz(PAGE_SIZE) - __builtin_ctz(_NEXT_POW2(PKT_BUF_CNT))) /* log2(PAGE_SIZE) - ceil(log2(PKT_BUF_CNT)) */
+#define PKT_BUF_WHICH(p)         (((size_t)(p) & (PAGE_SIZE - 1)) >> PKT_BUF_IDX)
+#define PKT_BUF_PAGE_HEAD_OFFSET 0
+#define PKT_BUF_PACKET_OFFSET    CACHE_LINE_SIZE
 
 typedef struct _buf_page_head {
 	unsigned free_mask;
 	struct _buf_page_head *next;
 	struct _buf_page_head *prev;
 } buf_page_head_t;
+#endif
+
+#define PKT_BUF_CACHE_SIZE 16
+#define PKT_BUF_FREE_MASK  ((1 << PKT_BUF_CNT) - 1)
 
 
 static buf_page_head_t pkt_buf_lru = { .next = &pkt_buf_lru, .prev = &pkt_buf_lru };
 static unsigned pkt_bufs_free;
 
+const size_t net_maxDMAPbufSize = PKT_BUF_SIZE;
 
 static void net_listAdd(buf_page_head_t *ph, buf_page_head_t *after)
 {
@@ -64,19 +91,19 @@ static void net_listDel(buf_page_head_t *ph)
 
 static void net_freePktBuf(void *p)
 {
-	buf_page_head_t *ph = (void *)((uintptr_t)p & ~(PAGE_SIZE - 1));
-	unsigned which = ((size_t)p & (PAGE_SIZE - 1)) >> PKT_BUF_IDX;
+	buf_page_head_t *ph = (void *)((uintptr_t)p & ~(PAGE_SIZE - 1)) + PKT_BUF_PAGE_HEAD_OFFSET;
+	unsigned which = PKT_BUF_WHICH(p);
 	unsigned old_mask;
 
 	old_mask = ph->free_mask;
 	ph->free_mask |= 1 << which;
 	++pkt_bufs_free;
 
-	if (pkt_bufs_free > PKT_BUF_CACHE_SIZE && ph->free_mask == (1 << PKT_BUF_CNT) - 1) {
+	if (pkt_bufs_free > PKT_BUF_CACHE_SIZE && ph->free_mask == PKT_BUF_FREE_MASK) {
 		if (old_mask != 0) {
 			net_listDel(ph);
 		}
-		munmap(ph, PAGE_SIZE);
+		munmap((void *)ph - PKT_BUF_PAGE_HEAD_OFFSET, PKT_BUF_ALLOC_SIZE);
 		pkt_bufs_free -= PKT_BUF_CNT;
 		return;
 	}
@@ -114,14 +141,15 @@ static ssize_t net_allocPktBuf(void **bufp)
 	if (pkt_bufs_free == 0) {
 		SYS_ARCH_UNPROTECT(old_level);
 
-		ph = dmammap(PAGE_SIZE);
+		ph = dmammap(PKT_BUF_ALLOC_SIZE);
 		if (ph == NULL) {
 			printf("mmap: no memory?\n");
 			return 0;
 		}
+		ph = (void *)ph + PKT_BUF_PAGE_HEAD_OFFSET;
 
 		memset(ph, 0, CACHE_LINE_SIZE);
-		ph->free_mask = (1 << PKT_BUF_CNT) - 1;
+		ph->free_mask = PKT_BUF_FREE_MASK;
 
 		SYS_ARCH_PROTECT(old_level);
 		net_listAdd(ph, &pkt_buf_lru);
@@ -140,7 +168,7 @@ static ssize_t net_allocPktBuf(void **bufp)
 
 	SYS_ARCH_UNPROTECT(old_level);
 
-	*bufp = (void *)ph + CACHE_LINE_SIZE + i * PKT_BUF_SIZE;
+	*bufp = (void *)ph + PKT_BUF_PACKET_OFFSET + i * PKT_BUF_SIZE - PKT_BUF_PAGE_HEAD_OFFSET;
 
 	return PKT_BUF_SIZE;
 }

drivers/pktmem.h

@@ -12,11 +12,13 @@
 #define NET_PKTMEM_H_
 
 #include <stdint.h>
+#include <unistd.h>
 
 
 struct pbuf;
 
 
+extern const size_t net_maxDMAPbufSize;
 struct pbuf *net_allocDMAPbuf(addr_t *pa, size_t sz);
 struct pbuf *net_makeDMAPbuf(struct pbuf *p);
 

drivers/rtl8139cp.c

@@ -182,7 +182,6 @@ static const net_bufdesc_ops_t rtl_ring_ops = {
 
 	/* desc_size */		sizeof(rtl_buf_desc_t),
 	/* ring_alignment */	64,
-	/* pkt_buf_sz */	1524,	/* <= RXCMD_SZ_MASK */
 	/* max_tx_frag */	TXCMD_SZ_MASK,
 };