From 1a7bb2f60a459c2cca6646ccbe04a52fcd8272f6 Mon Sep 17 00:00:00 2001
From: jiangheng <jiangheng14@huawei.com>
Date: Wed, 31 Jan 2024 14:49:40 +0800
Subject: [PATCH] try to ensure arp packet can be sent

---
 src/lstack/core/lstack_dpdk.c              | 18 ++++++++++--------
 src/lstack/core/lstack_lwip.c              | 13 ++++++++++---
 src/lstack/core/lstack_protocol_stack.c    |  4 ++--
 src/lstack/include/lstack_dpdk.h           |  2 +-
 src/lstack/include/lstack_protocol_stack.h |  2 +-
 src/lstack/netif/lstack_ethdev.c           |  8 ++++----
 src/lstack/netif/lstack_vdev.c             |  2 +-
 7 files changed, 29 insertions(+), 20 deletions(-)

diff --git a/src/lstack/core/lstack_dpdk.c b/src/lstack/core/lstack_dpdk.c
index e352850..74e033b 100644
--- a/src/lstack/core/lstack_dpdk.c
+++ b/src/lstack/core/lstack_dpdk.c
@@ -280,14 +280,16 @@ int32_t create_shared_ring(struct protocol_stack *stack)
     return 0;
 }
 
-int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num)
+int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num, bool reserve)
 {
-    /*
-     * don't use rte_mempool_avail_count, it traverse cpu local cache,
-     * when RTE_MAX_LCORE is too large, it's time-consuming
-     */
-    if (rte_ring_count(pool->pool_data) < MBUFPOOL_RESERVE_NUM + num) {
-        return -ENOMEM;
+    if (reserve) {
+        /*
+         * don't use rte_mempool_avail_count, it traverse cpu local cache,
+         * when RTE_MAX_LCORE is too large, it's time-consuming
+         */
+        if (rte_ring_count(pool->pool_data) < MBUFPOOL_RESERVE_NUM + num) {
+            return -ENOMEM;
+        }
     }
 
     int32_t ret = rte_pktmbuf_alloc_bulk(pool, mbufs, num);
@@ -310,7 +312,7 @@ int32_t fill_mbuf_to_ring(struct rte_mempool *mempool, struct rte_ring *ring, ui
     while (remain > 0) {
         batch = LWIP_MIN(remain, RING_SIZE(FREE_RX_QUEUE_SZ));
 
-        ret = dpdk_alloc_pktmbuf(mempool, free_buf, batch);
+        ret = dpdk_alloc_pktmbuf(mempool, free_buf, batch, true);
         if (ret != 0) {
             LSTACK_LOG(ERR, LSTACK, "cannot alloc mbuf for ring, count: %u ret=%d\n", batch, ret);
             return -1;
diff --git a/src/lstack/core/lstack_lwip.c b/src/lstack/core/lstack_lwip.c
index 3f76424..da50fec 100644
--- a/src/lstack/core/lstack_lwip.c
+++ b/src/lstack/core/lstack_lwip.c
@@ -22,6 +22,7 @@
 #include <lwip/posix_api.h>
 #include <lwip/api.h>
 #include <lwip/tcp.h>
+#include <lwip/prot/etharp.h>
 #include <securec.h>
 #include <rte_errno.h>
 #include <rte_malloc.h>
@@ -101,7 +102,6 @@ static struct pbuf *init_mbuf_to_pbuf(struct rte_mbuf *mbuf, pbuf_layer layer, u
     struct pbuf *pbuf = pbuf_alloced_custom(layer, length, type, pbuf_custom, data, MAX_PACKET_SZ);
     if (pbuf) {
         pbuf->allow_in = 1;
-        pbuf->last = pbuf;
         pbuf->addr = *IP_ANY_TYPE;
         pbuf->port = 0;
         pthread_spin_init(&pbuf->pbuf_lock, PTHREAD_PROCESS_SHARED);
@@ -122,7 +122,7 @@ static bool replenish_send_idlembuf(struct protocol_stack *stack, struct lwip_so
         return false;
     }
 
-    if (dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, (struct rte_mbuf **)pbuf, replenish_cnt) != 0) {
+    if (dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, (struct rte_mbuf **)pbuf, replenish_cnt, true) != 0) {
         stack->stats.tx_allocmbuf_fail++;
         return true;
     }
@@ -209,10 +209,17 @@ void do_lwip_free_pbuf(struct pbuf *pbuf)
 
 struct pbuf *do_lwip_alloc_pbuf(pbuf_layer layer, uint16_t length, pbuf_type type)
 {
+    int ret;
     struct rte_mbuf *mbuf;
     struct protocol_stack *stack = get_protocol_stack();
 
-    if (dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf, 1) != 0) {
+    /* ensure arp packet can be sent */
+    if (layer == PBUF_LINK && length == SIZEOF_ETHARP_HDR) {
+        ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf, 1, false);
+    } else {
+        ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf, 1, true);
+    }
+    if (ret != 0) {
         stack->stats.tx_allocmbuf_fail++;
         return NULL;
     }
diff --git a/src/lstack/core/lstack_protocol_stack.c b/src/lstack/core/lstack_protocol_stack.c
index 8b99e82..cb39928 100644
--- a/src/lstack/core/lstack_protocol_stack.c
+++ b/src/lstack/core/lstack_protocol_stack.c
@@ -940,7 +940,7 @@ void stack_broadcast_arp(struct rte_mbuf *mbuf, struct protocol_stack *cur_stack
             continue;
         }
 
-        ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1);
+        ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1, true);
         if (ret != 0) {
             stack->stats.rx_allocmbuf_fail++;
             return;
@@ -952,7 +952,7 @@ void stack_broadcast_arp(struct rte_mbuf *mbuf, struct protocol_stack *cur_stack
             return;
         }
     }
-    ret = dpdk_alloc_pktmbuf(cur_stack->rxtx_mbuf_pool, &mbuf_copy, 1);
+    ret = dpdk_alloc_pktmbuf(cur_stack->rxtx_mbuf_pool, &mbuf_copy, 1, true);
     if (ret != 0) {
         cur_stack->stats.rx_allocmbuf_fail++;
         return;
diff --git a/src/lstack/include/lstack_dpdk.h b/src/lstack/include/lstack_dpdk.h
index b39d199..c40f3c0 100644
--- a/src/lstack/include/lstack_dpdk.h
+++ b/src/lstack/include/lstack_dpdk.h
@@ -56,6 +56,6 @@ struct rte_mempool *create_pktmbuf_mempool(const char *name, uint32_t nb_mbuf,
                                            uint32_t mbuf_cache_size, uint16_t queue_id, unsigned numa_id);
 
 void dpdk_nic_xstats_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id);
-int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num);
+int32_t dpdk_alloc_pktmbuf(struct rte_mempool *pool, struct rte_mbuf **mbufs, uint32_t num, bool reserve);
 void dpdk_nic_features_get(struct gazelle_stack_dfx_data *dfx, uint16_t port_id);
 #endif /* GAZELLE_DPDK_H */
diff --git a/src/lstack/include/lstack_protocol_stack.h b/src/lstack/include/lstack_protocol_stack.h
index c681547..0a523b7 100644
--- a/src/lstack/include/lstack_protocol_stack.h
+++ b/src/lstack/include/lstack_protocol_stack.h
@@ -31,7 +31,7 @@
 #define SOCK_SEND_REPLENISH_THRES   (16)
 #define WAKEUP_MAX_NUM              (32)
 
-#define MBUFPOOL_RESERVE_NUM 5000
+#define MBUFPOOL_RESERVE_NUM (get_global_cfg_params()->nic.rxqueue_size + 1024)
 
 struct rte_mempool;
 struct rte_ring;
diff --git a/src/lstack/netif/lstack_ethdev.c b/src/lstack/netif/lstack_ethdev.c
index 4d6f620..acf3c10 100644
--- a/src/lstack/netif/lstack_ethdev.c
+++ b/src/lstack/netif/lstack_ethdev.c
@@ -543,9 +543,9 @@ void parse_arp_and_transefer(char* buf)
     int32_t ret;
     for (int32_t i = 0; i < stack_group->stack_num; i++) {
         stack = stack_group->stacks[i];
-        ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1);
+        ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1, true);
         while (ret != 0) {
-            ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1);
+            ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1, true);
             stack->stats.rx_allocmbuf_fail++;
         }
         copy_mbuf(mbuf_copy, mbuf);
@@ -572,9 +572,9 @@ void parse_tcp_and_transefer(char* buf)
     struct rte_mbuf *mbuf_copy = NULL;
     struct protocol_stack *stack = stack_group->stacks[stk_index];
 
-    int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1);
+    int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1, true);
     while (ret != 0) {
-        ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1);
+        ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, &mbuf_copy, 1, true);
         stack->stats.rx_allocmbuf_fail++;
     }
 
diff --git a/src/lstack/netif/lstack_vdev.c b/src/lstack/netif/lstack_vdev.c
index fe17f59..f78e48a 100644
--- a/src/lstack/netif/lstack_vdev.c
+++ b/src/lstack/netif/lstack_vdev.c
@@ -59,7 +59,7 @@ static uint32_t ltran_rx_poll(struct protocol_stack *stack, struct rte_mbuf **pk
     stack->rx_ring_used += rcvd_pkts;
     if (unlikely(stack->rx_ring_used >= USED_RX_PKTS_WATERMARK)) {
         uint32_t free_cnt = LWIP_MIN(stack->rx_ring_used, RING_SIZE(DPDK_PKT_BURST_SIZE));
-        int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, (struct rte_mbuf **)free_buf, free_cnt);
+        int32_t ret = dpdk_alloc_pktmbuf(stack->rxtx_mbuf_pool, (struct rte_mbuf **)free_buf, free_cnt, true);
         if (likely(ret == 0)) {
             nr_pkts = gazelle_ring_sp_enqueue(stack->rx_ring, (void **)free_buf, free_cnt);
             stack->rx_ring_used -= nr_pkts;
-- 
2.27.0