From: Feifei Wang <wangfeife...@huawei.com>
This patch add package sending and receiving function codes.
Signed-off-by: Feifei Wang <wangfeife...@huawei.com>
Signed-off-by: Yi Chen <chenyi...@huawei.com>
Reviewed-by: Xin Wang <wangxin...@h-partners.com>
---
drivers/net/hinic3/hinic3_ethdev.c | 9 +-
drivers/net/hinic3/hinic3_rx.c | 301 +++++++++++-
drivers/net/hinic3/hinic3_tx.c | 754 +++++++++++++++++++++++++++++
drivers/net/hinic3/hinic3_tx.h | 1 +
4 files changed, 1054 insertions(+), 11 deletions(-)
diff --git a/drivers/net/hinic3/hinic3_ethdev.c
b/drivers/net/hinic3/hinic3_ethdev.c
index de380dddbb..7cd101e5c3 100644
--- a/drivers/net/hinic3/hinic3_ethdev.c
+++ b/drivers/net/hinic3/hinic3_ethdev.c
@@ -21,9 +21,9 @@
#include "base/hinic3_hw_comm.h"
#include "base/hinic3_nic_cfg.h"
#include "base/hinic3_nic_event.h"
-#include "hinic3_pmd_nic_io.h"
-#include "hinic3_pmd_tx.h"
-#include "hinic3_pmd_rx.h"
+#include "hinic3_nic_io.h"
+#include "hinic3_tx.h"
+#include "hinic3_rx.h"
#include "hinic3_ethdev.h"
#define HINIC3_MIN_RX_BUF_SIZE 1024
@@ -3337,6 +3337,9 @@ hinic3_dev_init(struct rte_eth_dev *eth_dev)
PMD_DRV_LOG(INFO, "Network Interface pmd driver version: %s",
HINIC3_PMD_DRV_VERSION);
+ eth_dev->rx_pkt_burst = hinic3_recv_pkts;
+ eth_dev->tx_pkt_burst = hinic3_xmit_pkts;
+
return hinic3_func_init(eth_dev);
}
diff --git a/drivers/net/hinic3/hinic3_rx.c b/drivers/net/hinic3/hinic3_rx.c
index a1dc960236..318d9aadc3 100644
--- a/drivers/net/hinic3/hinic3_rx.c
+++ b/drivers/net/hinic3/hinic3_rx.c
@@ -5,14 +5,14 @@
#include <rte_mbuf.h>
#include "base/hinic3_compat.h"
-#include "base/hinic3_pmd_hwif.h"
-#include "base/hinic3_pmd_hwdev.h"
-#include "base/hinic3_pmd_wq.h"
-#include "base/hinic3_pmd_nic_cfg.h"
-#include "hinic3_pmd_nic_io.h"
-#include "hinic3_pmd_ethdev.h"
-#include "hinic3_pmd_tx.h"
-#include "hinic3_pmd_rx.h"
+#include "base/hinic3_hwif.h"
+#include "base/hinic3_hwdev.h"
+#include "base/hinic3_wq.h"
+#include "base/hinic3_nic_cfg.h"
+#include "hinic3_nic_io.h"
+#include "hinic3_ethdev.h"
+#include "hinic3_tx.h"
+#include "hinic3_rx.h"
/**
* Get wqe from receive queue.
@@ -809,3 +809,288 @@ hinic3_start_rq(struct rte_eth_dev *eth_dev, struct
hinic3_rxq *rxq)
return err;
}
+
+
+static inline u64
+hinic3_rx_vlan(u32 offload_type, u32 vlan_len, u16 *vlan_tci)
+{
+ uint16_t vlan_tag;
+
+ vlan_tag = HINIC3_GET_RX_VLAN_TAG(vlan_len);
+ if (!HINIC3_GET_RX_VLAN_OFFLOAD_EN(offload_type) || vlan_tag == 0) {
+ *vlan_tci = 0;
+ return 0;
+ }
+
+ *vlan_tci = vlan_tag;
+
+ return HINIC3_PKT_RX_VLAN | HINIC3_PKT_RX_VLAN_STRIPPED;
+}
+
+static inline u64
+hinic3_rx_csum(uint32_t status, struct hinic3_rxq *rxq)
+{
+ struct hinic3_nic_dev *nic_dev = rxq->nic_dev;
+ u32 csum_err;
+ u64 flags;
+
+ if (unlikely(!(nic_dev->rx_csum_en & HINIC3_DEFAULT_RX_CSUM_OFFLOAD)))
+ return HINIC3_PKT_RX_IP_CKSUM_UNKNOWN;
+
+ csum_err = HINIC3_GET_RX_CSUM_ERR(status);
+ if (likely(csum_err == 0))
+ return (HINIC3_PKT_RX_IP_CKSUM_GOOD |
+ HINIC3_PKT_RX_L4_CKSUM_GOOD);
+
+ /*
+ * If bypass bit is set, all other err status indications should be
+ * ignored.
+ */
+ if (unlikely(csum_err & HINIC3_RX_CSUM_HW_CHECK_NONE))
+ return HINIC3_PKT_RX_IP_CKSUM_UNKNOWN;
+
+ flags = 0;
+
+ /* IP checksum error. */
+ if (csum_err & HINIC3_RX_CSUM_IP_CSUM_ERR) {
+ flags |= HINIC3_PKT_RX_IP_CKSUM_BAD;
+ rxq->rxq_stats.csum_errors++;
+ }
+
+ /* L4 checksum error. */
+ if ((csum_err & HINIC3_RX_CSUM_TCP_CSUM_ERR) ||
+ (csum_err & HINIC3_RX_CSUM_UDP_CSUM_ERR) ||
+ (csum_err & HINIC3_RX_CSUM_SCTP_CRC_ERR)) {
+ flags |= HINIC3_PKT_RX_L4_CKSUM_BAD;
+ rxq->rxq_stats.csum_errors++;
+ }
+
+ if (unlikely(csum_err == HINIC3_RX_CSUM_IPSU_OTHER_ERR))
+ rxq->rxq_stats.other_errors++;
+
+ return flags;
+}
+
+static inline u64
+hinic3_rx_rss_hash(u32 offload_type, u32 rss_hash_value, u32 *rss_hash)
+{
+ u32 rss_type;
+
+ rss_type = HINIC3_GET_RSS_TYPES(offload_type);
+ if (likely(rss_type != 0)) {
+ *rss_hash = rss_hash_value;
+ return HINIC3_PKT_RX_RSS_HASH;
+ }
+
+ return 0;
+}
+
+static void
+hinic3_recv_jumbo_pkt(struct hinic3_rxq *rxq, struct rte_mbuf *head_mbuf,
+ u32 remain_pkt_len)
+{
+ struct rte_mbuf *cur_mbuf = NULL;
+ struct rte_mbuf *rxm = NULL;
+ struct hinic3_rx_info *rx_info = NULL;
+ u16 sw_ci, rx_buf_len = rxq->buf_len;
+ u32 pkt_len;
+
+ while (remain_pkt_len > 0) {
+ sw_ci = hinic3_get_rq_local_ci(rxq);
+ rx_info = &rxq->rx_info[sw_ci];
+
+ hinic3_update_rq_local_ci(rxq, 1);
+
+ pkt_len = remain_pkt_len > rx_buf_len ? rx_buf_len
+ : remain_pkt_len;
+ remain_pkt_len -= pkt_len;
+
+ cur_mbuf = rx_info->mbuf;
+ cur_mbuf->data_len = (u16)pkt_len;
+ cur_mbuf->next = NULL;
+
+ head_mbuf->pkt_len += cur_mbuf->data_len;
+ head_mbuf->nb_segs++;
+#ifdef HINIC3_XSTAT_MBUF_USE
+ rxq->rxq_stats.rx_free_mbuf_bytes++;
+#endif
+ if (!rxm)
+ head_mbuf->next = cur_mbuf;
+ else
+ rxm->next = cur_mbuf;
+
+ rxm = cur_mbuf;
+ }
+}
+
+int
+hinic3_start_all_rqs(struct rte_eth_dev *eth_dev)
+{
+ struct hinic3_nic_dev *nic_dev = NULL;
+ struct hinic3_rxq *rxq = NULL;
+ int err = 0;
+ int i;
+
+ nic_dev = HINIC3_ETH_DEV_TO_PRIVATE_NIC_DEV(eth_dev);
+
+ for (i = 0; i < nic_dev->num_rqs; i++) {
+ rxq = eth_dev->data->rx_queues[i];
+ hinic3_add_rq_to_rx_queue_list(nic_dev, rxq->q_id);
+ err = hinic3_rearm_rxq_mbuf(rxq);
+ if (err) {
+ PMD_DRV_LOG(ERR,
+ "Fail to alloc mbuf for Rx queue %d, "
+ "qid = %u, need_mbuf: %d",
+ i, rxq->q_id, rxq->q_depth);
+ goto out;
+ }
+ hinic3_dev_rx_queue_intr_enable(eth_dev, rxq->q_id);
+ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STARTED;
+ }
+
+ if (nic_dev->rss_state == HINIC3_RSS_ENABLE) {
+ err = hinic3_refill_indir_rqid(rxq);
+ if (err) {
+ PMD_DRV_LOG(ERR,
+ "Refill rq to indrect table failed, "
+ "eth_dev:%s, queue_idx:%d, err:%d",
+ rxq->nic_dev->dev_name, rxq->q_id, err);
+ goto out;
+ }
+ }
+
+ return 0;
+out:
+ for (i = 0; i < nic_dev->num_rqs; i++) {
+ rxq = eth_dev->data->rx_queues[i];
+ hinic3_remove_rq_from_rx_queue_list(nic_dev, rxq->q_id);
+ hinic3_free_rxq_mbufs(rxq);
+ hinic3_dev_rx_queue_intr_disable(eth_dev, rxq->q_id);
+ eth_dev->data->rx_queue_state[i] = RTE_ETH_QUEUE_STATE_STOPPED;
+ }
+ return err;
+}
+
+#define HINIC3_RX_EMPTY_THRESHOLD 3
+u16
+hinic3_recv_pkts(void *rx_queue, struct rte_mbuf **rx_pkts, u16 nb_pkts)
+{
+ struct hinic3_rxq *rxq = rx_queue;
+ struct hinic3_rx_info *rx_info = NULL;
+ volatile struct hinic3_rq_cqe *rx_cqe = NULL;
+ struct rte_mbuf *rxm = NULL;
+ u16 sw_ci, rx_buf_len, wqebb_cnt = 0, pkts = 0;
+ u32 status, pkt_len, vlan_len, offload_type, lro_num;
+ u64 rx_bytes = 0;
+ u32 hash_value;
+
+#ifdef HINIC3_XSTAT_PROF_RX
+ uint64_t t1 = rte_get_tsc_cycles();
+ uint64_t t2;
+#endif
+ if (((rte_get_timer_cycles() - rxq->rxq_stats.tsc) <
rxq->wait_time_cycle) &&
+ rxq->rxq_stats.empty >= HINIC3_RX_EMPTY_THRESHOLD)
+ goto out;
+
+ sw_ci = hinic3_get_rq_local_ci(rxq);
+ rx_buf_len = rxq->buf_len;
+
+ while (pkts < nb_pkts) {
+ rx_cqe = &rxq->rx_cqe[sw_ci];
+ status =
hinic3_hw_cpu32((u32)(rte_atomic_load_explicit(&rx_cqe->status,
+ rte_memory_order_acquire)));
+ if (!HINIC3_GET_RX_DONE(status)) {
+ rxq->rxq_stats.empty++;
+ break;
+ }
+
+ vlan_len = hinic3_hw_cpu32(rx_cqe->vlan_len);
+
+ pkt_len = HINIC3_GET_RX_PKT_LEN(vlan_len);
+
+ rx_info = &rxq->rx_info[sw_ci];
+ rxm = rx_info->mbuf;
+
+ /* 1. Next ci point and prefetch. */
+ sw_ci++;
+ sw_ci &= rxq->q_mask;
+
+ /* 2. Prefetch next mbuf first 64B. */
+ rte_prefetch0(rxq->rx_info[sw_ci].mbuf);
+
+ /* 3. Jumbo frame process. */
+ if (likely(pkt_len <= (u32)rx_buf_len)) {
+ rxm->data_len = (u16)pkt_len;
+ rxm->pkt_len = pkt_len;
+ wqebb_cnt++;
+ } else {
+ rxm->data_len = rx_buf_len;
+ rxm->pkt_len = rx_buf_len;
+
+ /*
+ * If receive jumbo, updating ci will be done by
+ * hinic3_recv_jumbo_pkt function.
+ */
+ hinic3_update_rq_local_ci(rxq, wqebb_cnt + 1);
+ wqebb_cnt = 0;
+ hinic3_recv_jumbo_pkt(rxq, rxm, pkt_len - rx_buf_len);
+ sw_ci = hinic3_get_rq_local_ci(rxq);
+ }
+
+ rxm->data_off = RTE_PKTMBUF_HEADROOM;
+ rxm->port = rxq->port_id;
+
+ /* 4. Rx checksum offload. */
+ rxm->ol_flags |= hinic3_rx_csum(status, rxq);
+
+ /* 5. Vlan offload. */
+ offload_type = hinic3_hw_cpu32(rx_cqe->offload_type);
+
+ rxm->ol_flags |=
+ hinic3_rx_vlan(offload_type, vlan_len, &rxm->vlan_tci);
+
+ /* 6. RSS. */
+ hash_value = hinic3_hw_cpu32(rx_cqe->hash_val);
+ rxm->ol_flags |= hinic3_rx_rss_hash(offload_type, hash_value,
+ &rxm->hash.rss);
+ /* 8. LRO. */
+ lro_num = HINIC3_GET_RX_NUM_LRO(status);
+ if (unlikely(lro_num != 0)) {
+ rxm->ol_flags |= HINIC3_PKT_RX_LRO;
+ rxm->tso_segsz = pkt_len / lro_num;
+ }
+
+ rx_cqe->status = 0;
+
+ rx_bytes += pkt_len;
+ rx_pkts[pkts++] = rxm;
+ }
+
+ if (pkts) {
+ /* 9. Update local ci. */
+ hinic3_update_rq_local_ci(rxq, wqebb_cnt);
+
+ /* Update packet stats. */
+ rxq->rxq_stats.packets += pkts;
+ rxq->rxq_stats.bytes += rx_bytes;
+ rxq->rxq_stats.empty = 0;
+#ifdef HINIC3_XSTAT_MBUF_USE
+ rxq->rxq_stats.rx_free_mbuf_bytes += pkts;
+#endif
+ }
+ rxq->rxq_stats.burst_pkts = pkts;
+ rxq->rxq_stats.tsc = rte_get_timer_cycles();
+out:
+ /* 10. Rearm mbuf to rxq. */
+ hinic3_rearm_rxq_mbuf(rxq);
+
+#ifdef HINIC3_XSTAT_PROF_RX
+ /* Do profiling stats. */
+ t2 = rte_get_tsc_cycles();
+ rxq->rxq_stats.app_tsc = t1 - rxq->prof_rx_end_tsc;
+ rxq->prof_rx_end_tsc = t2;
+ rxq->rxq_stats.pmd_tsc = t2 - t1;
+#endif
+
+ return pkts;
+}
diff --git a/drivers/net/hinic3/hinic3_tx.c b/drivers/net/hinic3/hinic3_tx.c
index 6f8c42e0c3..c2157ab4b9 100644
--- a/drivers/net/hinic3/hinic3_tx.c
+++ b/drivers/net/hinic3/hinic3_tx.c
@@ -60,6 +60,98 @@ hinic3_get_sq_hw_ci(struct hinic3_txq *sq)
return MASKED_QUEUE_IDX(sq, hinic3_hw_cpu16(*sq->ci_vaddr_base));
}
+static void *
+hinic3_get_sq_wqe(struct hinic3_txq *sq, struct hinic3_wqe_info *wqe_info)
+{
+ u16 cur_pi = MASKED_QUEUE_IDX(sq, sq->prod_idx);
+ u32 end_pi;
+
+ end_pi = cur_pi + wqe_info->wqebb_cnt;
+ sq->prod_idx += wqe_info->wqebb_cnt;
+
+ wqe_info->owner = (u8)(sq->owner);
+ wqe_info->pi = cur_pi;
+ wqe_info->wrapped = 0;
+
+ if (unlikely(end_pi >= sq->q_depth)) {
+ sq->owner = !sq->owner;
+
+ if (likely(end_pi > sq->q_depth))
+ wqe_info->wrapped = (u8)(sq->q_depth - cur_pi);
+ }
+
+ return NIC_WQE_ADDR(sq, cur_pi);
+}
+
+static inline void
+hinic3_put_sq_wqe(struct hinic3_txq *sq, struct hinic3_wqe_info *wqe_info)
+{
+ if (wqe_info->owner != sq->owner)
+ sq->owner = wqe_info->owner;
+
+ sq->prod_idx -= wqe_info->wqebb_cnt;
+}
+
+/**
+ * Sets the WQE combination information in the transmit queue (SQ).
+ *
+ * @param[in] txq
+ * Point to send queue.
+ * @param[out] wqe_combo
+ * Point to wqe_combo of send queue(SQ).
+ * @param[in] wqe
+ * Point to wqe of send queue(SQ).
+ * @param[in] wqe_info
+ * Point to wqe_info of send queue(SQ).
+ */
+static void
+hinic3_set_wqe_combo(struct hinic3_txq *txq,
+ struct hinic3_sq_wqe_combo *wqe_combo,
+ struct hinic3_sq_wqe *wqe,
+ struct hinic3_wqe_info *wqe_info)
+{
+ wqe_combo->hdr = &wqe->compact_wqe.wqe_desc;
+
+ if (wqe_info->offload) {
+ if (wqe_info->wrapped == HINIC3_TX_TASK_WRAPPED) {
+ wqe_combo->task = (struct hinic3_sq_task *)
+ (void *)txq->sq_head_addr;
+ wqe_combo->bds_head = (struct hinic3_sq_bufdesc *)
+ (void *)(txq->sq_head_addr + txq->wqebb_size);
+ } else if (wqe_info->wrapped == HINIC3_TX_BD_DESC_WRAPPED) {
+ wqe_combo->task = &wqe->extend_wqe.task;
+ wqe_combo->bds_head = (struct hinic3_sq_bufdesc *)
+ (void *)(txq->sq_head_addr);
+ } else {
+ wqe_combo->task = &wqe->extend_wqe.task;
+ wqe_combo->bds_head = wqe->extend_wqe.buf_desc;
+ }
+
+ wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE;
+ wqe_combo->task_type = SQ_WQE_TASKSECT_16BYTES;
+
+ return;
+ }
+
+ if (wqe_info->wrapped == HINIC3_TX_TASK_WRAPPED) {
+ wqe_combo->bds_head = (struct hinic3_sq_bufdesc *)
+ (void *)(txq->sq_head_addr);
+ } else {
+ wqe_combo->bds_head =
+ (struct hinic3_sq_bufdesc *)(&wqe->extend_wqe.task);
+ }
+
+ if (wqe_info->wqebb_cnt > 1) {
+ wqe_combo->wqe_type = SQ_WQE_EXTENDED_TYPE;
+ wqe_combo->task_type = SQ_WQE_TASKSECT_46BITS;
+
+ /* This section used as vlan insert, needs to clear. */
+ wqe_combo->bds_head->rsvd = 0;
+ } else {
+ wqe_combo->wqe_type = SQ_WQE_COMPACT_TYPE;
+ }
+}
+
int
hinic3_start_all_sqs(struct rte_eth_dev *eth_dev)
{
@@ -220,6 +312,668 @@ hinic3_tx_done_cleanup(void *txq, u32 free_cnt)
return hinic3_xmit_mbuf_cleanup(tx_queue, try_free_cnt);
}
+/**
+ * Prepare the data packet to be sent and calculate the internal L3 offset.
+ *
+ * @param[in] mbuf
+ * Point to the mbuf to be processed.
+ * @param[out] inner_l3_offset
+ * Inner(IP Layer) L3 layer offset.
+ * @return
+ * 0 as success, -EINVAL as failure.
+ */
+static int
+hinic3_tx_offload_pkt_prepare(struct rte_mbuf *mbuf, u16 *inner_l3_offset)
+{
+ uint64_t ol_flags = mbuf->ol_flags;
+
+ /* Only support vxlan offload. */
+ if ((ol_flags & HINIC3_PKT_TX_TUNNEL_MASK) &&
+ (!(ol_flags & HINIC3_PKT_TX_TUNNEL_VXLAN)))
+ return -EINVAL;
+
+#ifdef RTE_LIBRTE_ETHDEV_DEBUG
+ if (rte_validate_tx_offload(mbuf) != 0)
+ return -EINVAL;
+#endif
+ /* Support tunnel. */
+ if ((ol_flags & HINIC3_PKT_TX_TUNNEL_MASK)) {
+ if ((ol_flags & HINIC3_PKT_TX_OUTER_IP_CKSUM) ||
+ (ol_flags & HINIC3_PKT_TX_OUTER_IPV6) ||
+ (ol_flags & HINIC3_PKT_TX_TCP_SEG)) {
+ /*
+ * For this senmatic, l2_len of mbuf means
+ * len(out_udp + vxlan + in_eth).
+ */
+ *inner_l3_offset = mbuf->l2_len + mbuf->outer_l2_len +
+ mbuf->outer_l3_len;
+ } else {
+ /*
+ * For this senmatic, l2_len of mbuf means
+ * len(out_eth + out_ip + out_udp + vxlan + in_eth).
+ */
+ *inner_l3_offset = mbuf->l2_len;
+ }
+ } else {
+ /* For non-tunnel type pkts. */
+ *inner_l3_offset = mbuf->l2_len;
+ }
+
+ return 0;
+}
+
+static inline void
+hinic3_set_vlan_tx_offload(struct hinic3_sq_task *task, u16 vlan_tag,
+ u8 vlan_type)
+{
+ task->vlan_offload = SQ_TASK_INFO3_SET(vlan_tag, VLAN_TAG) |
+ SQ_TASK_INFO3_SET(vlan_type, VLAN_TYPE) |
+ SQ_TASK_INFO3_SET(1U, VLAN_TAG_VALID);
+}
+
+/**
+ * Set the corresponding offload information based on ol_flags of the mbuf.
+ *
+ * @param[in] mbuf
+ * Point to the mbuf for which offload needs to be set in the sending queue.
+ * @param[out] task
+ * Point to task of send queue(SQ).
+ * @param[out] wqe_info
+ * Point to wqe_info of send queue(SQ).
+ * @return
+ * 0 as success, -EINVAL as failure.
+ */
+static int
+hinic3_set_tx_offload(struct rte_mbuf *mbuf, struct hinic3_sq_task *task,
+ struct hinic3_wqe_info *wqe_info)
+{
+ uint64_t ol_flags = mbuf->ol_flags;
+ u16 pld_offset = 0;
+ u32 queue_info = 0;
+ u16 vlan_tag;
+
+ task->pkt_info0 = 0;
+ task->ip_identify = 0;
+ task->pkt_info2 = 0;
+ task->vlan_offload = 0;
+
+ /* Vlan offload. */
+ if (unlikely(ol_flags & HINIC3_PKT_TX_VLAN_PKT)) {
+ vlan_tag = mbuf->vlan_tci;
+ hinic3_set_vlan_tx_offload(task, vlan_tag, HINIC3_TX_TPID0);
+ task->vlan_offload = hinic3_hw_be32(task->vlan_offload);
+ }
+ /* Cksum offload. */
+ if (!(ol_flags & HINIC3_TX_CKSUM_OFFLOAD_MASK))
+ return 0;
+
+ /* Tso offload. */
+ if (ol_flags & HINIC3_PKT_TX_TCP_SEG) {
+ pld_offset = wqe_info->payload_offset;
+ if ((pld_offset >> 1) > MAX_PAYLOAD_OFFSET)
+ return -EINVAL;
+
+ task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L4_EN);
+ task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L3_EN);
+
+ queue_info |= SQ_CTRL_QUEUE_INFO_SET(1U, TSO);
+ queue_info |= SQ_CTRL_QUEUE_INFO_SET(pld_offset >> 1, PLDOFF);
+
+ /* Set MSS value. */
+ queue_info = SQ_CTRL_QUEUE_INFO_CLEAR(queue_info, MSS);
+ queue_info |= SQ_CTRL_QUEUE_INFO_SET(mbuf->tso_segsz, MSS);
+ } else {
+ if (ol_flags & HINIC3_PKT_TX_IP_CKSUM)
+ task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L3_EN);
+
+ switch (ol_flags & HINIC3_PKT_TX_L4_MASK) {
+ case HINIC3_PKT_TX_TCP_CKSUM:
+ case HINIC3_PKT_TX_UDP_CKSUM:
+ case HINIC3_PKT_TX_SCTP_CKSUM:
+ task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, INNER_L4_EN);
+ break;
+
+ case HINIC3_PKT_TX_L4_NO_CKSUM:
+ break;
+
+ default:
+ PMD_DRV_LOG(INFO, "not support pkt type");
+ return -EINVAL;
+ }
+ }
+
+ /* For vxlan, also can support PKT_TX_TUNNEL_GRE, etc. */
+ switch (ol_flags & HINIC3_PKT_TX_TUNNEL_MASK) {
+ case HINIC3_PKT_TX_TUNNEL_VXLAN:
+ task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, TUNNEL_FLAG);
+ break;
+
+ case 0:
+ break;
+
+ default:
+ /* For non UDP/GRE tunneling, drop the tunnel packet. */
+ PMD_DRV_LOG(INFO, "not support tunnel pkt type");
+ return -EINVAL;
+ }
+
+ if (ol_flags & HINIC3_PKT_TX_OUTER_IP_CKSUM)
+ task->pkt_info0 |= SQ_TASK_INFO0_SET(1U, OUT_L3_EN);
+
+ task->pkt_info0 = hinic3_hw_be32(task->pkt_info0);
+ task->pkt_info2 = hinic3_hw_be32(task->pkt_info2);
+ wqe_info->queue_info = queue_info;
+
+ return 0;
+}
+
+/**
+ * Check whether the number of segments in the mbuf is valid.
+ *
+ * @param[in] mbuf
+ * Point to the mbuf to be verified.
+ * @param[in] wqe_info
+ * Point to wqe_info of send queue(SQ).
+ * @return
+ * true as valid, false as invalid.
+ */
+static bool
+hinic3_is_tso_sge_valid(struct rte_mbuf *mbuf, struct hinic3_wqe_info
*wqe_info)
+{
+ u32 total_len, limit_len, checked_len, left_len, adjust_mss;
+ u32 i, max_sges, left_sges, first_len;
+ struct rte_mbuf *mbuf_head, *mbuf_first;
+ struct rte_mbuf *mbuf_pre = mbuf;
+
+ left_sges = mbuf->nb_segs;
+ mbuf_head = mbuf;
+ mbuf_first = mbuf;
+
+ /* Tso sge number validation. */
+ if (unlikely(left_sges >= HINIC3_NONTSO_PKT_MAX_SGE)) {
+ checked_len = 0;
+ total_len = 0;
+ first_len = 0;
+ adjust_mss = mbuf->tso_segsz >= TX_MSS_MIN ? mbuf->tso_segsz
+ : TX_MSS_MIN;
+ max_sges = HINIC3_NONTSO_PKT_MAX_SGE - 1;
+ limit_len = adjust_mss + wqe_info->payload_offset;
+
+ for (i = 0; (i < max_sges) && (total_len < limit_len); i++) {
+ total_len += mbuf->data_len;
+ mbuf_pre = mbuf;
+ mbuf = mbuf->next;
+ }
+
+ /* Each continues 38 mbufs segmust do one check. */
+ while (left_sges >= HINIC3_NONTSO_PKT_MAX_SGE) {
+ if (total_len >= limit_len) {
+ /* Update the limit len. */
+ limit_len = adjust_mss;
+ /* Update checked len. */
+ checked_len += first_len;
+ /* Record the first len. */
+ first_len = mbuf_first->data_len;
+ /* First mbuf move to the next. */
+ mbuf_first = mbuf_first->next;
+ /* Update total len. */
+ total_len -= first_len;
+ left_sges--;
+ i--;
+ for (;
+ (i < max_sges) && (total_len < limit_len);
+ i++) {
+ total_len += mbuf->data_len;
+ mbuf_pre = mbuf;
+ mbuf = mbuf->next;
+ }
+ } else {
+ /* Try to copy if not valid. */
+ checked_len += (total_len - mbuf_pre->data_len);
+
+ left_len = mbuf_head->pkt_len - checked_len;
+ if (left_len > HINIC3_COPY_MBUF_SIZE)
+ return false;
+ wqe_info->sge_cnt = (u16)(mbuf_head->nb_segs +
+ i - left_sges);
+ wqe_info->cpy_mbuf_cnt = 1;
+
+ return true;
+ }
+ } /**< End of while. */
+ }
+
+ wqe_info->sge_cnt = mbuf_head->nb_segs;
+
+ return true;
+}
+
+/**
+ * Checks and processes transport offload information for data packets.
+ *
+ * @param[in] mbuf
+ * Point to the mbuf to send.
+ * @param[in] wqe_info
+ * Point to wqe_info of send queue(SQ).
+ * @return
+ * 0 as success, -EINVAL as failure.
+ */
+static int
+hinic3_get_tx_offload(struct rte_mbuf *mbuf, struct hinic3_wqe_info *wqe_info)
+{
+ uint64_t ol_flags = mbuf->ol_flags;
+ u16 i, total_len, inner_l3_offset = 0;
+ int err;
+ struct rte_mbuf *mbuf_pkt = NULL;
+
+ wqe_info->sge_cnt = mbuf->nb_segs;
+ /* Check if the packet set available offload flags. */
+ if (!(ol_flags & HINIC3_TX_OFFLOAD_MASK)) {
+ wqe_info->offload = 0;
+ return 0;
+ }
+
+ wqe_info->offload = 1;
+ err = hinic3_tx_offload_pkt_prepare(mbuf, &inner_l3_offset);
+ if (err)
+ return err;
+
+ /* Non tso mbuf only check sge num. */
+ if (likely(!(mbuf->ol_flags & HINIC3_PKT_TX_TCP_SEG))) {
+ if (unlikely(mbuf->pkt_len > MAX_SINGLE_SGE_SIZE))
+ /* Non tso packet len must less than 64KB. */
+ return -EINVAL;
+
+ if (likely(HINIC3_NONTSO_SEG_NUM_VALID(mbuf->nb_segs)))
+ /* Valid non-tso mbuf. */
+ return 0;
+
+ /*
+ * The number of non-tso packet fragments must be less than 38,
+ * and mbuf segs greater than 38 must be copied to other
+ * buffers.
+ */
+ total_len = 0;
+ mbuf_pkt = mbuf;
+ for (i = 0; i < (HINIC3_NONTSO_PKT_MAX_SGE - 1); i++) {
+ total_len += mbuf_pkt->data_len;
+ mbuf_pkt = mbuf_pkt->next;
+ }
+
+ /* Default support copy total 4k mbuf segs. */
+ if ((u32)(total_len + (u16)HINIC3_COPY_MBUF_SIZE) <
+ mbuf->pkt_len)
+ return -EINVAL;
+
+ wqe_info->sge_cnt = HINIC3_NONTSO_PKT_MAX_SGE;
+ wqe_info->cpy_mbuf_cnt = 1;
+
+ return 0;
+ }
+
+ /* Tso mbuf. */
+ wqe_info->payload_offset =
+ inner_l3_offset + mbuf->l3_len + mbuf->l4_len;
+
+ /* Too many mbuf segs. */
+ if (unlikely(HINIC3_TSO_SEG_NUM_INVALID(mbuf->nb_segs)))
+ return -EINVAL;
+
+ /* Check whether can cover all tso mbuf segs or not. */
+ if (unlikely(!hinic3_is_tso_sge_valid(mbuf, wqe_info)))
+ return -EINVAL;
+
+ return 0;
+}
+
+static inline void
+hinic3_set_buf_desc(struct hinic3_sq_bufdesc *buf_descs, rte_iova_t addr,
+ u32 len)
+{
+ buf_descs->hi_addr = hinic3_hw_be32(upper_32_bits(addr));
+ buf_descs->lo_addr = hinic3_hw_be32(lower_32_bits(addr));
+ buf_descs->len = hinic3_hw_be32(len);
+}
+
+static inline struct rte_mbuf *
+hinic3_alloc_cpy_mbuf(struct hinic3_nic_dev *nic_dev)
+{
+ return rte_pktmbuf_alloc(nic_dev->cpy_mpool);
+}
+
+/**
+ * Copy packets in the send queue(SQ).
+ *
+ * @param[in] nic_dev
+ * Point to nic device.
+ * @param[in] mbuf
+ * Point to the source mbuf.
+ * @param[in] seg_cnt
+ * Number of mbuf segments to be copied.
+ * @result
+ * The address of the copied mbuf.
+ */
+static void *
+hinic3_copy_tx_mbuf(struct hinic3_nic_dev *nic_dev, struct rte_mbuf *mbuf,
+ u16 sge_cnt)
+{
+ struct rte_mbuf *dst_mbuf;
+ u32 offset = 0;
+ u16 i;
+
+ if (unlikely(!nic_dev->cpy_mpool))
+ return NULL;
+
+ dst_mbuf = hinic3_alloc_cpy_mbuf(nic_dev);
+ if (unlikely(!dst_mbuf))
+ return NULL;
+
+ dst_mbuf->data_off = 0;
+ dst_mbuf->data_len = 0;
+ for (i = 0; i < sge_cnt; i++) {
+ rte_memcpy((u8 *)dst_mbuf->buf_addr + offset,
+ (u8 *)mbuf->buf_addr + mbuf->data_off,
+ mbuf->data_len);
+ dst_mbuf->data_len += mbuf->data_len;
+ offset += mbuf->data_len;
+ mbuf = mbuf->next;
+ }
+ dst_mbuf->pkt_len = dst_mbuf->data_len;
+
+ return dst_mbuf;
+}
+
+/**
+ * Map the TX mbuf to the DMA address space and set related information for
+ * subsequent DMA transmission.
+ *
+ * @param[in] txq
+ * Point to send queue.
+ * @param[in] mbuf
+ * Point to the tx mbuf.
+ * @param[out] wqe_combo
+ * Point to send queue wqe_combo.
+ * @param[in] wqe_info
+ * Point to wqe_info of send queue(SQ).
+ * @result
+ * 0 as success, -EINVAL as failure.
+ */
+static int
+hinic3_mbuf_dma_map_sge(struct hinic3_txq *txq, struct rte_mbuf *mbuf,
+ struct hinic3_sq_wqe_combo *wqe_combo,
+ struct hinic3_wqe_info *wqe_info)
+{
+ struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->hdr;
+ struct hinic3_sq_bufdesc *buf_desc = wqe_combo->bds_head;
+
+ uint16_t nb_segs = wqe_info->sge_cnt - wqe_info->cpy_mbuf_cnt;
+ uint16_t real_segs = mbuf->nb_segs;
+ rte_iova_t dma_addr;
+ u32 i;
+
+ for (i = 0; i < nb_segs; i++) {
+ if (unlikely(mbuf == NULL)) {
+ txq->txq_stats.mbuf_null++;
+ return -EINVAL;
+ }
+
+ if (unlikely(mbuf->data_len == 0)) {
+ txq->txq_stats.sge_len0++;
+ return -EINVAL;
+ }
+
+ dma_addr = rte_mbuf_data_iova(mbuf);
+ if (i == 0) {
+ if (wqe_combo->wqe_type == SQ_WQE_COMPACT_TYPE &&
+ mbuf->data_len > COMPACT_WQE_MAX_CTRL_LEN) {
+ txq->txq_stats.sge_len_too_large++;
+ return -EINVAL;
+ }
+
+ wqe_desc->hi_addr =
+ hinic3_hw_be32(upper_32_bits(dma_addr));
+ wqe_desc->lo_addr =
+ hinic3_hw_be32(lower_32_bits(dma_addr));
+ wqe_desc->ctrl_len = mbuf->data_len;
+ } else {
+ /*
+ * Parts of wqe is in sq bottom while parts
+ * of wqe is in sq head.
+ */
+ if (unlikely(wqe_info->wrapped &&
+ (u64)buf_desc == txq->sq_bot_sge_addr))
+ buf_desc = (struct hinic3_sq_bufdesc *)
+ (void *)txq->sq_head_addr;
+
+ hinic3_set_buf_desc(buf_desc, dma_addr, mbuf->data_len);
+ buf_desc++;
+ }
+ mbuf = mbuf->next;
+ }
+
+ /* For now: support over 38 sge, copy the last 2 mbuf. */
+ if (unlikely(wqe_info->cpy_mbuf_cnt != 0)) {
+ /*
+ * Copy invalid mbuf segs to a valid buffer, lost performance.
+ */
+ txq->txq_stats.cpy_pkts += 1;
+ mbuf = hinic3_copy_tx_mbuf(txq->nic_dev, mbuf,
+ real_segs - nb_segs);
+ if (unlikely(!mbuf))
+ return -EINVAL;
+
+ txq->tx_info[wqe_info->pi].cpy_mbuf = mbuf;
+
+ /* Deal with the last mbuf. */
+ dma_addr = rte_mbuf_data_iova(mbuf);
+ if (unlikely(mbuf->data_len == 0)) {
+ txq->txq_stats.sge_len0++;
+ return -EINVAL;
+ }
+ /*
+ * Parts of wqe is in sq bottom while parts
+ * of wqe is in sq head.
+ */
+ if (i == 0) {
+ wqe_desc->hi_addr =
+ hinic3_hw_be32(upper_32_bits(dma_addr));
+ wqe_desc->lo_addr =
+ hinic3_hw_be32(lower_32_bits(dma_addr));
+ wqe_desc->ctrl_len = mbuf->data_len;
+ } else {
+ if (unlikely(wqe_info->wrapped &&
+ ((u64)buf_desc == txq->sq_bot_sge_addr)))
+ buf_desc = (struct hinic3_sq_bufdesc *)
+ txq->sq_head_addr;
+
+ hinic3_set_buf_desc(buf_desc, dma_addr, mbuf->data_len);
+ }
+ }
+
+ return 0;
+}
+
+/**
+ * Sets and configures fields in the transmit queue control descriptor based on
+ * the WQE type.
+ *
+ * @param[out] wqe_combo
+ * Point to wqe_combo of send queue.
+ * @param[in] wqe_info
+ * Point to wqe_info of send queue.
+ */
+static void
+hinic3_prepare_sq_ctrl(struct hinic3_sq_wqe_combo *wqe_combo,
+ struct hinic3_wqe_info *wqe_info)
+{
+ struct hinic3_sq_wqe_desc *wqe_desc = wqe_combo->hdr;
+
+ if (wqe_combo->wqe_type == SQ_WQE_COMPACT_TYPE) {
+ wqe_desc->ctrl_len |=
+ SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) |
+ SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) |
+ SQ_CTRL_SET(wqe_info->owner, OWNER);
+ wqe_desc->ctrl_len = hinic3_hw_be32(wqe_desc->ctrl_len);
+
+ /* Compact wqe queue_info will transfer to ucode. */
+ wqe_desc->queue_info = 0;
+
+ return;
+ }
+
+ wqe_desc->ctrl_len |= SQ_CTRL_SET(wqe_info->sge_cnt, BUFDESC_NUM) |
+ SQ_CTRL_SET(wqe_combo->task_type, TASKSECT_LEN) |
+ SQ_CTRL_SET(SQ_NORMAL_WQE, DATA_FORMAT) |
+ SQ_CTRL_SET(wqe_combo->wqe_type, EXTENDED) |
+ SQ_CTRL_SET(wqe_info->owner, OWNER);
+
+ wqe_desc->ctrl_len = hinic3_hw_be32(wqe_desc->ctrl_len);
+
+ wqe_desc->queue_info = wqe_info->queue_info;
+ wqe_desc->queue_info |= SQ_CTRL_QUEUE_INFO_SET(1U, UC);
+
+ if (!SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS)) {
+ wqe_desc->queue_info |=
+ SQ_CTRL_QUEUE_INFO_SET(TX_MSS_DEFAULT, MSS);
+ } else if (SQ_CTRL_QUEUE_INFO_GET(wqe_desc->queue_info, MSS) <
+ TX_MSS_MIN) {
+ /* Mss should not less than 80. */
+ wqe_desc->queue_info =
+ SQ_CTRL_QUEUE_INFO_CLEAR(wqe_desc->queue_info, MSS);
+ wqe_desc->queue_info |= SQ_CTRL_QUEUE_INFO_SET(TX_MSS_MIN, MSS);
+ }
+
+ wqe_desc->queue_info = hinic3_hw_be32(wqe_desc->queue_info);
+}
+
+/**
+ * It is responsible for sending data packets.
+ *
+ * @param[in] tx_queue
+ * Point to send queue.
+ * @param[in] tx_pkts
+ * Pointer to the array of data packets to be sent.
+ * @param[in] nb_pkts
+ * Number of sent packets.
+ * @return
+ * Number of actually sent packets.
+ */
+u16
+hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, u16 nb_pkts)
+{
+ struct hinic3_txq *txq = tx_queue;
+ struct hinic3_tx_info *tx_info = NULL;
+ struct rte_mbuf *mbuf_pkt = NULL;
+ struct hinic3_sq_wqe_combo wqe_combo = {0};
+ struct hinic3_sq_wqe *sq_wqe = NULL;
+ struct hinic3_wqe_info wqe_info = {0};
+
+ u32 offload_err, free_cnt;
+ u64 tx_bytes = 0;
+ u16 free_wqebb_cnt, nb_tx;
+ int err;
+
+#ifdef HINIC3_XSTAT_PROF_TX
+ uint64_t t1, t2;
+ t1 = rte_get_tsc_cycles();
+#endif
+
+ if (unlikely(!HINIC3_TXQ_IS_STARTED(txq)))
+ return 0;
+
+ free_cnt = txq->tx_free_thresh;
+ /* Reclaim tx mbuf before xmit new packets. */
+ if (hinic3_get_sq_free_wqebbs(txq) < txq->tx_free_thresh)
+ hinic3_xmit_mbuf_cleanup(txq, free_cnt);
+
+ /* Tx loop routine. */
+ for (nb_tx = 0; nb_tx < nb_pkts; nb_tx++) {
+ mbuf_pkt = *tx_pkts++;
+ if (unlikely(hinic3_get_tx_offload(mbuf_pkt, &wqe_info))) {
+ txq->txq_stats.offload_errors++;
+ break;
+ }
+
+ if (!wqe_info.offload)
+ wqe_info.wqebb_cnt = wqe_info.sge_cnt;
+ else
+ /* Use extended sq wqe with normal TS. */
+ wqe_info.wqebb_cnt = wqe_info.sge_cnt + 1;
+
+ free_wqebb_cnt = hinic3_get_sq_free_wqebbs(txq);
+ if (unlikely(wqe_info.wqebb_cnt > free_wqebb_cnt)) {
+ /* Reclaim again. */
+ hinic3_xmit_mbuf_cleanup(txq, free_cnt);
+ free_wqebb_cnt = hinic3_get_sq_free_wqebbs(txq);
+ if (unlikely(wqe_info.wqebb_cnt > free_wqebb_cnt)) {
+ txq->txq_stats.tx_busy += (nb_pkts - nb_tx);
+ break;
+ }
+ }
+
+ /* Get sq wqe address from wqe_page. */
+ sq_wqe = hinic3_get_sq_wqe(txq, &wqe_info);
+ if (unlikely(!sq_wqe)) {
+ txq->txq_stats.tx_busy++;
+ break;
+ }
+
+ /* Task or bd section maybe warpped for one wqe. */
+ hinic3_set_wqe_combo(txq, &wqe_combo, sq_wqe, &wqe_info);
+
+ wqe_info.queue_info = 0;
+ /* Fill tx packet offload into qsf and task field. */
+ if (wqe_info.offload) {
+ offload_err = hinic3_set_tx_offload(mbuf_pkt,
+ wqe_combo.task,
+ &wqe_info);
+ if (unlikely(offload_err)) {
+ hinic3_put_sq_wqe(txq, &wqe_info);
+ txq->txq_stats.offload_errors++;
+ break;
+ }
+ }
+
+ /* Fill sq_wqe buf_desc and bd_desc. */
+ err = hinic3_mbuf_dma_map_sge(txq, mbuf_pkt, &wqe_combo,
+ &wqe_info);
+ if (err) {
+ hinic3_put_sq_wqe(txq, &wqe_info);
+ txq->txq_stats.offload_errors++;
+ break;
+ }
+
+ /* Record tx info. */
+ tx_info = &txq->tx_info[wqe_info.pi];
+ tx_info->mbuf = mbuf_pkt;
+ tx_info->wqebb_cnt = wqe_info.wqebb_cnt;
+
+ hinic3_prepare_sq_ctrl(&wqe_combo, &wqe_info);
+
+ tx_bytes += mbuf_pkt->pkt_len;
+ }
+
+ /* Update txq stats. */
+ if (nb_tx) {
+ hinic3_write_db(txq->db_addr, txq->q_id, (int)(txq->cos),
+ SQ_CFLAG_DP,
+ MASKED_QUEUE_IDX(txq, txq->prod_idx));
+ txq->txq_stats.packets += nb_tx;
+ txq->txq_stats.bytes += tx_bytes;
+ }
+ txq->txq_stats.burst_pkts = nb_tx;
+
+#ifdef HINIC3_XSTAT_PROF_TX
+ t2 = rte_get_tsc_cycles();
+ txq->txq_stats.app_tsc = t1 - txq->prof_tx_end_tsc;
+ txq->prof_tx_end_tsc = t2;
+ txq->txq_stats.pmd_tsc = t2 - t1;
+ txq->txq_stats.burst_pkts = nb_tx;
+#endif
+
+ return nb_tx;
+}
+
int
hinic3_stop_sq(struct hinic3_txq *txq)
{
diff --git a/drivers/net/hinic3/hinic3_tx.h b/drivers/net/hinic3/hinic3_tx.h
index f4c61ea1b1..6026b3fabc 100644
--- a/drivers/net/hinic3/hinic3_tx.h
+++ b/drivers/net/hinic3/hinic3_tx.h
@@ -308,6 +308,7 @@ struct __rte_cache_aligned hinic3_txq {
void hinic3_flush_txqs(struct hinic3_nic_dev *nic_dev);
void hinic3_free_txq_mbufs(struct hinic3_txq *txq);
void hinic3_free_all_txq_mbufs(struct hinic3_nic_dev *nic_dev);
+u16 hinic3_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, u16 nb_pkts);
int hinic3_stop_sq(struct hinic3_txq *txq);
int hinic3_start_all_sqs(struct rte_eth_dev *eth_dev);
int hinic3_tx_done_cleanup(void *txq, uint32_t free_cnt);
--
2.47.0.windows.2
