add non temporal store for few WQE fields to
optimize data path. This can be enable by making
CONFG_RTE_LIBRTE_MLX5_NT_STORE=y in dpdk config.
Signed-off-by: Aman Kumar <aman.ku...@vvdntech.in>
---
config/common_base | 1 +
drivers/net/mlx5/mlx5.c | 17 +++++++++++++++++
drivers/net/mlx5/mlx5.h | 4 ++++
drivers/net/mlx5/mlx5_rxq.c | 3 +++
drivers/net/mlx5/mlx5_rxtx.c | 20 +++++++++++++++++---
drivers/net/mlx5/mlx5_rxtx.h | 6 ++++++
drivers/net/mlx5/mlx5_rxtx_vec.h | 28 +++++++++++++++++++++++-----
drivers/net/mlx5/mlx5_txq.c | 3 +++
8 files changed, 74 insertions(+), 8 deletions(-)
diff --git a/config/common_base b/config/common_base
index 1476cf334..bbe74833b 100644
--- a/config/common_base
+++ b/config/common_base
@@ -372,6 +372,7 @@ CONFIG_RTE_LIBRTE_MLX4_DEBUG=n
#
CONFIG_RTE_LIBRTE_MLX5_PMD=n
CONFIG_RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY=n
+CONFIG_RTE_LIBRTE_MLX5_NT_STORE=n
CONFIG_RTE_LIBRTE_MLX5_DEBUG=n
#
diff --git a/drivers/net/mlx5/mlx5.c b/drivers/net/mlx5/mlx5.c
index 6eb85dfac..8e1b7df23 100644
--- a/drivers/net/mlx5/mlx5.c
+++ b/drivers/net/mlx5/mlx5.c
@@ -167,6 +167,13 @@
/* mprq_tstore_memcpy */
#define MLX5_MPRQ_TSTORE_MEMCPY "mprq_tstore_memcpy"
#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+/* tx_wqe_field_ntstore */
+#define MLX5_TX_WQE_FIELD_NTSTORE "tx_wqe_field_ntstore"
+
+/* vec_rx_wqe_field_ntstore */
+#define MLX5_VEC_RX_WQE_FIELD_NTSTORE "vec_rx_wqe_field_ntstore"
+#endif
/*
* Device parameter to configure the total data buffer size for a single
@@ -1629,6 +1636,12 @@ mlx5_args_check(const char *key, const char *val, void
*opaque)
#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
} else if (strcmp(MLX5_MPRQ_TSTORE_MEMCPY, key) == 0) {
config->mprq_tstore_memcpy = tmp;
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ } else if (strcmp(MLX5_TX_WQE_FIELD_NTSTORE, key) == 0) {
+ config->tx_wqe_field_ntstore = tmp;
+ } else if (strcmp(MLX5_VEC_RX_WQE_FIELD_NTSTORE, key) == 0) {
+ config->vec_rx_wqe_field_ntstore = tmp;
#endif
} else {
DRV_LOG(WARNING, "%s: unknown parameter", key);
@@ -1692,6 +1705,10 @@ mlx5_args(struct mlx5_dev_config *config, struct
rte_devargs *devargs)
MLX5_DECAP_EN,
#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
MLX5_MPRQ_TSTORE_MEMCPY,
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ MLX5_TX_WQE_FIELD_NTSTORE,
+ MLX5_VEC_RX_WQE_FIELD_NTSTORE,
#endif
NULL,
};
diff --git a/drivers/net/mlx5/mlx5.h b/drivers/net/mlx5/mlx5.h
index 09dc90953..4a816cb2e 100644
--- a/drivers/net/mlx5/mlx5.h
+++ b/drivers/net/mlx5/mlx5.h
@@ -239,6 +239,10 @@ struct mlx5_dev_config {
#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
unsigned int mprq_tstore_memcpy:1;
#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ unsigned int tx_wqe_field_ntstore:1;
+ unsigned int vec_rx_wqe_field_ntstore:1;
+#endif
};
diff --git a/drivers/net/mlx5/mlx5_rxq.c b/drivers/net/mlx5/mlx5_rxq.c
index bee5c03bc..ceb33e5c5 100644
--- a/drivers/net/mlx5/mlx5_rxq.c
+++ b/drivers/net/mlx5/mlx5_rxq.c
@@ -2304,6 +2304,9 @@ mlx5_rxq_new(struct rte_eth_dev *dev, uint16_t idx,
uint16_t desc,
tmpl->irq = 1;
#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
tmpl->rxq.mprq_tstore_memcpy = config->mprq_tstore_memcpy;
+#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ tmpl->rxq.vec_rx_wqe_field_ntstore = config->vec_rx_wqe_field_ntstore;
#endif
mprq_stride_nums = config->mprq.stride_num_n ?
config->mprq.stride_num_n : MLX5_MPRQ_STRIDE_NUM_N;
diff --git a/drivers/net/mlx5/mlx5_rxtx.c b/drivers/net/mlx5/mlx5_rxtx.c
index 62ade3775..6bcdc44a5 100644
--- a/drivers/net/mlx5/mlx5_rxtx.c
+++ b/drivers/net/mlx5/mlx5_rxtx.c
@@ -2318,6 +2318,9 @@ mlx5_tx_request_completion(struct mlx5_txq_data
*__rte_restrict txq,
{
uint16_t head = txq->elts_head;
unsigned int part;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ register uint32_t flags;
+#endif
part = MLX5_TXOFF_CONFIG(INLINE) ?
0 : loc->pkts_sent - loc->pkts_copy;
@@ -2331,9 +2334,20 @@ mlx5_tx_request_completion(struct mlx5_txq_data
*__rte_restrict txq,
txq->elts_comp = head;
if (MLX5_TXOFF_CONFIG(INLINE))
txq->wqe_comp = txq->wqe_ci;
- /* Request unconditional completion on last WQE. */
- last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
- MLX5_COMP_MODE_OFFSET);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ if (txq->tx_wqe_field_ntstore) {
+ flags = RTE_BE32(MLX5_COMP_ALWAYS <<
+ MLX5_COMP_MODE_OFFSET);
+ _mm_stream_si32(((void *)(uintptr_t)&last->cseg.flags),
+ flags);
+ } else {
+#endif
+ /* Request unconditional completion on last WQE. */
+ last->cseg.flags = RTE_BE32(MLX5_COMP_ALWAYS <<
+ MLX5_COMP_MODE_OFFSET);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ }
+#endif
/* Save elts_head in dedicated free on completion queue. */
#ifdef RTE_LIBRTE_MLX5_DEBUG
txq->fcqs[txq->cq_pi++ & txq->cqe_m] = head |
diff --git a/drivers/net/mlx5/mlx5_rxtx.h b/drivers/net/mlx5/mlx5_rxtx.h
index 72763962f..b031eff0b 100644
--- a/drivers/net/mlx5/mlx5_rxtx.h
+++ b/drivers/net/mlx5/mlx5_rxtx.h
@@ -156,6 +156,9 @@ struct mlx5_rxq_data {
#ifdef RTE_LIBRTE_MLX5_NTLOAD_TSTORE_ALIGN_COPY
unsigned int mprq_tstore_memcpy:1;
#endif
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ unsigned int vec_rx_wqe_field_ntstore:1;
+#endif
} __rte_cache_aligned;
enum mlx5_rxq_obj_type {
@@ -324,6 +327,9 @@ struct mlx5_txq_data {
int32_t ts_offset; /* Timestamp field dynamic offset. */
struct mlx5_dev_ctx_shared *sh; /* Shared context. */
struct mlx5_txq_stats stats; /* TX queue counters. */
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ unsigned int tx_wqe_field_ntstore:1;
+#endif
#ifndef RTE_ARCH_64
rte_spinlock_t *uar_lock;
/* UAR access lock required for 32bit implementations */
diff --git a/drivers/net/mlx5/mlx5_rxtx_vec.h b/drivers/net/mlx5/mlx5_rxtx_vec.h
index 6ddcbfb0a..62a07ef00 100644
--- a/drivers/net/mlx5/mlx5_rxtx_vec.h
+++ b/drivers/net/mlx5/mlx5_rxtx_vec.h
@@ -86,6 +86,10 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq,
uint16_t n)
volatile struct mlx5_wqe_data_seg *wq =
&((volatile struct mlx5_wqe_data_seg *)rxq->wqes)[elts_idx];
unsigned int i;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ register uint64_t buf_addr2;
+ register uint32_t lkey_t;
+#endif
MLX5_ASSERT(n >= MLX5_VPMD_RXQ_RPLNSH_THRESH(q_n));
MLX5_ASSERT(n <= (uint16_t)(q_n - (rxq->rq_ci - rxq->rq_pi)));
@@ -107,11 +111,25 @@ mlx5_rx_replenish_bulk_mbuf(struct mlx5_rxq_data *rxq,
uint16_t n)
* impact the performance.
*/
buf_addr = elts[i]->buf_addr;
- wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
- RTE_PKTMBUF_HEADROOM);
- /* If there's only one MR, no need to replace LKey in WQE. */
- if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh) > 1))
- wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ if (rxq->vec_rx_wqe_field_ntstore) {
+ buf_addr2 =
(uint64_t)rte_cpu_to_be_64((uintptr_t)buf_addr + RTE_PKTMBUF_HEADROOM);
+ _mm_stream_si64(((void *)(uintptr_t)&wq[i].addr),
buf_addr2);
+ /* If there's only one MR, no need to replace LKey in
WQE. */
+ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh)
> 1)) {
+ lkey_t = (uint32_t)mlx5_rx_mb2mr(rxq, elts[i]);
+ _mm_stream_si32(((void
*)(uintptr_t)&wq[i].lkey), lkey_t);
+ }
+ } else {
+#endif
+ wq[i].addr = rte_cpu_to_be_64((uintptr_t)buf_addr +
+ RTE_PKTMBUF_HEADROOM);
+ /* If there's only one MR, no need to replace LKey in
WQE. */
+ if (unlikely(mlx5_mr_btree_len(&rxq->mr_ctrl.cache_bh)
> 1))
+ wq[i].lkey = mlx5_rx_mb2mr(rxq, elts[i]);
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ }
+#endif
}
rxq->rq_ci += n;
/* Prevent overflowing into consumed mbufs. */
diff --git a/drivers/net/mlx5/mlx5_txq.c b/drivers/net/mlx5/mlx5_txq.c
index 21fe16b7e..8feac4bdc 100644
--- a/drivers/net/mlx5/mlx5_txq.c
+++ b/drivers/net/mlx5/mlx5_txq.c
@@ -1562,6 +1562,9 @@ txq_set_params(struct mlx5_txq_ctrl *txq_ctrl)
DEV_TX_OFFLOAD_UDP_TNL_TSO);
bool vlan_inline;
unsigned int temp;
+#ifdef RTE_LIBRTE_MLX5_NT_STORE
+ txq_ctrl->txq.tx_wqe_field_ntstore = config->tx_wqe_field_ntstore;
+#endif
if (config->txqs_inline == MLX5_ARG_UNSET)
txqs_inline =
--
2.25.1
--
_Disclaimer: _(c) 2020 VVDN Technologies Pvt. Ltd. This e-mail contains
PRIVILEGED AND CONFIDENTIAL INFORMATION intended solely for the use of the
addressee(s). If you are not the intended recipient, please notify the
sender by e-mail and delete the original message. Further, you are not to
copy, disclose, or distribute this e-mail or its contents to any other
person and any such actions are unlawful._
_
_
_
__
