From: Tariq Toukan <tar...@mellanox.com>
A single cacheline might not contain the packet header for
small L1_CACHE_BYTES values.
Issue an additional prefetch in this case.
Signed-off-by: Tariq Toukan <tar...@mellanox.com>
Signed-off-by: Saeed Mahameed <sae...@mellanox.com>
---
drivers/net/ethernet/mellanox/mlx5/core/en.h | 17 +++++++++++++++++
.../net/ethernet/mellanox/mlx5/core/en/xdp.c | 4 ++--
drivers/net/ethernet/mellanox/mlx5/core/en_rx.c | 13 ++++++-------
.../ethernet/mellanox/mlx5/core/en_selftest.c | 3 +--
4 files changed, 26 insertions(+), 11 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h
b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 51e109fdeec1..6147be23a9b9 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -50,6 +50,7 @@
#include <net/xdp.h>
#include <linux/net_dim.h>
#include <linux/bits.h>
+#include <linux/prefetch.h>
#include "wq.h"
#include "mlx5_core.h"
#include "en_stats.h"
@@ -986,6 +987,22 @@ static inline void mlx5e_cq_arm(struct mlx5e_cq *cq)
mlx5_cq_arm(mcq, MLX5_CQ_DB_REQ_NOT, mcq->uar->map, cq->wq.cc);
}
+static inline void mlx5e_prefetch(void *p)
+{
+ prefetch(p);
+#if L1_CACHE_BYTES < 128
+ prefetch(p + L1_CACHE_BYTES);
+#endif
+}
+
+static inline void mlx5e_prefetchw(void *p)
+{
+ prefetchw(p);
+#if L1_CACHE_BYTES < 128
+ prefetchw(p + L1_CACHE_BYTES);
+#endif
+}
+
extern const struct ethtool_ops mlx5e_ethtool_ops;
#ifdef CONFIG_MLX5_CORE_EN_DCB
extern const struct dcbnl_rtnl_ops mlx5e_dcbnl_ops;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index 03b2a9f9c589..6778bdeff1a2 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -111,7 +111,7 @@ static void mlx5e_xdp_mpwqe_session_start(struct
mlx5e_xdpsq *sq)
mlx5e_xdpsq_fetch_wqe(sq, &session->wqe);
- prefetchw(session->wqe->data);
+ mlx5e_prefetchw(session->wqe->data);
session->ds_count = MLX5E_XDP_TX_EMPTY_DS_COUNT;
pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc);
@@ -209,7 +209,7 @@ static bool mlx5e_xmit_xdp_frame(struct mlx5e_xdpsq *sq,
struct mlx5e_xdp_info *
struct mlx5e_xdpsq_stats *stats = sq->stats;
- prefetchw(wqe);
+ mlx5e_prefetchw(wqe);
if (unlikely(dma_len < MLX5E_XDP_MIN_INLINE || sq->hw_mtu < dma_len)) {
stats->err++;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index c3b3002ff62f..0148ca6ed4ae 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -30,7 +30,6 @@
* SOFTWARE.
*/
-#include <linux/prefetch.h>
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/tcp.h>
@@ -963,8 +962,8 @@ mlx5e_skb_from_cqe_linear(struct mlx5e_rq *rq, struct
mlx5_cqe64 *cqe,
dma_sync_single_range_for_cpu(rq->pdev, di->addr, wi->offset,
frag_size, DMA_FROM_DEVICE);
- prefetchw(va); /* xdp_frame data area */
- prefetch(data);
+ mlx5e_prefetchw(va); /* xdp_frame data area */
+ mlx5e_prefetch(data);
if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_RESP_SEND)) {
rq->stats->wqe_err++;
@@ -1013,7 +1012,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct
mlx5_cqe64 *cqe,
return NULL;
}
- prefetchw(skb->data);
+ mlx5e_prefetchw(skb->data);
while (byte_cnt) {
u16 frag_consumed_bytes =
@@ -1130,7 +1129,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq,
struct mlx5e_mpw_info *w
return NULL;
}
- prefetchw(skb->data);
+ mlx5e_prefetchw(skb->data);
if (unlikely(frag_offset >= PAGE_SIZE)) {
di++;
@@ -1182,8 +1181,8 @@ mlx5e_skb_from_cqe_mpwrq_linear(struct mlx5e_rq *rq,
struct mlx5e_mpw_info *wi,
dma_sync_single_range_for_cpu(rq->pdev, di->addr, head_offset,
frag_size, DMA_FROM_DEVICE);
- prefetchw(va); /* xdp_frame data area */
- prefetch(data);
+ mlx5e_prefetchw(va); /* xdp_frame data area */
+ mlx5e_prefetch(data);
rcu_read_lock();
consumed = mlx5e_xdp_handle(rq, di, va, &rx_headroom, &cqe_bcnt32);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
index 4382ef85488c..1b57c10f773f 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_selftest.c
@@ -30,7 +30,6 @@
* SOFTWARE.
*/
-#include <linux/prefetch.h>
#include <linux/ip.h>
#include <linux/udp.h>
#include <net/udp.h>
@@ -124,7 +123,7 @@ static struct sk_buff *mlx5e_test_get_udp_skb(struct
mlx5e_priv *priv)
return NULL;
}
- prefetchw(skb->data);
+ mlx5e_prefetchw(skb->data);
skb_reserve(skb, NET_IP_ALIGN);
/* Reserve for ethernet and IP header */
--
2.20.1
