Set rx_hash in xdp_rx_meta struct of xdp_buff/xdp_frame according to
the value reported by the hw.
Update the xmo_rx_hash callback of xdp_metadata_ops for the following
drivers:
- ice
- igc
- mlx4
- mlx5
- veth
- virtio_net
Set rx_hash value/type reported by the hw converting the xdp_frame into
a sk_buff.
Signed-off-by: Lorenzo Bianconi <lore...@kernel.org>
---
drivers/net/ethernet/intel/ice/ice_txrx_lib.c | 3 +++
drivers/net/ethernet/intel/igc/igc_main.c | 2 ++
drivers/net/ethernet/mellanox/mlx4/en_rx.c | 1 +
.../net/ethernet/mellanox/mlx5/core/en/xdp.c | 2 ++
drivers/net/veth.c | 1 +
drivers/net/virtio_net.c | 3 ++-
include/net/xdp.h | 14 +++++++++++++
net/core/xdp.c | 20 ++++++++++++++++++-
8 files changed, 44 insertions(+), 2 deletions(-)
diff --git a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c
b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c
index 2719f0e20933f..e4b051a8d99c7 100644
--- a/drivers/net/ethernet/intel/ice/ice_txrx_lib.c
+++ b/drivers/net/ethernet/intel/ice/ice_txrx_lib.c
@@ -520,12 +520,15 @@ static int ice_xdp_rx_hash(const struct xdp_md *ctx, u32
*hash,
enum xdp_rss_hash_type *rss_type)
{
const struct ice_xdp_buff *xdp_ext = (void *)ctx;
+ struct xdp_buff *xdp = (void *)&(xdp_ext->xdp_buff);
*hash = ice_get_rx_hash(xdp_ext->eop_desc);
*rss_type = ice_xdp_rx_hash_type(xdp_ext->eop_desc);
if (!likely(*hash))
return -ENODATA;
+ xdp_set_rx_meta_hash(xdp, *hash, *rss_type);
+
return 0;
}
diff --git a/drivers/net/ethernet/intel/igc/igc_main.c
b/drivers/net/ethernet/intel/igc/igc_main.c
index dfd6c00b4205d..ed22a7a70695e 100644
--- a/drivers/net/ethernet/intel/igc/igc_main.c
+++ b/drivers/net/ethernet/intel/igc/igc_main.c
@@ -6776,12 +6776,14 @@ static int igc_xdp_rx_hash(const struct xdp_md *_ctx,
u32 *hash,
enum xdp_rss_hash_type *rss_type)
{
const struct igc_xdp_buff *ctx = (void *)_ctx;
+ struct xdp_buff *xdp = (void *)&(ctx->xdp);
if (!(ctx->xdp.rxq->dev->features & NETIF_F_RXHASH))
return -ENODATA;
*hash = le32_to_cpu(ctx->rx_desc->wb.lower.hi_dword.rss);
*rss_type = igc_xdp_rss_type[igc_rss_type(ctx->rx_desc)];
+ xdp_set_rx_meta_hash(xdp, *hash, *rss_type);
return 0;
}
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index 15c57e9517e9a..ef6c687866f9d 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -708,6 +708,7 @@ int mlx4_en_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash,
if (cqe->ipv6_ext_mask)
xht |= XDP_RSS_L3_DYNHDR;
}
+ xdp_set_rx_meta_hash(&(_ctx->xdp), *hash, xht);
*rss_type = xht;
return 0;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
index 4610621a340e5..92fb98397751a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en/xdp.c
@@ -238,6 +238,7 @@ static int mlx5e_xdp_rx_hash(const struct xdp_md *ctx, u32
*hash,
enum xdp_rss_hash_type *rss_type)
{
const struct mlx5e_xdp_buff *_ctx = (void *)ctx;
+ struct xdp_buff *xdp = (void *)&(_ctx->xdp);
const struct mlx5_cqe64 *cqe = _ctx->cqe;
u32 hash_type, l4_type, ip_type, lookup;
@@ -252,6 +253,7 @@ static int mlx5e_xdp_rx_hash(const struct xdp_md *ctx, u32
*hash,
l4_type = FIELD_GET(CQE_RSS_HTYPE_L4, hash_type);
lookup = ip_type | l4_type;
*rss_type = mlx5_xdp_rss_type[lookup];
+ xdp_set_rx_meta_hash(xdp, *hash, *rss_type);
return 0;
}
diff --git a/drivers/net/veth.c b/drivers/net/veth.c
index 426e68a950672..cc8e90d330456 100644
--- a/drivers/net/veth.c
+++ b/drivers/net/veth.c
@@ -1633,6 +1633,7 @@ static int veth_xdp_rx_hash(const struct xdp_md *ctx, u32
*hash,
*hash = skb_get_hash(skb);
*rss_type = skb->l4_hash ? XDP_RSS_TYPE_L4_ANY : XDP_RSS_TYPE_NONE;
+ xdp_set_rx_meta_hash(&(_ctx->xdp), *hash, *rss_type);
return 0;
}
diff --git a/drivers/net/virtio_net.c b/drivers/net/virtio_net.c
index 3f10c72743e94..1d1ada8eeda0e 100644
--- a/drivers/net/virtio_net.c
+++ b/drivers/net/virtio_net.c
@@ -6250,8 +6250,8 @@ virtnet_xdp_rss_type[VIRTIO_NET_HASH_REPORT_MAX_TABLE] = {
static int virtnet_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
enum xdp_rss_hash_type *rss_type)
{
- const struct xdp_buff *xdp = (void *)_ctx;
struct virtio_net_hdr_v1_hash *hdr_hash;
+ struct xdp_buff *xdp = (void *)_ctx;
struct virtnet_info *vi;
u16 hash_report;
@@ -6267,6 +6267,7 @@ static int virtnet_xdp_rx_hash(const struct xdp_md *_ctx,
u32 *hash,
*rss_type = virtnet_xdp_rss_type[hash_report];
*hash = __le32_to_cpu(hdr_hash->hash_value);
+ xdp_set_rx_meta_hash(xdp, *hash, *rss_type);
return 0;
}
diff --git a/include/net/xdp.h b/include/net/xdp.h
index 5e08b58a2a10f..e1c344eb7e686 100644
--- a/include/net/xdp.h
+++ b/include/net/xdp.h
@@ -214,6 +214,11 @@ static __always_inline bool xdp_frame_has_rx_meta(struct
xdp_frame *frame)
return !!(frame->flags & XDP_FLAGS_META_RX);
}
+static __always_inline bool xdp_frame_has_rx_meta_hash(struct xdp_frame *frame)
+{
+ return !!(frame->flags & XDP_FLAGS_META_RX_HASH);
+}
+
#define XDP_BULK_QUEUE_SIZE 16
struct xdp_frame_bulk {
int count;
@@ -504,6 +509,15 @@ struct xdp_metadata_ops {
u16 *vlan_tci);
};
+static __always_inline void
+xdp_set_rx_meta_hash(struct xdp_buff *xdp, u32 hash,
+ enum xdp_rss_hash_type rss_type)
+{
+ xdp->rx_meta.hash.val = hash;
+ xdp->rx_meta.hash.type = rss_type;
+ xdp->flags |= XDP_FLAGS_META_RX_HASH;
+}
+
#ifdef CONFIG_NET
u32 bpf_xdp_metadata_kfunc_id(int id);
bool bpf_dev_bound_kfunc_id(u32 btf_id);
diff --git a/net/core/xdp.c b/net/core/xdp.c
index bcc5551c6424b..e2f4d01cf84cf 100644
--- a/net/core/xdp.c
+++ b/net/core/xdp.c
@@ -594,6 +594,23 @@ int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp)
}
EXPORT_SYMBOL_GPL(xdp_alloc_skb_bulk);
+static void xdp_set_skb_rx_hash_from_meta(struct xdp_frame *frame,
+ struct sk_buff *skb)
+{
+ enum pkt_hash_types hash_type = PKT_HASH_TYPE_NONE;
+
+ if (!xdp_frame_has_rx_meta_hash(frame))
+ return;
+
+ if (frame->rx_meta.hash.type & XDP_RSS_TYPE_L4_ANY)
+ hash_type = PKT_HASH_TYPE_L4;
+ else if (frame->rx_meta.hash.type & (XDP_RSS_TYPE_L3_IPV4 |
+ XDP_RSS_TYPE_L3_IPV6))
+ hash_type = PKT_HASH_TYPE_L3;
+
+ skb_set_hash(skb, frame->rx_meta.hash.val, hash_type);
+}
+
struct sk_buff *__xdp_build_skb_from_frame(struct xdp_frame *xdpf,
struct sk_buff *skb,
struct net_device *dev)
@@ -634,9 +651,10 @@ struct sk_buff *__xdp_build_skb_from_frame(struct
xdp_frame *xdpf,
/* Essential SKB info: protocol and skb->dev */
skb->protocol = eth_type_trans(skb, dev);
+ xdp_set_skb_rx_hash_from_meta(xdpf, skb);
+
/* Optional SKB info, currently missing:
* - HW checksum info (skb->ip_summed)
- * - HW RX hash (skb_set_hash)
* - RX ring dev queue index (skb_record_rx_queue)
*/
--
2.46.1
