Change XDP program management functional interface to correspond to new
XDP API.
Signed-off-by: Tom Herbert <t...@herbertland.com>
---
drivers/net/ethernet/mellanox/mlx4/en_netdev.c | 92 +++++---------------------
drivers/net/ethernet/mellanox/mlx4/en_rx.c | 27 ++++----
drivers/net/ethernet/mellanox/mlx4/en_tx.c | 1 +
drivers/net/ethernet/mellanox/mlx4/mlx4_en.h | 1 -
4 files changed, 29 insertions(+), 92 deletions(-)
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
index 748e9f6..613786a 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_netdev.c
@@ -42,6 +42,7 @@
#include <net/busy_poll.h>
#include <net/vxlan.h>
#include <net/devlink.h>
+#include <net/xdp.h>
#include <linux/mlx4/driver.h>
#include <linux/mlx4/device.h>
@@ -2195,8 +2196,7 @@ int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv,
struct mlx4_en_port_profile *prof,
bool carry_xdp_prog)
{
- struct bpf_prog *xdp_prog;
- int i, t;
+ int t;
mlx4_en_copy_priv(tmp, priv, prof);
@@ -2211,22 +2211,6 @@ int mlx4_en_try_alloc_resources(struct mlx4_en_priv
*priv,
return -ENOMEM;
}
- /* All rx_rings has the same xdp_prog. Pick the first one. */
- xdp_prog = rcu_dereference_protected(
- priv->rx_ring[0]->xdp_prog,
- lockdep_is_held(&priv->mdev->state_lock));
-
- if (xdp_prog && carry_xdp_prog) {
- xdp_prog = bpf_prog_add(xdp_prog, tmp->rx_ring_num);
- if (IS_ERR(xdp_prog)) {
- mlx4_en_free_resources(tmp);
- return PTR_ERR(xdp_prog);
- }
- for (i = 0; i < tmp->rx_ring_num; i++)
- rcu_assign_pointer(tmp->rx_ring[i]->xdp_prog,
- xdp_prog);
- }
-
return 0;
}
@@ -2713,42 +2697,20 @@ static int mlx4_en_set_tx_maxrate(struct net_device
*dev, int queue_index, u32 m
return err;
}
-static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog)
+static int mlx4_xdp_init(struct net_device *dev, bool enable)
{
struct mlx4_en_priv *priv = netdev_priv(dev);
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_port_profile new_prof;
- struct bpf_prog *old_prog;
struct mlx4_en_priv *tmp;
int tx_changed = 0;
- int xdp_ring_num;
int port_up = 0;
- int err;
- int i;
+ int xdp_ring_num, err;
- xdp_ring_num = prog ? priv->rx_ring_num : 0;
+ xdp_ring_num = enable ? ALIGN(priv->rx_ring_num, MLX4_EN_NUM_UP) : 0;
- /* No need to reconfigure buffers when simply swapping the
- * program for a new one.
- */
- if (priv->tx_ring_num[TX_XDP] == xdp_ring_num) {
- if (prog) {
- prog = bpf_prog_add(prog, priv->rx_ring_num - 1);
- if (IS_ERR(prog))
- return PTR_ERR(prog);
- }
- mutex_lock(&mdev->state_lock);
- for (i = 0; i < priv->rx_ring_num; i++) {
- old_prog = rcu_dereference_protected(
- priv->rx_ring[i]->xdp_prog,
- lockdep_is_held(&mdev->state_lock));
- rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog);
- if (old_prog)
- bpf_prog_put(old_prog);
- }
- mutex_unlock(&mdev->state_lock);
+ if (priv->tx_ring_num[TX_XDP] == xdp_ring_num)
return 0;
- }
if (!mlx4_en_check_xdp_mtu(dev, dev->mtu))
return -EOPNOTSUPP;
@@ -2757,14 +2719,6 @@ static int mlx4_xdp_set(struct net_device *dev, struct
bpf_prog *prog)
if (!tmp)
return -ENOMEM;
- if (prog) {
- prog = bpf_prog_add(prog, priv->rx_ring_num - 1);
- if (IS_ERR(prog)) {
- err = PTR_ERR(prog);
- goto out;
- }
- }
-
mutex_lock(&mdev->state_lock);
memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile));
new_prof.tx_ring_num[TX_XDP] = xdp_ring_num;
@@ -2777,11 +2731,8 @@ static int mlx4_xdp_set(struct net_device *dev, struct
bpf_prog *prog)
}
err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof, false);
- if (err) {
- if (prog)
- bpf_prog_sub(prog, priv->rx_ring_num - 1);
+ if (err)
goto unlock_out;
- }
if (priv->port_up) {
port_up = 1;
@@ -2792,15 +2743,6 @@ static int mlx4_xdp_set(struct net_device *dev, struct
bpf_prog *prog)
if (tx_changed)
netif_set_real_num_tx_queues(dev, priv->tx_ring_num[TX]);
- for (i = 0; i < priv->rx_ring_num; i++) {
- old_prog = rcu_dereference_protected(
- priv->rx_ring[i]->xdp_prog,
- lockdep_is_held(&mdev->state_lock));
- rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog);
- if (old_prog)
- bpf_prog_put(old_prog);
- }
-
if (port_up) {
err = mlx4_en_start_port(dev);
if (err) {
@@ -2812,26 +2754,24 @@ static int mlx4_xdp_set(struct net_device *dev, struct
bpf_prog *prog)
unlock_out:
mutex_unlock(&mdev->state_lock);
-out:
kfree(tmp);
return err;
}
-static bool mlx4_xdp_attached(struct net_device *dev)
+static int mlx4_xdp_check_bpf(struct net_device *dev, struct bpf_prog *prog)
{
- struct mlx4_en_priv *priv = netdev_priv(dev);
-
- return !!priv->tx_ring_num[TX_XDP];
+ return 0;
}
static int mlx4_xdp(struct net_device *dev, struct netdev_xdp *xdp)
{
switch (xdp->command) {
- case XDP_SETUP_PROG:
- return mlx4_xdp_set(dev, xdp->prog);
- case XDP_QUERY_PROG:
- xdp->prog_attached = mlx4_xdp_attached(dev);
- return 0;
+ case XDP_MODE_ON:
+ return mlx4_xdp_init(dev, true);
+ case XDP_MODE_OFF:
+ return mlx4_xdp_init(dev, false);
+ case XDP_CHECK_BPF_PROG:
+ return mlx4_xdp_check_bpf(dev, xdp->prog);
default:
return -EINVAL;
}
@@ -3331,7 +3271,7 @@ int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int
port,
dev->vlan_features = dev->hw_features;
- dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_RXHASH;
+ dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_RXHASH | NETIF_F_XDP;
dev->features = dev->hw_features | NETIF_F_HIGHDMA |
NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX |
NETIF_F_HW_VLAN_CTAG_FILTER;
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
index d85e644..a8fddc0 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_rx.c
@@ -43,6 +43,7 @@
#include <linux/if_vlan.h>
#include <linux/vmalloc.h>
#include <linux/irq.h>
+#include <net/xdp.h>
#if IS_ENABLED(CONFIG_IPV6)
#include <net/ip6_checksum.h>
@@ -547,13 +548,7 @@ void mlx4_en_destroy_rx_ring(struct mlx4_en_priv *priv,
{
struct mlx4_en_dev *mdev = priv->mdev;
struct mlx4_en_rx_ring *ring = *pring;
- struct bpf_prog *old_prog;
- old_prog = rcu_dereference_protected(
- ring->xdp_prog,
- lockdep_is_held(&mdev->state_lock));
- if (old_prog)
- bpf_prog_put(old_prog);
mlx4_free_hwq_res(mdev->dev, &ring->wqres, size * stride + TXBB_SIZE);
vfree(ring->rx_info);
ring->rx_info = NULL;
@@ -802,7 +797,6 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct
mlx4_en_cq *cq, int bud
struct mlx4_en_rx_ring *ring = priv->rx_ring[cq->ring];
struct mlx4_en_rx_alloc *frags;
struct mlx4_en_rx_desc *rx_desc;
- struct bpf_prog *xdp_prog;
int doorbell_pending;
struct sk_buff *skb;
int index;
@@ -813,6 +807,7 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct
mlx4_en_cq *cq, int bud
int factor = priv->cqe_factor;
u64 timestamp;
bool l2_tunnel;
+ bool run_xdp;
if (unlikely(!priv->port_up))
return 0;
@@ -820,9 +815,9 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct
mlx4_en_cq *cq, int bud
if (unlikely(budget <= 0))
return polled;
- /* Protect accesses to: ring->xdp_prog, priv->mac_hash list */
+ /* Protect accesses to: XDP hooks, priv->mac_hash list */
rcu_read_lock();
- xdp_prog = rcu_dereference(ring->xdp_prog);
+ run_xdp = xdp_hook_run_needed_check(dev, &cq->napi);
doorbell_pending = 0;
/* We assume a 1:1 mapping between CQEs and Rx descriptors, so Rx
@@ -895,13 +890,14 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct
mlx4_en_cq *cq, int bud
l2_tunnel = (dev->hw_enc_features & NETIF_F_RXCSUM) &&
(cqe->vlan_my_qpn & cpu_to_be32(MLX4_CQE_L2_TUNNEL));
- /* A bpf program gets first chance to drop the packet. It may
+ /* An xdp program gets first chance to drop the packet. It may
* read bytes but not past the end of the frag.
*/
- if (xdp_prog) {
+ if (run_xdp) {
struct xdp_buff xdp;
dma_addr_t dma;
void *orig_data;
+ struct xdp_hook *last_hook;
u32 act;
dma = be64_to_cpu(rx_desc->data[0].addr);
@@ -914,7 +910,8 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct
mlx4_en_cq *cq, int bud
xdp.data_end = xdp.data + length;
orig_data = xdp.data;
- act = bpf_prog_run_xdp(xdp_prog, &xdp);
+ act = xdp_hook_run_ret_last(&cq->napi, &xdp,
+ &last_hook);
if (xdp.data != orig_data) {
length = xdp.data_end - xdp.data;
@@ -930,12 +927,12 @@ int mlx4_en_process_rx_cq(struct net_device *dev, struct
mlx4_en_cq *cq, int bud
length, cq->ring,
&doorbell_pending)))
goto consumed;
- trace_xdp_exception(dev, xdp_prog, act);
+ trace_xdp_hook_exception(dev, last_hook, act);
goto xdp_drop_no_cnt; /* Drop on xmit failure */
default:
- bpf_warn_invalid_xdp_action(act);
+ xdp_warn_invalid_action(act);
case XDP_ABORTED:
- trace_xdp_exception(dev, xdp_prog, act);
+ trace_xdp_hook_exception(dev, last_hook, act);
case XDP_DROP:
ring->xdp_drop++;
xdp_drop_no_cnt:
diff --git a/drivers/net/ethernet/mellanox/mlx4/en_tx.c
b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
index 3ed4219..870acb7 100644
--- a/drivers/net/ethernet/mellanox/mlx4/en_tx.c
+++ b/drivers/net/ethernet/mellanox/mlx4/en_tx.c
@@ -43,6 +43,7 @@
#include <linux/ip.h>
#include <linux/ipv6.h>
#include <linux/moduleparam.h>
+#include <net/xdp.h>
#include "mlx4_en.h"
diff --git a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
index cec59bc..fbd1764 100644
--- a/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
+++ b/drivers/net/ethernet/mellanox/mlx4/mlx4_en.h
@@ -347,7 +347,6 @@ struct mlx4_en_rx_ring {
u8 fcs_del;
void *buf;
void *rx_info;
- struct bpf_prog __rcu *xdp_prog;
struct mlx4_en_page_cache page_cache;
unsigned long bytes;
unsigned long packets;
--
2.9.3
