Let bnxt_en act as a knod NIC: register with the knod core, feed received packets to the accelerator over the per-queue SPSC ring, and transmit accelerator verdicts back through the XDP TX ring. Restricted to page-mode MTU.
Signed-off-by: Taehee Yoo <[email protected]> (cherry picked from commit c9ef892dd5859cc71b0f8a546d87032cf70bb442) --- drivers/net/ethernet/broadcom/bnxt/bnxt.c | 155 +++++++++- drivers/net/ethernet/broadcom/bnxt/bnxt.h | 12 +- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c | 274 +++++++++++++++++- drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h | 12 +- 4 files changed, 442 insertions(+), 11 deletions(-) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 7513618793da..6ddf4bafc3d4 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -993,11 +993,36 @@ static inline u8 *__bnxt_alloc_rx_frag(struct bnxt *bp, dma_addr_t *mapping, return page_address(page) + offset; } +static int bnxt_alloc_rx_off_netmem(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr, + u16 prod, gfp_t gfp) +{ + struct rx_bd *rxbd = + &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; + struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; + unsigned int offset = 0; + dma_addr_t mapping; + netmem_ref netmem; + + netmem = __bnxt_alloc_rx_netmem(bp, &mapping, rxr, &offset, gfp); + if (!netmem) + return -ENOMEM; + + mapping += bp->rx_dma_offset; + rx_buf->data = (void *)netmem; + rx_buf->data_ptr = netmem_address(netmem) + bp->rx_offset; + rx_buf->mapping = mapping; + + rxbd->rx_bd_haddr = cpu_to_le64(mapping); + return 0; +} + int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 prod, gfp_t gfp) { struct rx_bd *rxbd = &rxr->rx_desc_ring[RX_RING(bp, prod)][RX_IDX(prod)]; struct bnxt_sw_rx_bd *rx_buf = &rxr->rx_buf_ring[RING_RX(bp, prod)]; + unsigned int offset = 0; dma_addr_t mapping; if (BNXT_RX_PAGE_MODE(bp)) { @@ -1012,6 +1037,16 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, rx_buf->data = page; rx_buf->data_ptr = page_address(page) + offset + bp->rx_offset; rx_buf->offset = offset; + } else if (BNXT_RX_OFFLOAD_MODE(bp)) { + netmem_ref netmem = __bnxt_alloc_rx_netmem(bp, &mapping, rxr, + &offset, gfp); + if (!netmem) + return -ENOMEM; + + mapping += bp->rx_dma_offset; + rx_buf->data = (void *)netmem; + rx_buf->data_ptr = netmem_address(netmem) + bp->rx_offset; + rx_buf->offset = offset; } else { u8 *data = __bnxt_alloc_rx_frag(bp, &mapping, rxr, gfp); @@ -1250,6 +1285,25 @@ static struct sk_buff *bnxt_rx_page_skb(struct bnxt *bp, return skb; } +static struct sk_buff *bnxt_rx_offload_netmem(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr, + u16 cons, void *data, + u8 *data_ptr, + dma_addr_t dma_addr, + unsigned int offset_and_len) +{ + u16 prod = rxr->rx_prod; + int err; + + err = bnxt_alloc_rx_off_netmem(bp, rxr, prod, GFP_ATOMIC); + if (unlikely(err)) { + bnxt_reuse_rx_data(rxr, cons, data); + return NULL; + } + + return NULL; +} + static struct sk_buff *bnxt_rx_skb(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, void *data, u8 *data_ptr, @@ -2249,6 +2303,36 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, len = flags >> RX_CMP_LEN_SHIFT; dma_addr = rx_buf->mapping; + if (BNXT_RX_OFFLOAD_MODE(bp)) { + if (bnxt_alloc_rx_off_netmem(bp, rxr, rxr->rx_prod, + GFP_ATOMIC)) { + bnxt_reuse_rx_data(rxr, cons, data); + bnapi->cp_ring.sw_stats->rx.rx_buf_errors++; + } else { + struct knod_work_priv *wpriv; + struct spsc_bd *bd; + + if (bnapi->index >= KNOD_SPSC_MAX) { + rc = 1; + goto next_rx; + } + + wpriv = &bp->knodev->wpriv[bnapi->index]; + if (spsc_produce(&wpriv->spsc_bds, (void **)&bd)) { + bnxt_reuse_rx_data(rxr, cons, data); + } else { + bd->netmem = (netmem_ref)data; + bd->len = len; + bd->off = bp->rx_offset; + bd->page_idx = net_iov_binding_idx( + netmem_to_net_iov((netmem_ref)data)); + spsc_produce_commit(&wpriv->spsc_bds); + } + } + rc = 1; + goto next_rx; + } + if (bnxt_xdp_attached(bp, rxr)) { bnxt_xdp.rxcmp = rxcmp; bnxt_xdp.rxcmp1 = rxcmp1; @@ -3272,6 +3356,11 @@ static int bnxt_poll(struct napi_struct *napi, int budget) napi_complete(napi); return 0; } + + if (BNXT_RX_OFFLOAD_MODE(bp)) + work_done += bnxt_rx_offload_act_handler(bnapi, + budget - work_done); + while (1) { work_done += bnxt_poll_work(bp, cpr, budget - work_done); @@ -3296,6 +3385,7 @@ static int bnxt_poll(struct napi_struct *napi, int budget) &dim_sample); net_dim(&cpr->dim, &dim_sample); } + return work_done; } @@ -3363,6 +3453,11 @@ static int bnxt_poll_p5(struct napi_struct *napi, int budget) cpr->has_more_work = 0; work_done = __bnxt_poll_cqs(bp, bnapi, budget); } + + if (BNXT_RX_OFFLOAD_MODE(bp)) + work_done += bnxt_rx_offload_act_handler(bnapi, + budget - work_done); + while (1) { u16 type; @@ -3427,6 +3522,7 @@ static int bnxt_poll_p5(struct napi_struct *napi, int budget) &dim_sample); net_dim(&cpr->dim, &dim_sample); } + return work_done; } @@ -3458,6 +3554,13 @@ static void bnxt_free_one_tx_ring_skbs(struct bnxt *bp, tx_buf->xdpf = NULL; i++; continue; + } else if (tx_buf->action == BNXT_NETMEM_TX) { + page_pool_recycle_direct_netmem( + netmem_get_pp(tx_buf->netmem), + tx_buf->netmem); + tx_buf->action = 0; + tx_buf->netmem = 0; + continue; } skb = tx_buf->skb; @@ -3551,6 +3654,10 @@ static void bnxt_free_one_rx_ring(struct bnxt *bp, struct bnxt_rx_ring_info *rxr rx_buf->data = NULL; if (BNXT_RX_PAGE_MODE(bp)) page_pool_recycle_direct(rxr->page_pool, data); + else if (BNXT_RX_OFFLOAD_MODE(bp)) + page_pool_put_full_netmem( + netmem_get_pp((netmem_ref)data), + (netmem_ref)data, false); else page_pool_free_va(rxr->head_pool, data, true); } @@ -3905,7 +4012,7 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp, struct page_pool *pool; pp.pool_size = bnxt_rx_agg_ring_fill_level(bp, rxr) / agg_size_fac; - if (BNXT_RX_PAGE_MODE(bp)) + if (BNXT_RX_PAGE_MODE(bp) || BNXT_RX_OFFLOAD_MODE(bp)) pp.pool_size += bp->rx_ring_size / rx_size_fac; pp.order = get_order(rxr->rx_page_size); @@ -3923,8 +4030,10 @@ static int bnxt_alloc_rx_page_pool(struct bnxt *bp, return PTR_ERR(pool); rxr->page_pool = pool; - rxr->need_head_pool = page_pool_is_unreadable(pool); - rxr->need_head_pool |= !!pp.order; + if (!BNXT_RX_OFFLOAD_MODE(bp)) { + rxr->need_head_pool = page_pool_is_unreadable(pool); + rxr->need_head_pool |= !!pp.order; + } if (bnxt_separate_head_pool(rxr)) { pp.order = 0; pp.max_len = PAGE_SIZE; @@ -4673,6 +4782,9 @@ static int bnxt_init_rx_rings(struct bnxt *bp) if (BNXT_RX_PAGE_MODE(bp)) { bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; bp->rx_dma_offset = XDP_PACKET_HEADROOM; + } else if (BNXT_RX_OFFLOAD_MODE(bp)) { + bp->rx_offset = NET_IP_ALIGN + XDP_PACKET_HEADROOM; + bp->rx_dma_offset = XDP_PACKET_HEADROOM; } else { bp->rx_offset = BNXT_RX_OFFSET; bp->rx_dma_offset = BNXT_RX_DMA_OFFSET; @@ -4961,12 +5073,14 @@ void bnxt_set_ring_params(struct bnxt *bp) /* Changing allocation mode of RX rings. * TODO: Update when extending xdp_rxq_info to support allocation modes. */ -static void __bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) +static void __bnxt_set_rx_skb_mode(struct bnxt *bp, int page_mode) { struct net_device *dev = bp->dev; - if (page_mode) { - bp->flags &= ~(BNXT_FLAG_AGG_RINGS | BNXT_FLAG_NO_AGG_RINGS); + if (page_mode & BNXT_FLAG_RX_PAGE_MODE) { + bp->flags &= ~(BNXT_FLAG_AGG_RINGS | + BNXT_FLAG_NO_AGG_RINGS | + BNXT_FLAG_RX_OFFLOAD_MODE); bp->flags |= BNXT_FLAG_RX_PAGE_MODE; if (bp->xdp_prog->aux->xdp_has_frags) @@ -4982,15 +5096,24 @@ static void __bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) bp->rx_skb_func = bnxt_rx_page_skb; } bp->rx_dir = DMA_BIDIRECTIONAL; + } else if (page_mode & BNXT_FLAG_RX_OFFLOAD_MODE) { + bp->flags &= ~(BNXT_FLAG_AGG_RINGS | + BNXT_FLAG_RX_PAGE_MODE); + bp->flags |= (BNXT_FLAG_RX_OFFLOAD_MODE | + BNXT_FLAG_NO_AGG_RINGS); + dev->max_mtu = min_t(u16, bp->max_mtu, BNXT_MAX_PAGE_MODE_MTU); + bp->rx_skb_func = bnxt_rx_offload_netmem; + bp->rx_dir = DMA_BIDIRECTIONAL; } else { dev->max_mtu = bp->max_mtu; bp->flags &= ~BNXT_FLAG_RX_PAGE_MODE; + bp->flags &= ~BNXT_FLAG_RX_OFFLOAD_MODE; bp->rx_dir = DMA_FROM_DEVICE; bp->rx_skb_func = bnxt_rx_skb; } } -void bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode) +void bnxt_set_rx_skb_mode(struct bnxt *bp, int page_mode) { __bnxt_set_rx_skb_mode(bp, page_mode); @@ -5663,6 +5786,12 @@ static int bnxt_alloc_mem(struct bnxt *bp, bool irq_re_init) BNXT_RING_TO_TC(bp, txr->txq_index); bnapi2->tx_ring[txr->tx_napi_idx] = txr; bnapi2->tx_int = bnxt_tx_int; + } else if (BNXT_RX_OFFLOAD_MODE(bp)) { + bnapi2 = bp->bnapi[j]; + bnapi2->flags |= BNXT_NAPI_FLAG_XDP; + bnapi2->tx_ring[0] = txr; + bnapi2->tx_int = bnxt_tx_int_xdp; + j++; } else { bnapi2 = bp->bnapi[j]; bnapi2->flags |= BNXT_NAPI_FLAG_XDP; @@ -11958,6 +12087,8 @@ static void bnxt_del_napi(struct bnxt *bp) if (!bp->bnapi) return; + bnxt_rx_offload_clear_napi(bp); + for (i = 0; i < bp->rx_nr_rings; i++) netif_queue_set_napi(bp->dev, i, NETDEV_QUEUE_TYPE_RX, NULL); for (i = 0; i < bp->tx_nr_rings - bp->tx_nr_rings_xdp; i++) @@ -11997,6 +12128,8 @@ static void bnxt_init_napi(struct bnxt *bp) bnapi = bp->bnapi[cp_nr_rings]; netif_napi_add_locked(bp->dev, &bnapi->napi, bnxt_poll_nitroa0); } + + bnxt_rx_offload_set_napi(bp); } static void bnxt_disable_napi(struct bnxt *bp) @@ -13356,6 +13489,7 @@ static int bnxt_open(struct net_device *dev) BNXT_RESTART_ULP_SP_EVENT); } } + bnxt_rx_offload_start(bp); return rc; } @@ -13443,6 +13577,7 @@ static int bnxt_close(struct net_device *dev) { struct bnxt *bp = netdev_priv(dev); + bnxt_rx_offload_stop(bp); bnxt_close_nic(bp, true, true); bnxt_hwrm_shutdown_link(bp); bnxt_hwrm_if_change(bp, false); @@ -13643,6 +13778,8 @@ bnxt_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) stats->tx_errors = BNXT_GET_TX_PORT_STATS64(tx, tx_err); } clear_bit(BNXT_STATE_READ_STATS, &bp->state); + if (BNXT_RX_OFFLOAD_MODE(bp)) + knod_dev_get_stats64(bp->knodev, stats); } static void bnxt_get_one_ring_drv_stats(struct bnxt *bp, @@ -16518,6 +16655,7 @@ static void bnxt_remove_one(struct pci_dev *pdev) bnxt_aux_devices_del(bp); + bnxt_knod_uninit(bp); unregister_netdev(dev); bnxt_ptp_clear(bp); @@ -17080,7 +17218,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) } dev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_RX_SG; + NETDEV_XDP_ACT_RX_SG | NETDEV_XDP_ACT_HW_OFFLOAD; #ifdef CONFIG_BNXT_SRIOV init_waitqueue_head(&bp->sriov_cfg_wait); @@ -17198,6 +17336,7 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) bnxt_print_device_info(bp); pci_save_state(pdev); + bnxt_knod_init(bp); return 0; init_err_cleanup: diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index 6335dfc14c98..d99590db6efa 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -29,6 +29,8 @@ #include <net/devlink.h> #include <net/dst_metadata.h> #include <net/xdp.h> +#include <net/knod.h> +#include <net/spsc_ring.h> #include <linux/dim.h> #include <linux/io-64-nonatomic-lo-hi.h> #include <linux/bnxt/ulp.h> @@ -884,10 +886,14 @@ struct nqe_cn { #define BNXT_REDIRECT_EVENT 8 #define BNXT_TX_CMP_EVENT 0x10 +#define BNXT_NETMEM_ACT 0xf0 +#define BNXT_NETMEM_TX (XDP_TX + BNXT_NETMEM_ACT) + struct bnxt_sw_tx_bd { union { struct sk_buff *skb; struct xdp_frame *xdpf; + netmem_ref netmem; }; DEFINE_DMA_UNMAP_ADDR(mapping); DEFINE_DMA_UNMAP_LEN(len); @@ -2300,6 +2306,7 @@ struct bnxt { #define BNXT_FLAG_TX_COAL_CMPL 0x8000000 #define BNXT_FLAG_PORT_STATS_EXT 0x10000000 #define BNXT_FLAG_HDS 0x20000000 + #define BNXT_FLAG_RX_OFFLOAD_MODE 0x40000000 #define BNXT_FLAG_AGG_RINGS (BNXT_FLAG_JUMBO | BNXT_FLAG_GRO | \ BNXT_FLAG_LRO | BNXT_FLAG_HDS) @@ -2324,6 +2331,7 @@ struct bnxt { (bp)->link_info.phy_state == BNXT_PHY_STATE_ENABLED) #define BNXT_CHIP_TYPE_NITRO_A0(bp) ((bp)->flags & BNXT_FLAG_CHIP_NITRO_A0) #define BNXT_RX_PAGE_MODE(bp) ((bp)->flags & BNXT_FLAG_RX_PAGE_MODE) +#define BNXT_RX_OFFLOAD_MODE(bp) ((bp)->flags & BNXT_FLAG_RX_OFFLOAD_MODE) #define BNXT_SUPPORTS_TPA(bp) (!BNXT_CHIP_TYPE_NITRO_A0(bp) && \ (!((bp)->flags & BNXT_FLAG_CHIP_P5_PLUS) ||\ (bp)->max_tpa_v2) && !is_kdump_kernel()) @@ -2748,6 +2756,8 @@ struct bnxt { #define BNXT_DUMP_LIVE_WITH_CTX_L1_CACHE 3 struct bpf_prog *xdp_prog; + struct knod_netdev *knetdev; + struct knod_dev *knodev; struct bnxt_ptp_cfg *ptp_cfg; u8 ptp_all_rx_tstamp; @@ -2964,7 +2974,7 @@ u32 bnxt_fw_health_readl(struct bnxt *bp, int reg_idx); bool bnxt_bs_trace_avail(struct bnxt *bp, u16 type); void bnxt_set_tpa_flags(struct bnxt *bp); void bnxt_set_ring_params(struct bnxt *); -void bnxt_set_rx_skb_mode(struct bnxt *bp, bool page_mode); +void bnxt_set_rx_skb_mode(struct bnxt *bp, int page_mode); void bnxt_insert_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr); void bnxt_del_one_usr_fltr(struct bnxt *bp, struct bnxt_filter_base *fltr); int bnxt_hwrm_func_drv_rgtr(struct bnxt *bp, unsigned long *bmap, diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index 9e5009be8e98..a264a2cf8302 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -156,6 +156,11 @@ void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget) tx_buf = &txr->tx_buf_ring[RING_TX(bp, tx_cons)]; page_pool_recycle_direct(rxr->page_pool, tx_buf->page); } + } else if (tx_buf->action == BNXT_NETMEM_TX) { + page_pool_recycle_direct_netmem(rxr->page_pool, + tx_buf->netmem); + tx_buf->action = 0; + tx_buf->netmem = 0; } else { bnxt_sched_reset_txr(bp, txr, tx_cons); return; @@ -423,7 +428,7 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) bpf_prog_put(old); if (prog) { - bnxt_set_rx_skb_mode(bp, true); + bnxt_set_rx_skb_mode(bp, BNXT_FLAG_RX_PAGE_MODE); xdp_features_set_redirect_target_locked(dev, true); } else { xdp_features_clear_redirect_target_locked(dev); @@ -441,19 +446,123 @@ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) return 0; } + +static int bnxt_xdp_offload_set(struct bnxt *bp, int enable) +{ + struct net_device *dev = bp->dev; + int tx_xdp = 0, tx_cp, rc, tc; + + netdev_assert_locked(dev); + + if (!(bp->flags & BNXT_FLAG_SHARED_RINGS)) { + netdev_warn(dev, "ethtool rx/tx channels must be combined to support XDP.\n"); + return -EOPNOTSUPP; + } + if (enable && dev->mtu > BNXT_MAX_PAGE_MODE_MTU) { + netdev_warn(dev, "MTU %d larger than %d for single-page RX offload.\n", + dev->mtu, BNXT_MAX_PAGE_MODE_MTU); + return -EOPNOTSUPP; + } + if (enable) + tx_xdp = bp->rx_nr_rings; + + tc = bp->num_tc; + if (!tc) + tc = 1; + rc = bnxt_check_rings(bp, bp->tx_nr_rings_per_tc, bp->rx_nr_rings, + true, tc, tx_xdp); + if (rc) { + netdev_warn(dev, "Unable to reserve enough TX rings to support XDP.\n"); + return rc; + } + if (netif_running(dev)) + bnxt_close_nic(bp, true, false); + + if (enable) { + bnxt_set_rx_skb_mode(bp, BNXT_FLAG_RX_OFFLOAD_MODE); + xdp_features_set_redirect_target_locked(dev, true); + } else { + xdp_features_clear_redirect_target_locked(dev); + bnxt_set_rx_skb_mode(bp, false); + } + bp->tx_nr_rings_xdp = tx_xdp; + bp->tx_nr_rings = bp->tx_nr_rings_per_tc * tc + tx_xdp; + tx_cp = bnxt_num_tx_to_cp(bp, bp->tx_nr_rings); + bp->cp_nr_rings = max_t(int, tx_cp, bp->rx_nr_rings); + bnxt_set_tpa_flags(bp); + bnxt_set_ring_params(bp); + + if (netif_running(dev)) + return bnxt_open_nic(bp, true, false); + + return 0; +} + +void bnxt_rx_offload_set_napi(struct bnxt *bp) +{ + struct knod_dev *knodev = bp->knodev; + int i; + + if (!BNXT_RX_OFFLOAD_MODE(bp)) + return; + + for (i = 0; i < KNOD_SPSC_MAX && i < bp->cp_nr_rings; i++) + WRITE_ONCE(knodev->wpriv[i].napi, &bp->bnapi[i]->napi); +} + +void bnxt_rx_offload_clear_napi(struct bnxt *bp) +{ + struct knod_dev *knodev = bp->knodev; + int i; + + if (!BNXT_RX_OFFLOAD_MODE(bp)) + return; + + for (i = 0; i < KNOD_SPSC_MAX; i++) + WRITE_ONCE(knodev->wpriv[i].napi, NULL); +} + +void bnxt_rx_offload_start(struct bnxt *bp) +{ + if (!BNXT_RX_OFFLOAD_MODE(bp)) + return; + + knod_dev_start(bp->knodev); +} + +void bnxt_rx_offload_stop(struct bnxt *bp) +{ + if (!BNXT_RX_OFFLOAD_MODE(bp)) + return; + + knod_dev_stop(bp->knodev); +} + int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp) { struct bnxt *bp = netdev_priv(dev); + struct knod_dev *knodev; int rc; + knodev = bp->knodev; + switch (xdp->command) { case XDP_SETUP_PROG: rc = bnxt_xdp_set(bp, xdp->prog); break; + case XDP_SETUP_PROG_HW: + case BPF_OFFLOAD_MAP_ALLOC: + case BPF_OFFLOAD_MAP_FREE: + if (!knodev) + return -EOPNOTSUPP; + + rc = knod_dev_xdp_install(knodev, xdp); + break; default: rc = -EINVAL; break; } + return rc; } @@ -528,3 +637,166 @@ int bnxt_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, *rss_type = hash_type; return 0; } + +int bnxt_rx_offload_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, + u16 cons, void *data, unsigned int len, int index, + u8 *event) +{ + struct knod_dev *knodev = READ_ONCE(bp->knodev); + + return knodev->accel_ops->xdp_ops->rx_netmem(bp->knodev, (netmem_ref)data, len, + bp->rx_offset, index); +} + +static int bnxt_rx_offload_xdp_attach(struct knod_dev *knodev) +{ + struct bnxt *bp = netdev_priv(knodev->netdev); + int rc; + + WRITE_ONCE(bp->knodev, knodev); + rc = bnxt_xdp_offload_set(bp, true); + if (rc) + WRITE_ONCE(bp->knodev, NULL); + + return rc; +} + +static int bnxt_rx_offload_xdp_detach(struct knod_dev *knodev) +{ + struct bnxt *bp = netdev_priv(knodev->netdev); + + WRITE_ONCE(bp->knodev, NULL); + bnxt_xdp_offload_set(bp, false); + + return 0; +} + +static void __bnxt_xmit_netmem(struct bnxt *bp, struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len, + netmem_ref netmem) +{ + struct bnxt_sw_tx_bd *tx_buf; + + tx_buf = bnxt_xmit_bd(bp, txr, mapping, len, NULL); + tx_buf->action = BNXT_NETMEM_TX; + tx_buf->netmem = netmem; + netmem_dma_unmap_addr_set(netmem, tx_buf, mapping, mapping); + dma_unmap_len_set(tx_buf, len, 0); +} + +struct knod_nic_ops nic_ops = { + .attach = bnxt_rx_offload_xdp_attach, + .detach = bnxt_rx_offload_xdp_detach, +}; + +int bnxt_rx_offload_act_handler(struct bnxt_napi *bnapi, int budget) +{ + struct bnxt_tx_ring_info *txr = bnapi->tx_ring[0]; + struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; + struct spsc_bd *bds[NAPI_POLL_WEIGHT]; + u32 tx_avail, cnt, i, nxmit = 0; + struct knod_dev *knodev; + struct knod_work_priv *wpriv; + struct napi_struct *napi; + struct bnxt *bp = bnapi->bp; + dma_addr_t mapping; + + knodev = bp->knodev; + if (!knodev) + return 0; + + if (bnapi->index >= bp->dev->real_num_tx_queues) { + this_cpu_inc(knodev->stats->tx_dropped); + return 0; + } + + wpriv = &knodev->wpriv[bnapi->index]; + napi = READ_ONCE(wpriv->napi); + if (!napi) + return 0; + + tx_avail = bnxt_tx_avail(bp, txr); + cnt = min_t(u32, NAPI_POLL_WEIGHT, tx_avail); + cnt = min_t(u32, cnt, budget); + if (!cnt) + return 0; + + spsc_release(&wpriv->spsc_bds, (void **)bds, cnt, &cnt); + if (!cnt) + return 0; + + for (i = 0; i < cnt; i++) { + switch (bds[i]->act) { + case KNOD_ACT_INFLIGHT: + case KNOD_IPSEC_INFLIGHT: + goto stop_release; + case XDP_TX: + mapping = netmem_to_net_iov(bds[i]->netmem)->desc.dma_addr + + bds[i]->off; + __bnxt_xmit_netmem(bp, txr, mapping, bds[i]->len, + bds[i]->netmem); + nxmit++; + break; + case XDP_ABORTED: + fallthrough; + case XDP_DROP: + fallthrough; + case XDP_PASS: + fallthrough; + case XDP_REDIRECT: + fallthrough; + default: + page_pool_recycle_direct_netmem(rxr->page_pool, + bds[i]->netmem); + this_cpu_inc(knodev->stats->tx_dropped); + break; + } + } +stop_release: + if (nxmit) { + wmb(); + bnxt_db_write(bp, &txr->tx_db, txr->tx_prod); + } + spsc_release_commit(&wpriv->spsc_bds, i); + + /* + * Device->host delivery: drain the framework pending ring for this NIC + * RX queue and push the built skbs up the stack. Covers all features + * (bpf/none deliver directly, ipsec via knod_dev->post_copy); no-op + * when nothing is attached. + */ + knod_dev_xdp_drain_pass(knodev, napi, bnapi->index, budget); + + return i; +} + +int bnxt_knod_init(struct bnxt *bp) +{ + struct knod_netdev *knetdev; + + knetdev = kzalloc_obj(struct knod_netdev, GFP_KERNEL); + if (!knetdev) { + pr_debug("Failed to allocate knetdev\n"); + return -ENOMEM; + } + + INIT_LIST_HEAD(&knetdev->list); + knetdev->dev = bp->dev; + knetdev->priv = bp; + knetdev->nic_ops = &nic_ops; + knetdev->owner = THIS_MODULE; + knetdev->flags |= KNOD_FLAGS_XDP; + knod_netdev_register(knetdev); + bp->knetdev = knetdev; + + return 0; +} + +void bnxt_knod_uninit(struct bnxt *bp) +{ + knod_netdev_unregister(bp->knetdev); + kfree(bp->knetdev); + bp->knetdev = NULL; + kfree(bp->knodev); + WRITE_ONCE(bp->knodev, NULL); +} diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h index fb4f9143929f..7f2a850e4787 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h @@ -24,9 +24,14 @@ struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, dma_addr_t mapping, u32 len, struct xdp_buff *xdp); void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int budget); +void bnxt_tx_int_offload(struct bnxt *bp, struct bnxt_napi *bnapi, int budget); bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, struct xdp_buff *xdp, struct page *page, u8 **data_ptr, unsigned int *len, u8 *event); +void bnxt_rx_offload_stop(struct bnxt *bp); +void bnxt_rx_offload_start(struct bnxt *bp); +void bnxt_rx_offload_set_napi(struct bnxt *bp); +void bnxt_rx_offload_clear_napi(struct bnxt *bp); int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp); int bnxt_xdp_xmit(struct net_device *dev, int num_frames, struct xdp_frame **frames, u32 flags); @@ -43,5 +48,10 @@ struct sk_buff *bnxt_xdp_build_skb(struct bnxt *bp, struct sk_buff *skb, struct xdp_buff *xdp); int bnxt_xdp_rx_hash(const struct xdp_md *ctx, u32 *hash, enum xdp_rss_hash_type *rss_type); - +int bnxt_rx_offload_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, + u16 cons, void *data, unsigned int len, int index, + u8 *event); +int bnxt_rx_offload_act_handler(struct bnxt_napi *bnapi, int budget); +int bnxt_knod_init(struct bnxt *bp); +void bnxt_knod_uninit(struct bnxt *bp); #endif -- 2.43.0
