ibmveth_poll() must handle several distinct skip/fail outcomes on each
RX slot, not only the happy path:
- interface close / napi_disable must not re-arm PHYP delivery
- bad correlators and harvest errors must not look like successful GRO
- oversize or wrap-around offset+length must not skb_put() past the
buffer
- an out-of-range queue index must napi_complete_done rather than
fall through and keep polling
Doing all of that inline turns the NAPI callback into a deeply nested
switchyard. Split each outcome into a small helper so ibmveth_poll()
stays a thin budget loop:
ibmveth_poll_stopping()
ibmveth_poll_harvest_slot() / recycle_invalid / skip_bad_correlator
ibmveth_poll_drop_oversize()
ibmveth_poll_deliver_frame()
ibmveth_poll_bump_invalid()
Drop schedule_work from rxq_get_buffer() (skip_bad_correlator owns
reset escalation). ibmveth_poll_stopping() ensures close/napi_disable
does not re-arm PHYP. Runtime is still single-queue: helpers are defined
and called from the existing SQ poll path in this same patch (first
use). The next patch turns on MQ and reuses this loop; subordinate
register helpers stay there.
Signed-off-by: Mingming Cao <[email protected]>
Reviewed-by: Dave Marquardt <[email protected]>
Tested-by: Shaik Abdulla <[email protected]>
---
Changes in v5:
- Wrap poll_stopping return after complete so it stays under 80 cols
- On poll_stopping after the budget loop: return < budget after
napi_complete_done (was frames_processed-1 even when under budget)
- New in v5: peel SQ poll harden + helpers before MQ enable so tip P10
stays bring-up focused (mailed v4 09/14 MQ enable -> tip P10; also
14->15). Kitchen-sink / enable-path poll fixes land here on the live
single-queue path first.
- After napi_complete_done, check poll_stopping again before enable_irq
(avoid re-arm while resize/close waits on napi_disable)
drivers/net/ethernet/ibm/ibmveth.c | 283 ++++++++++++++++++++---------
1 file changed, 193 insertions(+), 90 deletions(-)
diff --git a/drivers/net/ethernet/ibm/ibmveth.c
b/drivers/net/ethernet/ibm/ibmveth.c
index 58a639a962a6..86299c62d4ec 100644
--- a/drivers/net/ethernet/ibm/ibmveth.c
+++ b/drivers/net/ethernet/ibm/ibmveth.c
@@ -1346,10 +1346,8 @@ ibmveth_rxq_get_buffer(struct ibmveth_adapter *adapter,
unsigned int pool = correlator >> 32;
unsigned int index = correlator & 0xffffffffUL;
- if (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator)) {
- schedule_work(&adapter->work);
+ if (!ibmveth_rxq_correlator_valid(adapter, queue_index, correlator))
return NULL;
- }
return adapter->rx_buff_pool[queue_index][pool].skbuff[index];
}
@@ -2338,125 +2336,230 @@ static void ibmveth_rx_csum_helper(struct sk_buff
*skb,
}
}
+static void ibmveth_poll_bump_invalid(struct ibmveth_adapter *adapter,
+ int queue_index)
+{
+ adapter->rx_invalid_buffer++;
+}
+
+static bool ibmveth_poll_stopping(struct net_device *netdev,
+ struct napi_struct *napi)
+{
+ return !netif_running(netdev) || napi_disable_pending(napi);
+}
+
+static bool ibmveth_poll_harvest_slot(struct ibmveth_adapter *adapter,
+ int queue_index, bool reuse)
+{
+ int rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, reuse);
+
+ return !rc || rc == -EINVAL || rc == -EFAULT;
+}
+
+static bool ibmveth_poll_recycle_invalid(struct net_device *netdev,
+ struct ibmveth_adapter *adapter,
+ int queue_index)
+{
+ netdev_dbg(netdev, "recycling invalid buffer\n");
+ ibmveth_poll_bump_invalid(adapter, queue_index);
+ return ibmveth_poll_harvest_slot(adapter, queue_index, true);
+}
+
+static bool ibmveth_poll_skip_bad_correlator(struct net_device *netdev,
+ struct ibmveth_adapter *adapter,
+ int queue_index)
+{
+ if (net_ratelimit())
+ netdev_err(netdev,
+ "bad correlator on queue %d, skipping slot\n",
+ queue_index);
+ /* Residual stale slot after resize: recover via reset rather
+ * than spinning forever. Always escalate; only the log is
+ * rate-limited.
+ */
+ schedule_work(&adapter->work);
+ ibmveth_poll_bump_invalid(adapter, queue_index);
+ return ibmveth_poll_harvest_slot(adapter, queue_index, true);
+}
+
+static bool ibmveth_poll_drop_oversize(struct net_device *netdev,
+ struct ibmveth_adapter *adapter,
+ int queue_index, unsigned int off,
+ unsigned int len, unsigned int room)
+{
+ if (net_ratelimit())
+ netdev_err(netdev,
+ "RX frame %u+%u exceeds buffer %u on queue %d,
dropping\n",
+ off, len, room, queue_index);
+ ibmveth_poll_bump_invalid(adapter, queue_index);
+ return ibmveth_poll_harvest_slot(adapter, queue_index, true);
+}
+
+/**
+ * ibmveth_poll_deliver_frame - Build SKB from one valid RX slot and GRO it
+ * @napi: NAPI context for this RX queue
+ * @adapter: ibmveth adapter
+ * @netdev: net_device for @adapter
+ * @queue_index: RX queue index
+ *
+ * Return: 1 frame delivered, 0 if the slot was skipped cleanly, -1 on error.
+ */
+static int ibmveth_poll_deliver_frame(struct napi_struct *napi,
+ struct ibmveth_adapter *adapter,
+ struct net_device *netdev,
+ int queue_index)
+{
+ struct sk_buff *skb, *new_skb;
+ unsigned int room, off, len;
+ int length, offset, csum_good, lrg_pkt;
+ __sum16 iph_check = 0;
+ u16 mss = 0;
+ int rc;
+
+ length = ibmveth_rxq_frame_length(adapter, queue_index);
+ offset = ibmveth_rxq_frame_offset(adapter, queue_index);
+ csum_good = ibmveth_rxq_csum_good(adapter, queue_index);
+ lrg_pkt = ibmveth_rxq_large_packet(adapter, queue_index);
+
+ skb = ibmveth_rxq_get_buffer(adapter, queue_index);
+ if (unlikely(!skb)) {
+ if (!ibmveth_poll_skip_bad_correlator(netdev, adapter,
+ queue_index))
+ return -1;
+ return 0;
+ }
+
+ room = skb_tailroom(skb);
+ off = offset;
+ len = length;
+ if (unlikely(off >= room || len > room - off)) {
+ if (!ibmveth_poll_drop_oversize(netdev, adapter, queue_index,
+ off, len, room))
+ return -1;
+ return 0;
+ }
+
+ if (lrg_pkt) {
+ __be64 *rxmss = (__be64 *)(skb->data + 8);
+
+ mss = (u16)be64_to_cpu(*rxmss);
+ }
+
+ new_skb = NULL;
+ if (length < rx_copybreak)
+ new_skb = netdev_alloc_skb(netdev, length);
+
+ if (new_skb) {
+ skb_copy_to_linear_data(new_skb, skb->data + offset, length);
+ if (rx_flush)
+ ibmveth_flush_buffer(skb->data, length + offset);
+ rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, true);
+ if (unlikely(rc)) {
+ kfree_skb(new_skb);
+ return -1;
+ }
+ skb = new_skb;
+ } else {
+ rc = ibmveth_rxq_harvest_buffer(adapter, queue_index, false);
+ if (unlikely(rc))
+ return -1;
+ skb_reserve(skb, offset);
+ }
+
+ skb_put(skb, length);
+ skb->protocol = eth_type_trans(skb, netdev);
+
+ if (skb->protocol == cpu_to_be16(ETH_P_IP))
+ iph_check = ip_hdr(skb)->check;
+
+ if ((length > netdev->mtu + ETH_HLEN) || lrg_pkt ||
+ iph_check == 0xffff) {
+ ibmveth_rx_mss_helper(skb, mss, lrg_pkt);
+ adapter->rx_large_packets++;
+ }
+
+ if (csum_good) {
+ skb->ip_summed = CHECKSUM_UNNECESSARY;
+ ibmveth_rx_csum_helper(skb, adapter);
+ }
+
+ napi_gro_receive(napi, skb);
+
+ netdev->stats.rx_packets++;
+ netdev->stats.rx_bytes += length;
+
+ return 1;
+}
+
static int ibmveth_poll(struct napi_struct *napi, int budget)
{
struct net_device *netdev = napi->dev;
struct ibmveth_adapter *adapter = netdev_priv(netdev);
int frames_processed = 0;
int queue_index, rc;
- u16 mss = 0;
queue_index = napi - adapter->napi;
+ if (WARN_ON(queue_index < 0 ||
+ queue_index >= adapter->num_rx_queues)) {
+ napi_complete_done(napi, 0);
+ return 0;
+ }
+
+ if (ibmveth_poll_stopping(netdev, napi)) {
+ napi_complete_done(napi, 0);
+ return 0;
+ }
+
restart_poll:
while (frames_processed < budget) {
+ if (ibmveth_poll_stopping(netdev, napi))
+ break;
+
if (!ibmveth_rxq_pending_buffer(adapter, queue_index))
break;
smp_rmb();
if (!ibmveth_rxq_buffer_valid(adapter, queue_index)) {
wmb(); /* suggested by larson1 */
- adapter->rx_invalid_buffer++;
- netdev_dbg(netdev, "recycling invalid buffer\n");
- rc = ibmveth_rxq_harvest_buffer(adapter,
- queue_index, true);
- if (unlikely(rc))
+ if (!ibmveth_poll_recycle_invalid(netdev, adapter,
+ queue_index))
break;
} else {
- struct sk_buff *skb, *new_skb;
- int length = ibmveth_rxq_frame_length(adapter,
- queue_index);
- int offset = ibmveth_rxq_frame_offset(adapter,
- queue_index);
- int csum_good = ibmveth_rxq_csum_good(adapter,
- queue_index);
- int lrg_pkt = ibmveth_rxq_large_packet(adapter,
- queue_index);
- __sum16 iph_check = 0;
-
- skb = ibmveth_rxq_get_buffer(adapter, queue_index);
- if (unlikely(!skb))
+ rc = ibmveth_poll_deliver_frame(napi, adapter, netdev,
+ queue_index);
+ if (rc < 0)
break;
-
- /* if the large packet bit is set in the rx queue
- * descriptor, the mss will be written by PHYP eight
- * bytes from the start of the rx buffer, which is
- * skb->data at this stage
- */
- if (lrg_pkt) {
- __be64 *rxmss = (__be64 *)(skb->data + 8);
-
- mss = (u16)be64_to_cpu(*rxmss);
- }
-
- new_skb = NULL;
- if (length < rx_copybreak)
- new_skb = netdev_alloc_skb(netdev, length);
-
- if (new_skb) {
- skb_copy_to_linear_data(new_skb,
- skb->data + offset,
- length);
- if (rx_flush)
- ibmveth_flush_buffer(skb->data,
- length + offset);
- rc = ibmveth_rxq_harvest_buffer(adapter,
- queue_index,
- true);
- if (unlikely(rc))
- break;
- skb = new_skb;
- } else {
- rc = ibmveth_rxq_harvest_buffer(adapter,
- queue_index,
- false);
- if (unlikely(rc))
- break;
- skb_reserve(skb, offset);
- }
-
- skb_put(skb, length);
- skb->protocol = eth_type_trans(skb, netdev);
-
- /* PHYP without PLSO support places a -1 in the ip
- * checksum for large send frames.
- */
- if (skb->protocol == cpu_to_be16(ETH_P_IP)) {
- struct iphdr *iph = (struct iphdr *)skb->data;
-
- iph_check = iph->check;
- }
-
- if ((length > netdev->mtu + ETH_HLEN) ||
- lrg_pkt || iph_check == 0xffff) {
- ibmveth_rx_mss_helper(skb, mss, lrg_pkt);
- adapter->rx_large_packets++;
- }
-
- if (csum_good) {
- skb->ip_summed = CHECKSUM_UNNECESSARY;
- ibmveth_rx_csum_helper(skb, adapter);
- }
-
- napi_gro_receive(napi, skb); /* send it up */
-
- netdev->stats.rx_packets++;
- netdev->stats.rx_bytes += length;
- frames_processed++;
+ if (rc > 0)
+ frames_processed++;
}
}
ibmveth_replenish_task(adapter, queue_index);
+ if (ibmveth_poll_stopping(netdev, napi)) {
+ napi_complete_done(napi, frames_processed);
+ /* After complete, must not return budget (NAPI resched). */
+ if (frames_processed < budget)
+ return frames_processed;
+ return budget - 1;
+ }
+
if (frames_processed == budget)
goto out;
if (!napi_complete_done(napi, frames_processed))
goto out;
- /* We think we are done - reenable interrupts,
- * then check once more to make sure we are done.
+ /*
+ * napi_disable() sets DISABLE then waits for this poll. Without a
+ * second stopping check here, enable_irq() can re-arm PHYP after
+ * resize already masked the queue ??? late IRQs then hit the handler
+ * after num_rx_queues was published lower (lab WARN at interrupt).
*/
+ if (ibmveth_poll_stopping(netdev, napi))
+ goto out;
+
rc = ibmveth_enable_irq(adapter, queue_index);
if (rc) {
netdev_err(netdev,
--
2.50.1 (Apple Git-155)
