Author: np
Date: Sat Aug 11 04:55:47 2018
New Revision: 337609
URL: https://svnweb.freebsd.org/changeset/base/337609

Log:
  cxgbe(4): Create two variants of service_iq, one for queues with
  freelists and one for those without.
  
  MFH:          3 weeks
  Sponsored by: Chelsio Communications

Modified:
  head/sys/dev/cxgbe/adapter.h
  head/sys/dev/cxgbe/t4_main.c
  head/sys/dev/cxgbe/t4_netmap.c
  head/sys/dev/cxgbe/t4_sge.c

Modified: head/sys/dev/cxgbe/adapter.h
==============================================================================
--- head/sys/dev/cxgbe/adapter.h        Sat Aug 11 04:26:29 2018        
(r337608)
+++ head/sys/dev/cxgbe/adapter.h        Sat Aug 11 04:55:47 2018        
(r337609)
@@ -670,6 +670,7 @@ struct sge_wrq {
 
 #define INVALID_NM_RXQ_CNTXT_ID ((uint16_t)(-1))
 struct sge_nm_rxq {
+       volatile int nm_state;  /* NM_OFF, NM_ON, or NM_BUSY */
        struct vi_info *vi;
 
        struct iq_desc *iq_desc;
@@ -797,7 +798,6 @@ struct adapter {
        struct irq {
                struct resource *res;
                int rid;
-               volatile int nm_state;  /* NM_OFF, NM_ON, or NM_BUSY */
                void *tag;
                struct sge_rxq *rxq;
                struct sge_nm_rxq *nm_rxq;
@@ -1186,9 +1186,10 @@ void release_tid(struct adapter *, int, struct sge_wrq
 
 #ifdef DEV_NETMAP
 /* t4_netmap.c */
+struct sge_nm_rxq;
 void cxgbe_nm_attach(struct vi_info *);
 void cxgbe_nm_detach(struct vi_info *);
-void t4_nm_intr(void *);
+void service_nm_rxq(struct sge_nm_rxq *);
 #endif
 
 /* t4_sge.c */
@@ -1207,7 +1208,10 @@ int t4_setup_vi_queues(struct vi_info *);
 int t4_teardown_vi_queues(struct vi_info *);
 void t4_intr_all(void *);
 void t4_intr(void *);
+#ifdef DEV_NETMAP
+void t4_nm_intr(void *);
 void t4_vi_intr(void *);
+#endif
 void t4_intr_err(void *);
 void t4_intr_evt(void *);
 void t4_wrq_tx_locked(struct adapter *, struct sge_wrq *, struct wrqe *);

Modified: head/sys/dev/cxgbe/t4_main.c
==============================================================================
--- head/sys/dev/cxgbe/t4_main.c        Sat Aug 11 04:26:29 2018        
(r337608)
+++ head/sys/dev/cxgbe/t4_main.c        Sat Aug 11 04:55:47 2018        
(r337609)
@@ -684,8 +684,8 @@ struct {
 
 #ifdef TCP_OFFLOAD
 /*
- * service_iq() has an iq and needs the fl.  Offset of fl from the iq should be
- * exactly the same for both rxq and ofld_rxq.
+ * service_iq_fl() has an iq and needs the fl.  Offset of fl from the iq should
+ * be exactly the same for both rxq and ofld_rxq.
  */
 CTASSERT(offsetof(struct sge_ofld_rxq, iq) == offsetof(struct sge_rxq, iq));
 CTASSERT(offsetof(struct sge_ofld_rxq, fl) == offsetof(struct sge_rxq, fl));
@@ -4819,9 +4819,26 @@ t4_setup_intr_handlers(struct adapter *sc)
 #ifdef DEV_NETMAP
                                        if (q < vi->nnmrxq)
                                                irq->nm_rxq = nm_rxq++;
+
+                                       if (irq->nm_rxq != NULL &&
+                                           irq->rxq == NULL) {
+                                               /* Netmap rx only */
+                                               rc = t4_alloc_irq(sc, irq, rid,
+                                                   t4_nm_intr, irq->nm_rxq, s);
+                                       }
+                                       if (irq->nm_rxq != NULL &&
+                                           irq->rxq != NULL) {
+                                               /* NIC and Netmap rx */
+                                               rc = t4_alloc_irq(sc, irq, rid,
+                                                   t4_vi_intr, irq, s);
+                                       }
 #endif
-                                       rc = t4_alloc_irq(sc, irq, rid,
-                                           t4_vi_intr, irq, s);
+                                       if (irq->rxq != NULL &&
+                                           irq->nm_rxq == NULL) {
+                                               /* NIC rx only */
+                                               rc = t4_alloc_irq(sc, irq, rid,
+                                                   t4_intr, irq->rxq, s);
+                                       }
                                        if (rc != 0)
                                                return (rc);
 #ifdef RSS

Modified: head/sys/dev/cxgbe/t4_netmap.c
==============================================================================
--- head/sys/dev/cxgbe/t4_netmap.c      Sat Aug 11 04:26:29 2018        
(r337608)
+++ head/sys/dev/cxgbe/t4_netmap.c      Sat Aug 11 04:55:47 2018        
(r337609)
@@ -358,8 +358,6 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi
        nm_set_native_flags(na);
 
        for_each_nm_rxq(vi, i, nm_rxq) {
-               struct irq *irq = &sc->irq[vi->first_intr + i];
-
                kring = na->rx_rings[nm_rxq->nid];
                if (!nm_kring_pending_on(kring) ||
                    nm_rxq->iq_cntxt_id != INVALID_NM_RXQ_CNTXT_ID)
@@ -387,7 +385,7 @@ cxgbe_netmap_on(struct adapter *sc, struct vi_info *vi
                t4_write_reg(sc, sc->sge_kdoorbell_reg,
                    nm_rxq->fl_db_val | V_PIDX(j));
 
-               atomic_cmpset_int(&irq->nm_state, NM_OFF, NM_ON);
+               atomic_cmpset_int(&nm_rxq->nm_state, NM_OFF, NM_ON);
        }
 
        for_each_nm_txq(vi, i, nm_txq) {
@@ -459,14 +457,12 @@ cxgbe_netmap_off(struct adapter *sc, struct vi_info *v
                free_nm_txq_hwq(vi, nm_txq);
        }
        for_each_nm_rxq(vi, i, nm_rxq) {
-               struct irq *irq = &sc->irq[vi->first_intr + i];
-
                kring = na->rx_rings[nm_rxq->nid];
                if (!nm_kring_pending_off(kring) ||
                    nm_rxq->iq_cntxt_id == INVALID_NM_RXQ_CNTXT_ID)
                        continue;
 
-               while (!atomic_cmpset_int(&irq->nm_state, NM_ON, NM_OFF))
+               while (!atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_OFF))
                        pause("nmst", 1);
 
                free_nm_rxq_hwq(vi, nm_rxq);
@@ -955,9 +951,8 @@ handle_nm_sge_egr_update(struct adapter *sc, struct if
 }
 
 void
-t4_nm_intr(void *arg)
+service_nm_rxq(struct sge_nm_rxq *nm_rxq)
 {
-       struct sge_nm_rxq *nm_rxq = arg;
        struct vi_info *vi = nm_rxq->vi;
        struct adapter *sc = vi->pi->adapter;
        struct ifnet *ifp = vi->ifp;

Modified: head/sys/dev/cxgbe/t4_sge.c
==============================================================================
--- head/sys/dev/cxgbe/t4_sge.c Sat Aug 11 04:26:29 2018        (r337608)
+++ head/sys/dev/cxgbe/t4_sge.c Sat Aug 11 04:55:47 2018        (r337609)
@@ -204,6 +204,7 @@ struct sgl {
 };
 
 static int service_iq(struct sge_iq *, int);
+static int service_iq_fl(struct sge_iq *, int);
 static struct mbuf *get_fl_payload(struct adapter *, struct sge_fl *, 
uint32_t);
 static int t4_eth_rx(struct sge_iq *, const struct rss_header *, struct mbuf 
*);
 static inline void init_iq(struct sge_iq *, struct adapter *, int, int, int);
@@ -1333,8 +1334,12 @@ t4_teardown_vi_queues(struct vi_info *vi)
 }
 
 /*
- * Deals with errors and the firmware event queue.  All data rx queues forward
- * their interrupt to the firmware event queue.
+ * Interrupt handler when the driver is using only 1 interrupt.  This is a very
+ * unusual scenario.
+ *
+ * a) Deals with errors, if any.
+ * b) Services firmware event queue, which is taking interrupts for all other
+ *    queues.
  */
 void
 t4_intr_all(void *arg)
@@ -1342,14 +1347,16 @@ t4_intr_all(void *arg)
        struct adapter *sc = arg;
        struct sge_iq *fwq = &sc->sge.fwq;
 
+       MPASS(sc->intr_count == 1);
+
        t4_intr_err(arg);
-       if (atomic_cmpset_int(&fwq->state, IQS_IDLE, IQS_BUSY)) {
-               service_iq(fwq, 0);
-               atomic_cmpset_int(&fwq->state, IQS_BUSY, IQS_IDLE);
-       }
+       t4_intr_evt(fwq);
 }
 
-/* Deals with error interrupts */
+/*
+ * Interrupt handler for errors (installed directly when multiple interrupts 
are
+ * being used, or called by t4_intr_all).
+ */
 void
 t4_intr_err(void *arg)
 {
@@ -1359,6 +1366,10 @@ t4_intr_err(void *arg)
        t4_slow_intr_handler(sc);
 }
 
+/*
+ * Interrupt handler for iq-only queues.  The firmware event queue is the only
+ * such queue right now.
+ */
 void
 t4_intr_evt(void *arg)
 {
@@ -1370,90 +1381,74 @@ t4_intr_evt(void *arg)
        }
 }
 
+/*
+ * Interrupt handler for iq+fl queues.
+ */
 void
 t4_intr(void *arg)
 {
        struct sge_iq *iq = arg;
 
        if (atomic_cmpset_int(&iq->state, IQS_IDLE, IQS_BUSY)) {
-               service_iq(iq, 0);
+               service_iq_fl(iq, 0);
                atomic_cmpset_int(&iq->state, IQS_BUSY, IQS_IDLE);
        }
 }
 
+#ifdef DEV_NETMAP
+/*
+ * Interrupt handler for netmap rx queues.
+ */
 void
-t4_vi_intr(void *arg)
+t4_nm_intr(void *arg)
 {
-       struct irq *irq = arg;
+       struct sge_nm_rxq *nm_rxq = arg;
 
-#ifdef DEV_NETMAP
-       if (atomic_cmpset_int(&irq->nm_state, NM_ON, NM_BUSY)) {
-               t4_nm_intr(irq->nm_rxq);
-               atomic_cmpset_int(&irq->nm_state, NM_BUSY, NM_ON);
+       if (atomic_cmpset_int(&nm_rxq->nm_state, NM_ON, NM_BUSY)) {
+               service_nm_rxq(nm_rxq);
+               atomic_cmpset_int(&nm_rxq->nm_state, NM_BUSY, NM_ON);
        }
-#endif
-       if (irq->rxq != NULL)
-               t4_intr(irq->rxq);
 }
 
-static inline int
-sort_before_lro(struct lro_ctrl *lro)
+/*
+ * Interrupt handler for vectors shared between NIC and netmap rx queues.
+ */
+void
+t4_vi_intr(void *arg)
 {
+       struct irq *irq = arg;
 
-       return (lro->lro_mbuf_max != 0);
+       MPASS(irq->nm_rxq != NULL);
+       t4_nm_intr(irq->nm_rxq);
+
+       MPASS(irq->rxq != NULL);
+       t4_intr(irq->rxq);
 }
+#endif
 
 /*
- * Deals with anything and everything on the given ingress queue.
+ * Deals with interrupts on an iq-only (no freelist) queue.
  */
 static int
 service_iq(struct sge_iq *iq, int budget)
 {
        struct sge_iq *q;
-       struct sge_rxq *rxq = iq_to_rxq(iq);    /* Use iff iq is part of rxq */
-       struct sge_fl *fl;                      /* Use iff IQ_HAS_FL */
        struct adapter *sc = iq->adapter;
        struct iq_desc *d = &iq->desc[iq->cidx];
        int ndescs = 0, limit;
-       int rsp_type, refill;
+       int rsp_type;
        uint32_t lq;
-       uint16_t fl_hw_cidx;
-       struct mbuf *m0;
        STAILQ_HEAD(, sge_iq) iql = STAILQ_HEAD_INITIALIZER(iql);
-#if defined(INET) || defined(INET6)
-       const struct timeval lro_timeout = {0, sc->lro_timeout};
-       struct lro_ctrl *lro = &rxq->lro;
-#endif
 
        KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
+       KASSERT((iq->flags & IQ_HAS_FL) == 0,
+           ("%s: called for iq %p with fl (iq->flags 0x%x)", __func__, iq,
+           iq->flags));
+       MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
+       MPASS((iq->flags & IQ_LRO_ENABLED) == 0);
 
        limit = budget ? budget : iq->qsize / 16;
 
-       if (iq->flags & IQ_HAS_FL) {
-               fl = &rxq->fl;
-               fl_hw_cidx = fl->hw_cidx;       /* stable snapshot */
-       } else {
-               fl = NULL;
-               fl_hw_cidx = 0;                 /* to silence gcc warning */
-       }
-
-#if defined(INET) || defined(INET6)
-       if (iq->flags & IQ_ADJ_CREDIT) {
-               MPASS(sort_before_lro(lro));
-               iq->flags &= ~IQ_ADJ_CREDIT;
-               if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
-                       tcp_lro_flush_all(lro);
-                       t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
-                           V_INGRESSQID((u32)iq->cntxt_id) |
-                           V_SEINTARM(iq->intr_params));
-                       return (0);
-               }
-               ndescs = 1;
-       }
-#else
-       MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
-#endif
-
        /*
         * We always come back and check the descriptor ring for new indirect
         * interrupts and other responses after running a single handler.
@@ -1463,74 +1458,40 @@ service_iq(struct sge_iq *iq, int budget)
 
                        rmb();
 
-                       refill = 0;
-                       m0 = NULL;
                        rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
                        lq = be32toh(d->rsp.pldbuflen_qid);
 
                        switch (rsp_type) {
                        case X_RSPD_TYPE_FLBUF:
+                               panic("%s: data for an iq (%p) with no 
freelist",
+                                   __func__, iq);
 
-                               KASSERT(iq->flags & IQ_HAS_FL,
-                                   ("%s: data for an iq (%p) with no freelist",
-                                   __func__, iq));
+                               /* NOTREACHED */
 
-                               m0 = get_fl_payload(sc, fl, lq);
-                               if (__predict_false(m0 == NULL))
-                                       goto process_iql;
-                               refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, 
fl->sidx) > 2;
-#ifdef T4_PKT_TIMESTAMP
-                               /*
-                                * 60 bit timestamp for the payload is
-                                * *(uint64_t *)m0->m_pktdat.  Note that it is
-                                * in the leading free-space in the mbuf.  The
-                                * kernel can clobber it during a pullup,
-                                * m_copymdata, etc.  You need to make sure that
-                                * the mbuf reaches you unmolested if you care
-                                * about the timestamp.
-                                */
-                               *(uint64_t *)m0->m_pktdat =
-                                   be64toh(ctrl->u.last_flit) &
-                                   0xfffffffffffffff;
-#endif
-
-                               /* fall through */
-
                        case X_RSPD_TYPE_CPL:
                                KASSERT(d->rss.opcode < NUM_CPL_CMDS,
                                    ("%s: bad opcode %02x.", __func__,
                                    d->rss.opcode));
-                               t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
+                               t4_cpl_handler[d->rss.opcode](iq, &d->rss, 
NULL);
                                break;
 
                        case X_RSPD_TYPE_INTR:
-
                                /*
-                                * Interrupts should be forwarded only to queues
-                                * that are not forwarding their interrupts.
-                                * This means service_iq can recurse but only 1
-                                * level deep.
-                                */
-                               KASSERT(budget == 0,
-                                   ("%s: budget %u, rsp_type %u", __func__,
-                                   budget, rsp_type));
-
-                               /*
                                 * There are 1K interrupt-capable queues (qids 0
                                 * through 1023).  A response type indicating a
                                 * forwarded interrupt with a qid >= 1K is an
                                 * iWARP async notification.
                                 */
-                               if (lq >= 1024) {
-                                        t4_an_handler(iq, &d->rsp);
-                                        break;
-                                }
+                               if (__predict_true(lq >= 1024)) {
+                                       t4_an_handler(iq, &d->rsp);
+                                       break;
+                               }
 
                                q = sc->sge.iqmap[lq - sc->sge.iq_start -
                                    sc->sge.iq_base];
                                if (atomic_cmpset_int(&q->state, IQS_IDLE,
                                    IQS_BUSY)) {
-                                       if (service_iq(q, q->qsize / 16) == 0) {
+                                       if (service_iq_fl(q, q->qsize / 16) == 
0) {
                                                atomic_cmpset_int(&q->state,
                                                    IQS_BUSY, IQS_IDLE);
                                        } else {
@@ -1563,33 +1524,12 @@ service_iq(struct sge_iq *iq, int budget)
                                    
V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
                                ndescs = 0;
 
-#if defined(INET) || defined(INET6)
-                               if (iq->flags & IQ_LRO_ENABLED &&
-                                   !sort_before_lro(lro) &&
-                                   sc->lro_timeout != 0) {
-                                       tcp_lro_flush_inactive(lro,
-                                           &lro_timeout);
-                               }
-#endif
-
                                if (budget) {
-                                       if (iq->flags & IQ_HAS_FL) {
-                                               FL_LOCK(fl);
-                                               refill_fl(sc, fl, 32);
-                                               FL_UNLOCK(fl);
-                                       }
                                        return (EINPROGRESS);
                                }
                        }
-                       if (refill) {
-                               FL_LOCK(fl);
-                               refill_fl(sc, fl, 32);
-                               FL_UNLOCK(fl);
-                               fl_hw_cidx = fl->hw_cidx;
-                       }
                }
 
-process_iql:
                if (STAILQ_EMPTY(&iql))
                        break;
 
@@ -1599,13 +1539,168 @@ process_iql:
                 */
                q = STAILQ_FIRST(&iql);
                STAILQ_REMOVE_HEAD(&iql, link);
-               if (service_iq(q, q->qsize / 8) == 0)
+               if (service_iq_fl(q, q->qsize / 8) == 0)
                        atomic_cmpset_int(&q->state, IQS_BUSY, IQS_IDLE);
                else
                        STAILQ_INSERT_TAIL(&iql, q, link);
        }
 
+       t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
+           V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
+
+       return (0);
+}
+
+static inline int
+sort_before_lro(struct lro_ctrl *lro)
+{
+
+       return (lro->lro_mbuf_max != 0);
+}
+
+/*
+ * Deals with interrupts on an iq+fl queue.
+ */
+static int
+service_iq_fl(struct sge_iq *iq, int budget)
+{
+       struct sge_rxq *rxq = iq_to_rxq(iq);
+       struct sge_fl *fl;
+       struct adapter *sc = iq->adapter;
+       struct iq_desc *d = &iq->desc[iq->cidx];
+       int ndescs = 0, limit;
+       int rsp_type, refill, starved;
+       uint32_t lq;
+       uint16_t fl_hw_cidx;
+       struct mbuf *m0;
 #if defined(INET) || defined(INET6)
+       const struct timeval lro_timeout = {0, sc->lro_timeout};
+       struct lro_ctrl *lro = &rxq->lro;
+#endif
+
+       KASSERT(iq->state == IQS_BUSY, ("%s: iq %p not BUSY", __func__, iq));
+       MPASS(iq->flags & IQ_HAS_FL);
+
+       limit = budget ? budget : iq->qsize / 16;
+       fl = &rxq->fl;
+       fl_hw_cidx = fl->hw_cidx;       /* stable snapshot */
+
+#if defined(INET) || defined(INET6)
+       if (iq->flags & IQ_ADJ_CREDIT) {
+               MPASS(sort_before_lro(lro));
+               iq->flags &= ~IQ_ADJ_CREDIT;
+               if ((d->rsp.u.type_gen & F_RSPD_GEN) != iq->gen) {
+                       tcp_lro_flush_all(lro);
+                       t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(1) |
+                           V_INGRESSQID((u32)iq->cntxt_id) |
+                           V_SEINTARM(iq->intr_params));
+                       return (0);
+               }
+               ndescs = 1;
+       }
+#else
+       MPASS((iq->flags & IQ_ADJ_CREDIT) == 0);
+#endif
+
+       while ((d->rsp.u.type_gen & F_RSPD_GEN) == iq->gen) {
+
+               rmb();
+
+               refill = 0;
+               m0 = NULL;
+               rsp_type = G_RSPD_TYPE(d->rsp.u.type_gen);
+               lq = be32toh(d->rsp.pldbuflen_qid);
+
+               switch (rsp_type) {
+               case X_RSPD_TYPE_FLBUF:
+
+                       m0 = get_fl_payload(sc, fl, lq);
+                       if (__predict_false(m0 == NULL))
+                               goto out;
+                       refill = IDXDIFF(fl->hw_cidx, fl_hw_cidx, fl->sidx) > 2;
+#ifdef T4_PKT_TIMESTAMP
+                       /*
+                        * 60 bit timestamp for the payload is
+                        * *(uint64_t *)m0->m_pktdat.  Note that it is
+                        * in the leading free-space in the mbuf.  The
+                        * kernel can clobber it during a pullup,
+                        * m_copymdata, etc.  You need to make sure that
+                        * the mbuf reaches you unmolested if you care
+                        * about the timestamp.
+                        */
+                       *(uint64_t *)m0->m_pktdat =
+                           be64toh(ctrl->u.last_flit) & 0xfffffffffffffff;
+#endif
+
+                       /* fall through */
+
+               case X_RSPD_TYPE_CPL:
+                       KASSERT(d->rss.opcode < NUM_CPL_CMDS,
+                           ("%s: bad opcode %02x.", __func__, d->rss.opcode));
+                       t4_cpl_handler[d->rss.opcode](iq, &d->rss, m0);
+                       break;
+
+               case X_RSPD_TYPE_INTR:
+
+                       /*
+                        * There are 1K interrupt-capable queues (qids 0
+                        * through 1023).  A response type indicating a
+                        * forwarded interrupt with a qid >= 1K is an
+                        * iWARP async notification.  That is the only
+                        * acceptable indirect interrupt on this queue.
+                        */
+                       if (__predict_false(lq < 1024)) {
+                               panic("%s: indirect interrupt on iq_fl %p "
+                                   "with qid %u", __func__, iq, lq);
+                       }
+
+                       t4_an_handler(iq, &d->rsp);
+                       break;
+
+               default:
+                       KASSERT(0, ("%s: illegal response type %d on iq %p",
+                           __func__, rsp_type, iq));
+                       log(LOG_ERR, "%s: illegal response type %d on iq %p",
+                           device_get_nameunit(sc->dev), rsp_type, iq);
+                       break;
+               }
+
+               d++;
+               if (__predict_false(++iq->cidx == iq->sidx)) {
+                       iq->cidx = 0;
+                       iq->gen ^= F_RSPD_GEN;
+                       d = &iq->desc[0];
+               }
+               if (__predict_false(++ndescs == limit)) {
+                       t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
+                           V_INGRESSQID(iq->cntxt_id) |
+                           
V_SEINTARM(V_QINTR_TIMER_IDX(X_TIMERREG_UPDATE_CIDX)));
+                       ndescs = 0;
+
+#if defined(INET) || defined(INET6)
+                       if (iq->flags & IQ_LRO_ENABLED &&
+                           !sort_before_lro(lro) &&
+                           sc->lro_timeout != 0) {
+                               tcp_lro_flush_inactive(lro, &lro_timeout);
+                       }
+#endif
+                       if (budget) {
+                               FL_LOCK(fl);
+                               refill_fl(sc, fl, 32);
+                               FL_UNLOCK(fl);
+
+                               return (EINPROGRESS);
+                       }
+               }
+               if (refill) {
+                       FL_LOCK(fl);
+                       refill_fl(sc, fl, 32);
+                       FL_UNLOCK(fl);
+                       fl_hw_cidx = fl->hw_cidx;
+               }
+       }
+out:
+#if defined(INET) || defined(INET6)
        if (iq->flags & IQ_LRO_ENABLED) {
                if (ndescs > 0 && lro->lro_mbuf_count > 8) {
                        MPASS(sort_before_lro(lro));
@@ -1621,15 +1716,11 @@ process_iql:
        t4_write_reg(sc, sc->sge_gts_reg, V_CIDXINC(ndescs) |
            V_INGRESSQID((u32)iq->cntxt_id) | V_SEINTARM(iq->intr_params));
 
-       if (iq->flags & IQ_HAS_FL) {
-               int starved;
-
-               FL_LOCK(fl);
-               starved = refill_fl(sc, fl, 64);
-               FL_UNLOCK(fl);
-               if (__predict_false(starved != 0))
-                       add_fl_to_sfl(sc, fl);
-       }
+       FL_LOCK(fl);
+       starved = refill_fl(sc, fl, 64);
+       FL_UNLOCK(fl);
+       if (__predict_false(starved != 0))
+               add_fl_to_sfl(sc, fl);
 
        return (0);
 }
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"

Reply via email to