The branch main has been updated by np:
URL:
https://cgit.FreeBSD.org/src/commit/?id=0a9d1da6e6cede5e9c0ff63240d724049ad72b5b
commit 0a9d1da6e6cede5e9c0ff63240d724049ad72b5b
Author: Navdeep Parhar <n...@freebsd.org>
AuthorDate: 2024-07-31 19:27:18 +0000
Commit: Navdeep Parhar <n...@freebsd.org>
CommitDate: 2024-08-17 18:23:32 +0000
cxgbe(4): Stop work request queues in a reliable manner.
Clear the EQ_HW_ALLOCATED flag with the wrq lock held and discard all
work requests, pending or new, when it's not set.
MFC after: 1 week
Sponsored by: Chelsio Communications
---
sys/dev/cxgbe/adapter.h | 5 ++++-
sys/dev/cxgbe/t4_main.c | 20 +++++++++++++++++++-
sys/dev/cxgbe/t4_sge.c | 15 ++++++++++++++-
3 files changed, 37 insertions(+), 3 deletions(-)
diff --git a/sys/dev/cxgbe/adapter.h b/sys/dev/cxgbe/adapter.h
index 0d731e736823..3922bd3909fe 100644
--- a/sys/dev/cxgbe/adapter.h
+++ b/sys/dev/cxgbe/adapter.h
@@ -1561,7 +1561,10 @@ t4_wrq_tx(struct adapter *sc, struct wrqe *wr)
struct sge_wrq *wrq = wr->wrq;
TXQ_LOCK(wrq);
- t4_wrq_tx_locked(sc, wrq, wr);
+ if (__predict_true(wrq->eq.flags & EQ_HW_ALLOCATED))
+ t4_wrq_tx_locked(sc, wrq, wr);
+ else
+ free(wr, M_CXGBE);
TXQ_UNLOCK(wrq);
}
diff --git a/sys/dev/cxgbe/t4_main.c b/sys/dev/cxgbe/t4_main.c
index 795c8d7e2e37..57c1eeceab22 100644
--- a/sys/dev/cxgbe/t4_main.c
+++ b/sys/dev/cxgbe/t4_main.c
@@ -2060,7 +2060,9 @@ stop_lld(struct adapter *sc)
}
#if defined(TCP_OFFLOAD) || defined(RATELIMIT)
for_each_ofld_txq(vi, k, ofld_txq) {
+ TXQ_LOCK(&ofld_txq->wrq);
ofld_txq->wrq.eq.flags &= ~EQ_HW_ALLOCATED;
+ TXQ_UNLOCK(&ofld_txq->wrq);
}
#endif
for_each_rxq(vi, k, rxq) {
@@ -2078,7 +2080,9 @@ stop_lld(struct adapter *sc)
if (sc->flags & FULL_INIT_DONE) {
/* Control queue */
wrq = &sc->sge.ctrlq[i];
+ TXQ_LOCK(wrq);
wrq->eq.flags &= ~EQ_HW_ALLOCATED;
+ TXQ_UNLOCK(wrq);
quiesce_wrq(wrq);
}
}
@@ -7047,8 +7051,22 @@ quiesce_txq(struct sge_txq *txq)
static void
quiesce_wrq(struct sge_wrq *wrq)
{
+ struct wrqe *wr;
- /* XXXTX */
+ TXQ_LOCK(wrq);
+ while ((wr = STAILQ_FIRST(&wrq->wr_list)) != NULL) {
+ STAILQ_REMOVE_HEAD(&wrq->wr_list, link);
+#ifdef INVARIANTS
+ wrq->nwr_pending--;
+ wrq->ndesc_needed -= howmany(wr->wr_len, EQ_ESIZE);
+#endif
+ free(wr, M_CXGBE);
+ }
+ MPASS(wrq->nwr_pending == 0);
+ MPASS(wrq->ndesc_needed == 0);
+ wrq->nwr_pending = 0;
+ wrq->ndesc_needed = 0;
+ TXQ_UNLOCK(wrq);
}
static void
diff --git a/sys/dev/cxgbe/t4_sge.c b/sys/dev/cxgbe/t4_sge.c
index b4eb0701821a..bc81a0251deb 100644
--- a/sys/dev/cxgbe/t4_sge.c
+++ b/sys/dev/cxgbe/t4_sge.c
@@ -2921,6 +2921,10 @@ start_wrq_wr(struct sge_wrq *wrq, int len16, struct
wrq_cookie *cookie)
MPASS(ndesc > 0 && ndesc <= SGE_MAX_WR_NDESC);
EQ_LOCK(eq);
+ if (__predict_false((eq->flags & EQ_HW_ALLOCATED) == 0)) {
+ EQ_UNLOCK(eq);
+ return (NULL);
+ }
if (TAILQ_EMPTY(&wrq->incomplete_wrs) && !STAILQ_EMPTY(&wrq->wr_list))
drain_wrq_wr_list(sc, wrq);
@@ -3016,7 +3020,10 @@ commit_wrq_wr(struct sge_wrq *wrq, void *w, struct
wrq_cookie *cookie)
F_FW_WR_EQUEQ);
}
- ring_eq_db(wrq->adapter, eq, ndesc);
+ if (__predict_true(eq->flags & EQ_HW_ALLOCATED))
+ ring_eq_db(wrq->adapter, eq, ndesc);
+ else
+ IDXINCR(eq->dbidx, ndesc, eq->sidx);
} else {
MPASS(IDXDIFF(next->pidx, pidx, eq->sidx) == ndesc);
next->pidx = pidx;
@@ -3852,6 +3859,8 @@ alloc_ctrlq(struct adapter *sc, int idx)
if (!(ctrlq->eq.flags & EQ_HW_ALLOCATED)) {
MPASS(ctrlq->eq.flags & EQ_SW_ALLOCATED);
+ MPASS(ctrlq->nwr_pending == 0);
+ MPASS(ctrlq->ndesc_needed == 0);
rc = alloc_eq_hwq(sc, NULL, &ctrlq->eq);
if (rc != 0) {
@@ -4554,6 +4563,7 @@ free_wrq(struct adapter *sc, struct sge_wrq *wrq)
{
free_eq(sc, &wrq->eq);
MPASS(wrq->nwr_pending == 0);
+ MPASS(wrq->ndesc_needed == 0);
MPASS(TAILQ_EMPTY(&wrq->incomplete_wrs));
MPASS(STAILQ_EMPTY(&wrq->wr_list));
bzero(wrq, sizeof(*wrq));
@@ -4848,6 +4858,9 @@ alloc_ofld_txq(struct vi_info *vi, struct sge_ofld_txq
*ofld_txq, int idx)
}
if (!(eq->flags & EQ_HW_ALLOCATED)) {
+ MPASS(eq->flags & EQ_SW_ALLOCATED);
+ MPASS(ofld_txq->wrq.nwr_pending == 0);
+ MPASS(ofld_txq->wrq.ndesc_needed == 0);
rc = alloc_eq_hwq(sc, vi, eq);
if (rc != 0) {
CH_ERR(vi, "failed to create hw ofld_txq%d: %d\n", idx,
