The branch main has been updated by jhb:
URL:
https://cgit.FreeBSD.org/src/commit/?id=4d3b659f24a88f76557019e6f3192760bdebfbdf
commit 4d3b659f24a88f76557019e6f3192760bdebfbdf
Author: John Baldwin <j...@freebsd.org>
AuthorDate: 2024-11-11 16:39:05 +0000
Commit: John Baldwin <j...@freebsd.org>
CommitDate: 2024-11-11 16:39:05 +0000
nvmf: Track SQ flow control
This isn't really needed since the host driver never submits more
commands to a queue than it can hold, but I noticed that the
recently-added SQ head and tail sysctl nodes were not updating. This
fixes that and also uses these values to assert that there we never
submit a command while a queue pair is full.
Sponsored by: Chelsio Communications
---
sys/dev/nvmf/host/nvmf_qpair.c | 33 ++++++++++++++++++++++++++++-----
sys/dev/nvmf/nvmf_transport.c | 8 ++++++++
sys/dev/nvmf/nvmf_transport.h | 1 +
3 files changed, 37 insertions(+), 5 deletions(-)
diff --git a/sys/dev/nvmf/host/nvmf_qpair.c b/sys/dev/nvmf/host/nvmf_qpair.c
index 1aeb0535eacf..b03ecfa081d3 100644
--- a/sys/dev/nvmf/host/nvmf_qpair.c
+++ b/sys/dev/nvmf/host/nvmf_qpair.c
@@ -115,8 +115,23 @@ nvmf_dispatch_command(struct nvmf_host_qpair *qp, struct
nvmf_host_command *cmd)
struct nvmf_softc *sc = qp->sc;
struct nvme_command *sqe;
struct nvmf_capsule *nc;
+ uint16_t new_sqtail;
int error;
+ mtx_assert(&qp->lock, MA_OWNED);
+
+ qp->submitted++;
+
+ /*
+ * Update flow control tracking. This is just a sanity check.
+ * Since num_commands == qsize - 1, there can never be too
+ * many commands in flight.
+ */
+ new_sqtail = (qp->sqtail + 1) % (qp->num_commands + 1);
+ KASSERT(new_sqtail != qp->sqhd, ("%s: qp %p is full", __func__, qp));
+ qp->sqtail = new_sqtail;
+ mtx_unlock(&qp->lock);
+
nc = cmd->req->nc;
sqe = nvmf_capsule_sqe(nc);
@@ -180,11 +195,23 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc)
return;
}
+ /* Update flow control tracking. */
+ mtx_lock(&qp->lock);
+ if (qp->sq_flow_control) {
+ if (nvmf_sqhd_valid(nc))
+ qp->sqhd = le16toh(cqe->sqhd);
+ } else {
+ /*
+ * If SQ FC is disabled, just advance the head for
+ * each response capsule received.
+ */
+ qp->sqhd = (qp->sqhd + 1) % (qp->num_commands + 1);
+ }
+
/*
* If the queue has been shutdown due to an error, silently
* drop the response.
*/
- mtx_lock(&qp->lock);
if (qp->qp == NULL) {
device_printf(sc->dev,
"received completion for CID %u on shutdown %s\n", cid,
@@ -215,8 +242,6 @@ nvmf_receive_capsule(void *arg, struct nvmf_capsule *nc)
} else {
cmd->req = STAILQ_FIRST(&qp->pending_requests);
STAILQ_REMOVE_HEAD(&qp->pending_requests, link);
- qp->submitted++;
- mtx_unlock(&qp->lock);
nvmf_dispatch_command(qp, cmd);
}
@@ -420,7 +445,5 @@ nvmf_submit_request(struct nvmf_request *req)
("%s: CID already busy", __func__));
qp->active_commands[cmd->cid] = cmd;
cmd->req = req;
- qp->submitted++;
- mtx_unlock(&qp->lock);
nvmf_dispatch_command(qp, cmd);
}
diff --git a/sys/dev/nvmf/nvmf_transport.c b/sys/dev/nvmf/nvmf_transport.c
index ea4aee8cc7ae..316d1571e61d 100644
--- a/sys/dev/nvmf/nvmf_transport.c
+++ b/sys/dev/nvmf/nvmf_transport.c
@@ -180,6 +180,14 @@ nvmf_capsule_cqe(struct nvmf_capsule *nc)
return (&nc->nc_cqe);
}
+bool
+nvmf_sqhd_valid(struct nvmf_capsule *nc)
+{
+ KASSERT(nc->nc_qe_len == sizeof(struct nvme_completion),
+ ("%s: capsule %p is not a response capsule", __func__, nc));
+ return (nc->nc_sqhd_valid);
+}
+
uint8_t
nvmf_validate_command_capsule(struct nvmf_capsule *nc)
{
diff --git a/sys/dev/nvmf/nvmf_transport.h b/sys/dev/nvmf/nvmf_transport.h
index 549170b25940..bbd830eba576 100644
--- a/sys/dev/nvmf/nvmf_transport.h
+++ b/sys/dev/nvmf/nvmf_transport.h
@@ -78,6 +78,7 @@ int nvmf_transmit_capsule(struct nvmf_capsule *nc);
void nvmf_abort_capsule_data(struct nvmf_capsule *nc, int error);
void *nvmf_capsule_sqe(struct nvmf_capsule *nc);
void *nvmf_capsule_cqe(struct nvmf_capsule *nc);
+bool nvmf_sqhd_valid(struct nvmf_capsule *nc);
/* Controller-specific APIs. */
