From: Yufeng Wang <[email protected]>
Implement SQ/CQ doorbell polling in the guest kernel:
- virtio_ring.c: SQ/CQ DMA allocation, virtqueue_notify() writes
sq->idx via smp_store_release instead of MMIO kick; completion
detection via more_used() (direct used-ring check)
- virtio_sqcq_poll.c: per-device CQ poll thread with need_resched()
cooperative spin, time-based spin budget, 1ms idle sleep
- virtio_pci_modern.c: feature negotiation, PCI config space write
for SQ/CQ DMA addresses, poll thread lifecycle
- virtio.c: reject VIRTIO_F_SQCQ_POLL + VIRTIO_F_RING_PACKED
- virtio_scsi.c: 10s window io_stats diagnostics output
- do_softirq() after each completion callback (no hardware interrupt)
Signed-off-by: Yufeng Wang <[email protected]>
---
drivers/scsi/virtio_scsi.c | 113 ++++++++-
drivers/virtio/Makefile | 2 +-
drivers/virtio/virtio.c | 10 +
drivers/virtio/virtio_pci_common.c | 3 +
drivers/virtio/virtio_pci_modern.c | 39 +++
drivers/virtio/virtio_pci_modern_dev.c | 34 ++-
drivers/virtio/virtio_ring.c | 262 +++++++++++++++++++
drivers/virtio/virtio_sqcq_poll.c | 338 +++++++++++++++++++++++++
include/linux/virtio.h | 25 ++
include/linux/virtio_pci_modern.h | 2 +
10 files changed, 825 insertions(+), 3 deletions(-)
create mode 100644 drivers/virtio/virtio_sqcq_poll.c
diff --git a/drivers/scsi/virtio_scsi.c b/drivers/scsi/virtio_scsi.c
index 0ed8558dad72..f803bb62d22a 100644
--- a/drivers/scsi/virtio_scsi.c
+++ b/drivers/scsi/virtio_scsi.c
@@ -30,6 +30,7 @@
#include <scsi/scsi_devinfo.h>
#include <linux/seqlock.h>
#include <linux/dma-mapping.h>
+#include <linux/ktime.h>
#include "sd.h"
@@ -46,6 +47,8 @@ MODULE_PARM_DESC(virtscsi_poll_queues,
struct virtio_scsi_cmd {
struct scsi_cmnd *sc;
struct completion *comp;
+ u64 submit_ns; /* ktime_get_ns() at submission */
+ u64 notify_ns; /* ktime_get_ns() after SQ doorbell */
union {
struct virtio_scsi_cmd_req cmd;
struct virtio_scsi_cmd_req_pi cmd_pi;
@@ -95,6 +98,22 @@ struct virtio_scsi {
struct virtio_scsi_event events[VIRTIO_SCSI_EVENT_LEN];
__dma_from_device_group_end();
+ /* I/O latency statistics (debugging) */
+ spinlock_t lat_lock;
+ u64 lat_sum_ns;
+ u64 lat_min_ns;
+ u64 lat_max_ns;
+ u64 lat_count;
+ u64 host_sum_ns; /* notify → cq_detect (host transit) */
+ u64 cb_wait_sum_ns; /* cq_detect → process (poll sched latency) */
+ u64 det_count;
+ u64 last_cq_detect_ns; /* timestamp of last CQ detection */
+ unsigned long lat_last_print;
+
+ /* Completion interval tracking (matches host interval metric) */
+ u64 last_complete_ns; /* ktime of last completion */
+ u64 ema_cmp_int_ns; /* EMA of completion-to-completion interval */
+
struct virtio_scsi_vq req_vqs[];
};
@@ -122,6 +141,7 @@ static void virtscsi_complete_cmd(struct virtio_scsi
*vscsi, void *buf)
struct virtio_scsi_cmd *cmd = buf;
struct scsi_cmnd *sc = cmd->sc;
struct virtio_scsi_cmd_resp *resp = &cmd->resp.cmd;
+ u64 cb_entry_ns = ktime_get_ns();
dev_dbg(&sc->device->sdev_gendev,
"cmd %p response %u status %#02x sense_len %u\n",
@@ -175,6 +195,88 @@ static void virtscsi_complete_cmd(struct virtio_scsi
*vscsi, void *buf)
VIRTIO_SCSI_SENSE_SIZE));
}
+ /* I/O latency tracking */
+ if (cmd->submit_ns) {
+ u64 now = ktime_get_ns();
+ u64 lat = now - cmd->submit_ns;
+ unsigned long flags;
+
+ spin_lock_irqsave(&vscsi->lat_lock, flags);
+ vscsi->lat_sum_ns += lat;
+ vscsi->lat_count++;
+ if (lat < vscsi->lat_min_ns || vscsi->lat_min_ns == 0)
+ vscsi->lat_min_ns = lat;
+ if (lat > vscsi->lat_max_ns)
+ vscsi->lat_max_ns = lat;
+
+ /* Completion interval EMA (matches host interval metric) */
+ if (vscsi->last_complete_ns) {
+ u64 cmp_int = now - vscsi->last_complete_ns;
+
+ if (vscsi->ema_cmp_int_ns == 0)
+ vscsi->ema_cmp_int_ns = cmp_int;
+ else
+ vscsi->ema_cmp_int_ns =
+ vscsi->ema_cmp_int_ns -
+ vscsi->ema_cmp_int_ns / 8 +
+ cmp_int / 8;
+ }
+ vscsi->last_complete_ns = now;
+
+ if (cmd->notify_ns && vscsi->last_cq_detect_ns) {
+ u64 host_lat = vscsi->last_cq_detect_ns -
+ cmd->notify_ns;
+ u64 cb_wait = cb_entry_ns -
+ vscsi->last_cq_detect_ns;
+
+ if (host_lat < 10ULL * NSEC_PER_SEC) {
+ vscsi->host_sum_ns += host_lat;
+ vscsi->cb_wait_sum_ns += cb_wait;
+ vscsi->det_count++;
+ }
+ }
+
+ /* 10s window — single consolidated output */
+ if (time_after(jiffies, vscsi->lat_last_print +
+ msecs_to_jiffies(10000)) &&
+ vscsi->lat_count > 0) {
+ u64 avg_lat = vscsi->lat_sum_ns / vscsi->lat_count;
+ u64 host_lat = vscsi->det_count > 0 ?
+ vscsi->host_sum_ns / vscsi->det_count : 0;
+ u64 poll_lat = vscsi->det_count > 0 ?
+ vscsi->cb_wait_sum_ns / vscsi->det_count : 0;
+ u64 interval = 0;
+ unsigned int wk_kick;
+ unsigned long sleep_cnt;
+
+ wk_kick = virtqueue_get_sq_notify_stats(
+ vscsi->req_vqs[0].vq, &interval);
+ sleep_cnt =
virtio_sqcq_poll_get_sleep_count(vscsi->vdev);
+
+ dev_info(&vscsi->vdev->dev,
+ "io_stats: cq=%llu avg_lat=%lluns
host_lat=%lluns poll_lat=%lluns min_lat=%lluns max_lat=%lluns interval=%lluns
cmp_interval=%lluns wk_kick=%u sleep=%lu\n",
+ vscsi->lat_count,
+ avg_lat,
+ host_lat,
+ poll_lat,
+ vscsi->lat_min_ns,
+ vscsi->lat_max_ns,
+ interval,
+ vscsi->ema_cmp_int_ns,
+ wk_kick,
+ sleep_cnt);
+ vscsi->lat_sum_ns = 0;
+ vscsi->lat_count = 0;
+ vscsi->lat_min_ns = 0;
+ vscsi->lat_max_ns = 0;
+ vscsi->host_sum_ns = 0;
+ vscsi->cb_wait_sum_ns = 0;
+ vscsi->det_count = 0;
+ vscsi->lat_last_print = jiffies;
+ }
+ spin_unlock_irqrestore(&vscsi->lat_lock, flags);
+ }
+
scsi_done(sc);
}
@@ -204,6 +306,7 @@ static void virtscsi_req_done(struct virtqueue *vq)
int index = vq->index - VIRTIO_SCSI_VQ_BASE;
struct virtio_scsi_vq *req_vq = &vscsi->req_vqs[index];
+ vscsi->last_cq_detect_ns = virtqueue_get_cq_detect_ns(vq);
virtscsi_vq_done(vscsi, req_vq, virtscsi_complete_cmd);
};
@@ -514,8 +617,12 @@ static int virtscsi_add_cmd(struct virtio_scsi_vq *vq,
spin_unlock_irqrestore(&vq->vq_lock, flags);
- if (needs_kick)
+ if (needs_kick) {
+ cmd->notify_ns = ktime_get_ns();
virtqueue_notify(vq->vq);
+ } else {
+ cmd->notify_ns = 0;
+ }
return err;
}
@@ -605,6 +712,7 @@ static enum scsi_qc_status virtscsi_queuecommand(struct
Scsi_Host *shost,
}
kick = (sc->flags & SCMD_LAST) != 0;
+ cmd->submit_ns = ktime_get_ns();
ret = virtscsi_add_cmd(req_vq, cmd, req_size, sizeof(cmd->resp.cmd),
kick);
if (ret == -EIO) {
cmd->resp.cmd.response = VIRTIO_SCSI_S_BAD_TARGET;
@@ -895,6 +1003,9 @@ static int virtscsi_init(struct virtio_device *vdev,
virtscsi_init_vq(&vscsi->req_vqs[i - VIRTIO_SCSI_VQ_BASE],
vqs[i]);
+ spin_lock_init(&vscsi->lat_lock);
+ vscsi->lat_last_print = jiffies;
+
virtscsi_config_set(vdev, cdb_size, VIRTIO_SCSI_CDB_SIZE);
virtscsi_config_set(vdev, sense_size, VIRTIO_SCSI_SENSE_SIZE);
diff --git a/drivers/virtio/Makefile b/drivers/virtio/Makefile
index eefcfe90d6b8..769e95ee5be4 100644
--- a/drivers/virtio/Makefile
+++ b/drivers/virtio/Makefile
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: GPL-2.0
-obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o
+obj-$(CONFIG_VIRTIO) += virtio.o virtio_ring.o virtio_sqcq_poll.o
obj-$(CONFIG_VIRTIO_ANCHOR) += virtio_anchor.o
obj-$(CONFIG_VIRTIO_PCI_LIB) += virtio_pci_modern_dev.o
obj-$(CONFIG_VIRTIO_PCI_LIB_LEGACY) += virtio_pci_legacy_dev.o
diff --git a/drivers/virtio/virtio.c b/drivers/virtio/virtio.c
index 5bdc6b82b30b..d440e920f8e6 100644
--- a/drivers/virtio/virtio.c
+++ b/drivers/virtio/virtio.c
@@ -221,6 +221,16 @@ static int virtio_features_ok(struct virtio_device *dev)
}
}
+ /* SQ/CQ poll only supports split ring; reject packed ring
+ * to avoid split-specific field access in virtqueue_notify().
+ */
+ if (virtio_has_feature(dev, VIRTIO_F_SQCQ_POLL) &&
+ virtio_has_feature(dev, VIRTIO_F_RING_PACKED)) {
+ dev_warn(&dev->dev,
+ "VIRTIO_F_SQCQ_POLL is incompatible with
VIRTIO_F_RING_PACKED\n");
+ return -ENODEV;
+ }
+
if (!virtio_has_feature(dev, VIRTIO_F_VERSION_1))
return 0;
diff --git a/drivers/virtio/virtio_pci_common.c
b/drivers/virtio/virtio_pci_common.c
index da97b6a988de..cd22651c73e2 100644
--- a/drivers/virtio/virtio_pci_common.c
+++ b/drivers/virtio/virtio_pci_common.c
@@ -268,6 +268,9 @@ void vp_del_vqs(struct virtio_device *vdev)
struct virtqueue *vq, *n;
int i;
+ /* Stop SQ/CQ poll thread before deleting virtqueues */
+ virtio_stop_sqcq_poll(vdev);
+
list_for_each_entry_safe(vq, n, &vdev->vqs, list) {
info = vp_is_avq(vdev, vq->index) ? vp_dev->admin_vq.info :
vp_dev->vqs[vq->index];
diff --git a/drivers/virtio/virtio_pci_modern.c
b/drivers/virtio/virtio_pci_modern.c
index 6d8ae2a6a8ca..7a36ec4584a6 100644
--- a/drivers/virtio/virtio_pci_modern.c
+++ b/drivers/virtio/virtio_pci_modern.c
@@ -18,6 +18,7 @@
#include <linux/virtio_pci_admin.h>
#define VIRTIO_PCI_NO_LEGACY
#define VIRTIO_RING_NO_LEGACY
+
#include "virtio_pci_common.h"
#define VIRTIO_AVQ_SGS_MAX 4
@@ -378,6 +379,9 @@ static void vp_transport_features(struct virtio_device
*vdev, u64 features)
if (features & BIT_ULL(VIRTIO_F_ADMIN_VQ))
__virtio_set_bit(vdev, VIRTIO_F_ADMIN_VQ);
+
+ if (features & BIT_ULL(VIRTIO_F_SQCQ_POLL))
+ __virtio_set_bit(vdev, VIRTIO_F_SQCQ_POLL);
}
static int __vp_check_common_size_one_feature(struct virtio_device *vdev, u32
fbit,
@@ -413,6 +417,9 @@ static int vp_check_common_size(struct virtio_device *vdev)
if (vp_check_common_size_one_feature(vdev, VIRTIO_F_ADMIN_VQ,
admin_queue_num))
return -EINVAL;
+ if (vp_check_common_size_one_feature(vdev, VIRTIO_F_SQCQ_POLL,
cqe_ring_hi))
+ return -EINVAL;
+
return 0;
}
@@ -578,6 +585,11 @@ static int vp_active_vq(struct virtqueue *vq, u16 msix_vec)
virtqueue_get_avail_addr(vq),
virtqueue_get_used_addr(vq));
+ if (virtqueue_sqcq_poll_active(vq))
+ vp_modern_queue_address_sqcq(mdev, index,
+ virtqueue_get_sq_dma_addr(vq),
+ virtqueue_get_cq_dma_addr(vq));
+
if (msix_vec != VIRTIO_MSI_NO_VECTOR) {
msix_vec = vp_modern_queue_vector(mdev, index, msix_vec);
if (msix_vec == VIRTIO_MSI_NO_VECTOR)
@@ -753,6 +765,27 @@ static int vp_modern_find_vqs(struct virtio_device *vdev,
unsigned int nvqs,
if (rc)
return rc;
+ /* Start SQ/CQ poll thread if feature negotiated */
+ if (virtio_has_feature(vdev, VIRTIO_F_SQCQ_POLL)) {
+ dev_info(&vdev->dev, "VIRTIO_F_SQCQ_POLL negotiated, starting
poll thread\n");
+ rc = virtio_start_sqcq_poll(vdev, 1000, 1000);
+ if (rc)
+ goto err_del_vqs;
+
+ list_for_each_entry(vq, &vdev->vqs, list) {
+ if (!virtqueue_sqcq_poll_active(vq))
+ continue;
+ rc = virtio_sqcq_poll_register_vq(vdev, vq);
+ if (rc) {
+ dev_err(&vdev->dev,
+ "failed to register vq %d (%s) with
poll thread: %d\n",
+ vq->index,
+ vq->name ? vq->name : "?", rc);
+ goto err_stop_poll;
+ }
+ }
+ }
+
/* Select and activate all queues. Has to be done last: once we do
* this, there's no way to go back except reset.
*/
@@ -760,6 +793,12 @@ static int vp_modern_find_vqs(struct virtio_device *vdev,
unsigned int nvqs,
vp_modern_set_queue_enable(&vp_dev->mdev, vq->index, true);
return 0;
+
+err_stop_poll:
+ virtio_stop_sqcq_poll(vdev);
+err_del_vqs:
+ vp_del_vqs(vdev);
+ return rc;
}
static void del_vq(struct virtio_pci_vq_info *info)
diff --git a/drivers/virtio/virtio_pci_modern_dev.c
b/drivers/virtio/virtio_pci_modern_dev.c
index 413a8c353463..642b9a58751c 100644
--- a/drivers/virtio/virtio_pci_modern_dev.c
+++ b/drivers/virtio/virtio_pci_modern_dev.c
@@ -211,6 +211,14 @@ static inline void check_offsets(void)
offsetof(struct virtio_pci_modern_common_cfg,
admin_queue_index));
BUILD_BUG_ON(VIRTIO_PCI_COMMON_ADM_Q_NUM !=
offsetof(struct virtio_pci_modern_common_cfg,
admin_queue_num));
+ BUILD_BUG_ON(VIRTIO_PCI_COMMON_SQE_LO !=
+ offsetof(struct virtio_pci_modern_common_cfg,
sqe_ring_lo));
+ BUILD_BUG_ON(VIRTIO_PCI_COMMON_SQE_HI !=
+ offsetof(struct virtio_pci_modern_common_cfg,
sqe_ring_hi));
+ BUILD_BUG_ON(VIRTIO_PCI_COMMON_CQE_LO !=
+ offsetof(struct virtio_pci_modern_common_cfg,
cqe_ring_lo));
+ BUILD_BUG_ON(VIRTIO_PCI_COMMON_CQE_HI !=
+ offsetof(struct virtio_pci_modern_common_cfg,
cqe_ring_hi));
}
/*
@@ -300,7 +308,7 @@ int vp_modern_probe(struct virtio_pci_modern_device *mdev)
mdev->common = vp_modern_map_capability(mdev, common,
sizeof(struct virtio_pci_common_cfg), 4, 0,
offsetofend(struct virtio_pci_modern_common_cfg,
- admin_queue_num),
+ cqe_ring_hi),
&mdev->common_len, NULL);
if (!mdev->common)
goto err_map_common;
@@ -607,6 +615,30 @@ void vp_modern_queue_address(struct
virtio_pci_modern_device *mdev,
}
EXPORT_SYMBOL_GPL(vp_modern_queue_address);
+/*
+ * vp_modern_queue_address_sqcq - set the SQ/CQ doorbell DMA addresses
+ * @mdev: the modern virtio-pci device
+ * @index: the queue index
+ * @sq_addr: DMA address of the SQ doorbell
+ * @cq_addr: DMA address of the CQ doorbell
+ */
+void vp_modern_queue_address_sqcq(struct virtio_pci_modern_device *mdev,
+ u16 index, u64 sq_addr, u64 cq_addr)
+{
+ struct virtio_pci_modern_common_cfg __iomem *cfg;
+
+ cfg = (struct virtio_pci_modern_common_cfg __iomem *)mdev->common;
+
+ vp_iowrite16(index, &cfg->cfg.queue_select);
+
+ vp_iowrite64_twopart(sq_addr, &cfg->sqe_ring_lo,
+ &cfg->sqe_ring_hi);
+
+ vp_iowrite64_twopart(cq_addr, &cfg->cqe_ring_lo,
+ &cfg->cqe_ring_hi);
+}
+EXPORT_SYMBOL_GPL(vp_modern_queue_address_sqcq);
+
/*
* vp_modern_set_queue_enable - enable a virtqueue
* @mdev: the modern virtio-pci device
diff --git a/drivers/virtio/virtio_ring.c b/drivers/virtio/virtio_ring.c
index 335692d41617..a3aaf123ee74 100644
--- a/drivers/virtio/virtio_ring.c
+++ b/drivers/virtio/virtio_ring.c
@@ -270,10 +270,25 @@ struct vring_virtqueue {
bool last_add_time_valid;
ktime_t last_add_time;
#endif
+
+ /* SQ/CQ polling state (VIRTIO_F_SQCQ_POLL) */
+ bool sqcq_poll; /* Is SQ/CQ polling active? */
+ struct vring_sq *sq; /* DMA-coherent SQ doorbell */
+ struct vring_cq *cq; /* DMA-coherent CQ doorbell */
+ dma_addr_t sq_dma_addr; /* DMA address of SQ */
+ dma_addr_t cq_dma_addr; /* DMA address of CQ */
+ atomic_t sq_mmio_count; /* MMIO kicks (sq_need_wakeup=1) */
+ u64 last_cq_detect_ns; /* ktime_get_ns() when CQ work first
detected */
+ u64 sq_last_notify_ns; /* ktime of last sq->idx store */
+ u64 sq_ema_interval_ns; /* EMA of sq->idx update interval (ns) */
};
static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num);
static void vring_free(struct virtqueue *_vq);
+static int virtio_alloc_sqcq(struct virtio_device *vdev,
+ struct vring_virtqueue *vq);
+static void virtio_free_sqcq(struct virtio_device *vdev,
+ struct vring_virtqueue *vq);
/*
* Helpers.
@@ -1334,6 +1349,10 @@ static struct virtqueue
*__vring_new_virtqueue_split(unsigned int index,
vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ?
VQ_LAYOUT_SPLIT_IN_ORDER : VQ_LAYOUT_SPLIT;
+ vq->sqcq_poll = false;
+ vq->sq = NULL;
+ vq->cq = NULL;
+
if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
vq->weak_barriers = false;
@@ -1385,6 +1404,17 @@ static struct virtqueue *vring_create_virtqueue_split(
to_vvq(vq)->we_own_ring = true;
+ if (virtio_has_feature(vdev, VIRTIO_F_SQCQ_POLL)) {
+ if (virtio_alloc_sqcq(vdev, to_vvq(vq))) {
+ /* vq is on vdev->vqs list; use vring_del_virtqueue()
+ * to remove and free. vring_free_split() would
+ * double-free via later vp_del_vqs().
+ */
+ vring_del_virtqueue(vq);
+ return NULL;
+ }
+ }
+
return vq;
}
@@ -2554,6 +2584,10 @@ static struct virtqueue
*__vring_new_virtqueue_packed(unsigned int index,
vq->layout = virtio_has_feature(vdev, VIRTIO_F_IN_ORDER) ?
VQ_LAYOUT_PACKED_IN_ORDER : VQ_LAYOUT_PACKED;
+ vq->sqcq_poll = false;
+ vq->sq = NULL;
+ vq->cq = NULL;
+
if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
vq->weak_barriers = false;
@@ -3013,6 +3047,9 @@ bool virtqueue_kick_prepare(struct virtqueue *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
+ if (vq->sqcq_poll)
+ return true;
+
return VIRTQUEUE_CALL(vq, kick_prepare);
}
EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
@@ -3032,6 +3069,45 @@ bool virtqueue_notify(struct virtqueue *_vq)
if (unlikely(vq->broken))
return false;
+ /* SQ/CQ polling mode: update SQ idx instead of MMIO kick. */
+ if (vq->sqcq_poll) {
+ u16 new_sq_idx = vq->split.avail_idx_shadow;
+ u8 need_wakeup;
+
+ /* Release: make descriptor writes visible before sq->idx */
+ smp_store_release(&vq->sq->idx, new_sq_idx);
+
+ /* Track submission interval (matches host's interval metric) */
+ {
+ u64 _now = ktime_get_ns();
+
+ if (vq->sq_last_notify_ns) {
+ u64 _int = _now - vq->sq_last_notify_ns;
+
+ if (vq->sq_ema_interval_ns == 0)
+ vq->sq_ema_interval_ns = _int;
+ else
+ vq->sq_ema_interval_ns =
+ vq->sq_ema_interval_ns -
+ vq->sq_ema_interval_ns / 8 +
+ _int / 8;
+ }
+ vq->sq_last_notify_ns = _now;
+ }
+
+ /* Acquire: order sq->idx store before reading sq_need_wakeup.
+ * Prevents stale read on ARM/RISC-V causing missed kick
+ * when host has set sq_need_wakeup=1 and gone to sleep.
+ */
+ need_wakeup = smp_load_acquire(&vq->sq->sq_need_wakeup);
+
+ /* Device poll thread is sleeping -- need MMIO kick to wake it
*/
+ if (!need_wakeup)
+ return true;
+ atomic_inc(&vq->sq_mmio_count);
+ /* Fall through to MMIO notify */
+ }
+
/* Prod other side to tell it about changes. */
if (!vq->notify(_vq)) {
vq->broken = true;
@@ -3230,6 +3306,12 @@ irqreturn_t vring_interrupt(int irq, void *_vq)
{
struct vring_virtqueue *vq = to_vvq(_vq);
+ /* SQ/CQ poll mode: wake the poll thread, let it handle completions */
+ if (vq->sqcq_poll) {
+ virtio_sqcq_poll_wake_thread(vq->vq.vdev);
+ return IRQ_HANDLED;
+ }
+
if (!more_used(vq)) {
pr_debug("virtqueue interrupt with no work for %p\n", vq);
return IRQ_NONE;
@@ -3250,6 +3332,7 @@ irqreturn_t vring_interrupt(int irq, void *_vq)
data_race(vq->event_triggered = true);
pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
+
if (vq->vq.callback)
vq->vq.callback(&vq->vq);
@@ -3468,6 +3551,8 @@ static void vring_free(struct virtqueue *_vq)
kfree(vq->split.desc_state);
kfree(vq->split.desc_extra);
}
+ if (vq->sqcq_poll)
+ virtio_free_sqcq(vq->vq.vdev, vq);
}
void vring_del_virtqueue(struct virtqueue *_vq)
@@ -3524,6 +3609,8 @@ void vring_transport_features(struct virtio_device *vdev)
break;
case VIRTIO_F_IN_ORDER:
break;
+ case VIRTIO_F_SQCQ_POLL:
+ break;
default:
/* We don't understand this bit. */
__virtio_clear_bit(vdev, i);
@@ -3663,6 +3750,181 @@ dma_addr_t virtqueue_get_used_addr(const struct
virtqueue *_vq)
}
EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
+bool virtqueue_sqcq_poll_active(const struct virtqueue *_vq)
+{
+ const struct vring_virtqueue *vq = to_vvq(_vq);
+
+ return vq->sqcq_poll;
+}
+EXPORT_SYMBOL_GPL(virtqueue_sqcq_poll_active);
+
+/**
+ * virtqueue_disable_sqcq_poll - disable SQ/CQ poll mode for a virtqueue.
+ * @_vq: the virtqueue
+ *
+ * Clears sqcq_poll flag; restores MMIO kick + interrupt for this vq.
+ * Used for control/event queues that must not use the poll thread.
+ */
+void virtqueue_disable_sqcq_poll(struct virtqueue *_vq)
+{
+ struct vring_virtqueue *vq = to_vvq(_vq);
+
+ vq->sqcq_poll = false;
+}
+EXPORT_SYMBOL_GPL(virtqueue_disable_sqcq_poll);
+
+dma_addr_t virtqueue_get_sq_dma_addr(const struct virtqueue *_vq)
+{
+ const struct vring_virtqueue *vq = to_vvq(_vq);
+
+ return vq->sq_dma_addr;
+}
+EXPORT_SYMBOL_GPL(virtqueue_get_sq_dma_addr);
+
+dma_addr_t virtqueue_get_cq_dma_addr(const struct virtqueue *_vq)
+{
+ const struct vring_virtqueue *vq = to_vvq(_vq);
+
+ return vq->cq_dma_addr;
+}
+EXPORT_SYMBOL_GPL(virtqueue_get_cq_dma_addr);
+
+/* Completion detection and sleep/wake accessors for the poll thread. */
+
+bool virtqueue_cq_poll_has_work(struct virtqueue *_vq)
+{
+ struct vring_virtqueue *vq = to_vvq(_vq);
+
+ if (!vq->cq)
+ return false;
+
+ if (more_used(vq)) {
+ WRITE_ONCE(vq->last_cq_detect_ns, ktime_get_ns());
+ return true;
+ }
+
+ return false;
+}
+EXPORT_SYMBOL_GPL(virtqueue_cq_poll_has_work);
+
+unsigned int virtqueue_get_sq_notify_stats(struct virtqueue *_vq, u64
*interval)
+{
+ struct vring_virtqueue *vq = to_vvq(_vq);
+ unsigned int mmio;
+
+ mmio = atomic_xchg(&vq->sq_mmio_count, 0);
+ if (interval)
+ *interval = vq->sq_ema_interval_ns;
+ return mmio;
+}
+EXPORT_SYMBOL_GPL(virtqueue_get_sq_notify_stats);
+
+void virtqueue_cq_set_need_wakeup(struct virtqueue *_vq)
+{
+ struct vring_virtqueue *vq = to_vvq(_vq);
+
+ if (vq->cq)
+ /* Release: order prior used-ring reads before wakeup flag */
+ smp_store_release(&vq->cq->cq_need_wakeup, 1);
+}
+EXPORT_SYMBOL_GPL(virtqueue_cq_set_need_wakeup);
+
+void virtqueue_cq_clear_need_wakeup(struct virtqueue *_vq)
+{
+ struct vring_virtqueue *vq = to_vvq(_vq);
+
+ if (vq->cq)
+ WRITE_ONCE(vq->cq->cq_need_wakeup, 0);
+}
+EXPORT_SYMBOL_GPL(virtqueue_cq_clear_need_wakeup);
+
+bool virtqueue_cq_recheck(struct virtqueue *_vq)
+{
+ struct vring_virtqueue *vq = to_vvq(_vq);
+
+ if (!vq->cq)
+ return false;
+
+ /* Full barrier: order cq_need_wakeup store before used ring read */
+ smp_mb();
+ return more_used(vq);
+}
+EXPORT_SYMBOL_GPL(virtqueue_cq_recheck);
+
+u64 virtqueue_get_cq_detect_ns(const struct virtqueue *_vq)
+{
+ const struct vring_virtqueue *vq = to_vvq(_vq);
+
+ return READ_ONCE(vq->last_cq_detect_ns);
+}
+EXPORT_SYMBOL_GPL(virtqueue_get_cq_detect_ns);
+
+/*
+ * virtio_alloc_sqcq - allocate SQ/CQ doorbell structures for polling mode
+ * @vdev: the virtio device
+ * @vq: the vring_virtqueue to allocate for
+ *
+ * Called when VIRTIO_F_SQCQ_POLL is negotiated. Returns 0 on success.
+ */
+static int virtio_alloc_sqcq(struct virtio_device *vdev,
+ struct vring_virtqueue *vq)
+{
+ struct device *dma_dev = vdev->dev.parent;
+
+ /* Control and event queues use traditional interrupt-driven I/O. */
+ if (vq->vq.name &&
+ (strcmp(vq->vq.name, "control") == 0 ||
+ strcmp(vq->vq.name, "event") == 0))
+ return 0;
+ size_t total = sizeof(struct vring_sq) + sizeof(struct vring_cq);
+ void *buf;
+
+ buf = dma_alloc_coherent(dma_dev, total, &vq->sq_dma_addr, GFP_KERNEL);
+ if (!buf)
+ return -ENOMEM;
+
+ memset(buf, 0, total);
+
+ vq->sq = buf;
+ vq->cq = buf + sizeof(struct vring_sq);
+ vq->cq_dma_addr = vq->sq_dma_addr + sizeof(struct vring_sq);
+ vq->sqcq_poll = true;
+ atomic_set(&vq->sq_mmio_count, 0);
+ vq->last_cq_detect_ns = 0;
+ vq->sq_last_notify_ns = 0;
+ vq->sq_ema_interval_ns = 0;
+
+ /* Diagnostic: dump SQ/CQ addresses for debug */
+ dev_dbg(&vdev->dev,
+ "vq %s: SQ/CQ doorbells kvirt=%p sq_dma=0x%llx cq_dma=0x%llx\n",
+ vq->vq.name ? vq->vq.name : "?",
+ buf,
+ (unsigned long long)vq->sq_dma_addr,
+ (unsigned long long)vq->cq_dma_addr);
+
+ return 0;
+}
+
+/*
+ * virtio_free_sqcq - free SQ/CQ doorbell structures
+ * @vdev: the virtio device
+ * @vq: the vring_virtqueue to free for
+ */
+static void virtio_free_sqcq(struct virtio_device *vdev,
+ struct vring_virtqueue *vq)
+{
+ struct device *dma_dev = vdev->dev.parent;
+ size_t total = sizeof(struct vring_sq) + sizeof(struct vring_cq);
+
+ if (vq->sq) {
+ dma_free_coherent(dma_dev, total,
+ vq->sq, vq->sq_dma_addr);
+ vq->sq = NULL;
+ vq->cq = NULL;
+ }
+ vq->sqcq_poll = false;
+}
+
/* Only available for split ring */
const struct vring *virtqueue_get_vring(const struct virtqueue *vq)
{
diff --git a/drivers/virtio/virtio_sqcq_poll.c
b/drivers/virtio/virtio_sqcq_poll.c
new file mode 100644
index 000000000000..411493218e93
--- /dev/null
+++ b/drivers/virtio/virtio_sqcq_poll.c
@@ -0,0 +1,338 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/*
+ * virtio SQ/CQ polling thread.
+ *
+ * Replaces interrupt-based completion notification with a dedicated
+ * kernel thread that checks the used ring for completions. Uses the
+ * io_uring sqpoll pattern: busy-wait while active, then set
+ * NEED_WAKEUP and sleep when idle for a configurable timeout.
+ *
+ * Copyright (C) 2026
+ */
+
+#include <linux/kernel.h>
+#include <linux/device.h>
+#include <linux/sched.h>
+#include <linux/completion.h>
+#include <linux/mutex.h>
+#include <linux/wait.h>
+#include <linux/sched/clock.h>
+#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/kthread.h>
+#include <linux/virtio.h>
+#include <linux/virtio_config.h>
+#include <linux/virtio_ring.h>
+#include <linux/interrupt.h>
+
+/**
+ * struct virtio_sqcq_poll_data - per-device poll thread state.
+ *
+ * Follows the Host vhost_sqcq_poll_thread pattern.
+ */
+struct virtio_sqcq_poll_data {
+ struct mutex lock;
+
+ /* List of virtqueues served by this poll thread */
+ struct list_head vq_list;
+
+ struct task_struct *thread;
+ struct wait_queue_head wait;
+
+ unsigned int idle_timeout_us; /* Sleep timeout (us) */
+ unsigned int busy_spin_us; /* Busy-spin budget (us) */
+
+ unsigned long state;
+#define VIRTIO_SQ_THREAD_SHOULD_STOP 0
+
+ struct completion exited;
+ struct device *dev; /* for dev_info() logging */
+
+ /* Poll thread diagnostic statistics */
+ unsigned long stat_sleep_count;
+};
+
+/**
+ * struct virtio_sqcq_vq_node - links a virtqueue to its poll_data.
+ */
+struct virtio_sqcq_vq_node {
+ struct virtqueue *vq;
+ struct list_head node;
+};
+
+static void virtio_sqcq_poll_wake(struct virtio_sqcq_poll_data *sqd)
+{
+ wake_up(&sqd->wait);
+}
+
+/**
+ * virtio_sqcq_poll_wake_thread - wake the CQ poll thread from IRQ context.
+ * @vdev: the virtio device
+ *
+ * Called by vring_interrupt() when SQ/CQ poll mode is active. Wakes the
+ * poll thread so it can check the used ring and invoke completion
+ * callbacks in its own context, instead of handling completions directly
+ * from the hard-IRQ handler.
+ */
+void virtio_sqcq_poll_wake_thread(struct virtio_device *vdev)
+{
+ struct virtio_sqcq_poll_data *sqd = vdev->sqcq_poll_data;
+
+ if (sqd)
+ virtio_sqcq_poll_wake(sqd);
+}
+EXPORT_SYMBOL_GPL(virtio_sqcq_poll_wake_thread);
+
+unsigned long virtio_sqcq_poll_get_sleep_count(struct virtio_device *vdev)
+{
+ struct virtio_sqcq_poll_data *sqd = vdev->sqcq_poll_data;
+
+ if (!sqd)
+ return 0;
+
+ return xchg(&sqd->stat_sleep_count, 0);
+}
+EXPORT_SYMBOL_GPL(virtio_sqcq_poll_get_sleep_count);
+
+static int virtio_sqcq_poll_thread(void *data)
+{
+ struct virtio_sqcq_poll_data *sqd = data;
+ struct virtio_sqcq_vq_node *node;
+ ktime_t last_io_ts = ktime_get();
+ u64 spin_budget_ns = (u64)sqd->busy_spin_us * NSEC_PER_USEC;
+ int vq_count = 0;
+
+ /* Count registered vqs for the startup message */
+ mutex_lock(&sqd->lock);
+ list_for_each_entry(node, &sqd->vq_list, node)
+ vq_count++;
+ mutex_unlock(&sqd->lock);
+
+ dev_info(sqd->dev, "poll thread running, %d vqs, spin=%uus,
idle=%uus\n",
+ vq_count, sqd->busy_spin_us, sqd->idle_timeout_us);
+
+ while (!kthread_should_stop()) {
+ bool did_work = false;
+
+ if (test_bit(VIRTIO_SQ_THREAD_SHOULD_STOP, &sqd->state)) {
+ dev_info(sqd->dev, "poll thread stopping\n");
+ break;
+ }
+
+ if (signal_pending(current)) {
+ flush_signals(current);
+ continue;
+ }
+
+ mutex_lock(&sqd->lock);
+
+ /* Process completions on all registered virtqueues */
+ list_for_each_entry(node, &sqd->vq_list, node) {
+ while (virtqueue_cq_poll_has_work(node->vq)) {
+ if (node->vq->callback)
+ node->vq->callback(node->vq);
+ if (local_softirq_pending())
+ do_softirq();
+ did_work = true;
+ }
+ }
+
+ if (did_work) {
+ mutex_unlock(&sqd->lock);
+ last_io_ts = ktime_get();
+ continue;
+ }
+
+ /* No work — spin within the time budget from last_io_ts.
+ * need_resched() + cpu_relax() for cooperative yielding,
+ * same pattern as Host vhost_sqcq_poll_thread.
+ */
+ if (spin_budget_ns > 0 &&
+ ktime_to_ns(ktime_sub(ktime_get(), last_io_ts)) <
+ (s64)spin_budget_ns) {
+ mutex_unlock(&sqd->lock);
+ if (need_resched())
+ cond_resched();
+ cpu_relax();
+ continue;
+ }
+
+ /* Spin budget expired — set cq_need_wakeup on all VQs
+ * and sleep. Double-check for work before sleeping to
+ * close the lost-wakeup race.
+ */
+ {
+ bool found_work = false;
+ DEFINE_WAIT(wait);
+
+ list_for_each_entry(node, &sqd->vq_list, node)
+ virtqueue_cq_set_need_wakeup(node->vq);
+
+ prepare_to_wait(&sqd->wait, &wait,
+ TASK_INTERRUPTIBLE);
+
+ list_for_each_entry(node, &sqd->vq_list, node) {
+ if (virtqueue_cq_recheck(node->vq)) {
+ found_work = true;
+ break;
+ }
+ }
+
+ if (!found_work) {
+ mutex_unlock(&sqd->lock);
+ schedule_timeout(
+ usecs_to_jiffies(sqd->idle_timeout_us));
+ sqd->stat_sleep_count++;
+ mutex_lock(&sqd->lock);
+ }
+
+ finish_wait(&sqd->wait, &wait);
+
+ list_for_each_entry(node, &sqd->vq_list, node)
+ virtqueue_cq_clear_need_wakeup(node->vq);
+ }
+
+ mutex_unlock(&sqd->lock);
+ }
+
+ complete(&sqd->exited);
+ return 0;
+}
+
+/**
+ * virtio_start_sqcq_poll - start the SQ/CQ poll thread for a device.
+ * @vdev: the virtio device
+ * @idle_timeout_us: idle timeout in microseconds before sleeping
+ * @busy_spin_us: busy-spin duration in microseconds before yielding
+ *
+ * Returns 0 on success, negative error on failure.
+ */
+int virtio_start_sqcq_poll(struct virtio_device *vdev,
+ unsigned int idle_timeout_us,
+ unsigned int busy_spin_us)
+{
+ struct virtio_sqcq_poll_data *sqd;
+
+ sqd = kzalloc(sizeof(*sqd), GFP_KERNEL);
+ if (!sqd)
+ return -ENOMEM;
+
+ mutex_init(&sqd->lock);
+ init_waitqueue_head(&sqd->wait);
+ INIT_LIST_HEAD(&sqd->vq_list);
+ init_completion(&sqd->exited);
+ sqd->idle_timeout_us = idle_timeout_us ? idle_timeout_us : 1000;
+ sqd->busy_spin_us = busy_spin_us;
+ sqd->dev = &vdev->dev;
+
+ sqd->thread = kthread_create(virtio_sqcq_poll_thread, sqd,
+ "virtio-sqcq/%s", dev_name(&vdev->dev));
+ if (IS_ERR(sqd->thread)) {
+ kfree(sqd);
+ return PTR_ERR(sqd->thread);
+ }
+
+ vdev->sqcq_poll_data = sqd;
+ wake_up_process(sqd->thread);
+
+ dev_info(sqd->dev, "SQ/CQ poll thread started, idle_timeout=%uus,
spin_us=%u\n",
+ sqd->idle_timeout_us, sqd->busy_spin_us);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(virtio_start_sqcq_poll);
+
+/**
+ * virtio_stop_sqcq_poll - stop the SQ/CQ poll thread for a device.
+ * @vdev: the virtio device
+ */
+void virtio_stop_sqcq_poll(struct virtio_device *vdev)
+{
+ struct virtio_sqcq_poll_data *sqd = vdev->sqcq_poll_data;
+ struct virtio_sqcq_vq_node *node, *tmp;
+
+ if (!sqd)
+ return;
+
+ set_bit(VIRTIO_SQ_THREAD_SHOULD_STOP, &sqd->state);
+
+ dev_info(sqd->dev, "SQ/CQ poll thread stopping\n");
+ virtio_sqcq_poll_wake(sqd);
+ wait_for_completion(&sqd->exited);
+
+ /* Reap kthread task_struct to avoid zombie thread leak.
+ * Thread already exited via complete(); safe to call on
+ * already-exited kthread.
+ */
+ kthread_stop(sqd->thread);
+
+ /* Free all vq nodes */
+ list_for_each_entry_safe(node, tmp, &sqd->vq_list, node) {
+ list_del(&node->node);
+ kfree(node);
+ }
+
+ vdev->sqcq_poll_data = NULL;
+ kfree(sqd);
+}
+EXPORT_SYMBOL_GPL(virtio_stop_sqcq_poll);
+
+/**
+ * virtio_sqcq_poll_register_vq - register a virtqueue with the poll thread.
+ * @vdev: the virtio device
+ * @vq: the virtqueue to register
+ *
+ * The virtqueue must have VIRTIO_F_SQCQ_POLL active.
+ * Returns 0 on success, negative error on failure.
+ */
+int virtio_sqcq_poll_register_vq(struct virtio_device *vdev,
+ struct virtqueue *vq)
+{
+ struct virtio_sqcq_poll_data *sqd = vdev->sqcq_poll_data;
+ struct virtio_sqcq_vq_node *node;
+
+ if (!sqd || !virtqueue_sqcq_poll_active(vq))
+ return -EINVAL;
+
+ node = kmalloc(sizeof(*node), GFP_KERNEL);
+ if (!node)
+ return -ENOMEM;
+
+ node->vq = vq;
+
+ mutex_lock(&sqd->lock);
+ list_add_tail(&node->node, &sqd->vq_list);
+ mutex_unlock(&sqd->lock);
+
+ dev_info(sqd->dev, "vq index=%d (%s) registered with poll thread\n",
+ vq->index, vq->name ? vq->name : "?");
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(virtio_sqcq_poll_register_vq);
+
+/**
+ * virtio_sqcq_poll_unregister_vq - unregister a virtqueue from the poll
thread.
+ * @vdev: the virtio device
+ * @vq: the virtqueue to unregister
+ */
+void virtio_sqcq_poll_unregister_vq(struct virtio_device *vdev,
+ struct virtqueue *vq)
+{
+ struct virtio_sqcq_poll_data *sqd = vdev->sqcq_poll_data;
+ struct virtio_sqcq_vq_node *node, *tmp;
+
+ if (!sqd)
+ return;
+
+ mutex_lock(&sqd->lock);
+ list_for_each_entry_safe(node, tmp, &sqd->vq_list, node) {
+ if (node->vq == vq) {
+ list_del(&node->node);
+ kfree(node);
+ break;
+ }
+ }
+ mutex_unlock(&sqd->lock);
+}
+EXPORT_SYMBOL_GPL(virtio_sqcq_poll_unregister_vq);
diff --git a/include/linux/virtio.h b/include/linux/virtio.h
index 3bbc4cb6a672..8300e0946719 100644
--- a/include/linux/virtio.h
+++ b/include/linux/virtio.h
@@ -125,6 +125,28 @@ dma_addr_t virtqueue_get_desc_addr(const struct virtqueue
*vq);
dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *vq);
dma_addr_t virtqueue_get_used_addr(const struct virtqueue *vq);
+/* SQ/CQ polling (VIRTIO_F_SQCQ_POLL) */
+bool virtqueue_sqcq_poll_active(const struct virtqueue *vq);
+void virtqueue_disable_sqcq_poll(struct virtqueue *vq);
+dma_addr_t virtqueue_get_sq_dma_addr(const struct virtqueue *vq);
+dma_addr_t virtqueue_get_cq_dma_addr(const struct virtqueue *vq);
+u64 virtqueue_get_cq_detect_ns(const struct virtqueue *vq);
+int virtio_start_sqcq_poll(struct virtio_device *vdev,
+ unsigned int idle_timeout_us,
+ unsigned int busy_spin_us);
+void virtio_stop_sqcq_poll(struct virtio_device *vdev);
+void virtio_sqcq_poll_wake_thread(struct virtio_device *vdev);
+int virtio_sqcq_poll_register_vq(struct virtio_device *vdev,
+ struct virtqueue *vq);
+void virtio_sqcq_poll_unregister_vq(struct virtio_device *vdev,
+ struct virtqueue *vq);
+bool virtqueue_cq_poll_has_work(struct virtqueue *vq);
+unsigned int virtqueue_get_sq_notify_stats(struct virtqueue *vq, u64
*interval);
+unsigned long virtio_sqcq_poll_get_sleep_count(struct virtio_device *vdev);
+void virtqueue_cq_set_need_wakeup(struct virtqueue *vq);
+void virtqueue_cq_clear_need_wakeup(struct virtqueue *vq);
+bool virtqueue_cq_recheck(struct virtqueue *vq);
+
int virtqueue_resize(struct virtqueue *vq, u32 num,
void (*recycle)(struct virtqueue *vq, void *buf),
void (*recycle_done)(struct virtqueue *vq));
@@ -165,6 +187,8 @@ struct virtio_admin_cmd {
* @debugfs_dir: debugfs directory entry.
* @debugfs_filter_features: features to be filtered set by debugfs.
*/
+struct virtio_sqcq_poll_data;
+
struct virtio_device {
int index;
bool failed;
@@ -182,6 +206,7 @@ struct virtio_device {
VIRTIO_DECLARE_FEATURES(features);
void *priv;
union virtio_map vmap;
+ struct virtio_sqcq_poll_data *sqcq_poll_data; /* SQ/CQ poll thread
state */
#ifdef CONFIG_VIRTIO_DEBUG
struct dentry *debugfs_dir;
u64 debugfs_filter_features[VIRTIO_FEATURES_U64S];
diff --git a/include/linux/virtio_pci_modern.h
b/include/linux/virtio_pci_modern.h
index 9a3f2fc53bd6..36810f9d23f3 100644
--- a/include/linux/virtio_pci_modern.h
+++ b/include/linux/virtio_pci_modern.h
@@ -145,6 +145,8 @@ u16 vp_modern_config_vector(struct virtio_pci_modern_device
*mdev,
void vp_modern_queue_address(struct virtio_pci_modern_device *mdev,
u16 index, u64 desc_addr, u64 driver_addr,
u64 device_addr);
+void vp_modern_queue_address_sqcq(struct virtio_pci_modern_device *mdev,
+ u16 index, u64 sq_addr, u64 cq_addr);
void vp_modern_set_queue_enable(struct virtio_pci_modern_device *mdev,
u16 idx, bool enable);
bool vp_modern_get_queue_enable(struct virtio_pci_modern_device *mdev,
--
2.34.1
