From: Yufeng Wang <[email protected]>
Implement SQ/CQ doorbell polling in the vhost kernel backend:
- vhost.c: per-VQ poll thread (kthread) that polls sq->idx for
submissions and processes completions via poll_complete callback;
need_resched() cooperative spin; round-robin CPU binding with
modulo to prevent overflow; sqcq_poll enabled after poll_task is
valid (NULL deref fix); get_user fault retry (3 attempts, 10ms
backoff); EMA overflow-safe formula
- vhost.h: vhost_virtqueue extensions (sq/cq pointers, poll thread
state, EMA fields)
- scsi.c: vhost_scsi_poll_complete() writes cq->idx and signals
Guest; SET_ENDPOINT error path stops started poll threads (fallback);
10s window diagnostic output
- vhost.h UAPI: VHOST_SET_VRING_SQCQ_ADDR ioctl
- vhost_types.h UAPI: struct vhost_vring_sqcq_addr
Signed-off-by: Yufeng Wang <[email protected]>
---
Documentation/compare-sqcq-results.sh | 239 +++++++
Documentation/run-sqcq-compare.sh | 182 ++++++
Documentation/virtio-sqcq-poll-spec.rst | 617 ++++++++++++++++++
.../virtio-sqcq-poll-virtio-spec.rst | 355 ++++++++++
build-riscv-kernel.sh | 26 +
drivers/vhost/scsi.c | 187 +++++-
drivers/vhost/vhost.c | 595 ++++++++++++++++-
drivers/vhost/vhost.h | 29 +-
include/uapi/linux/vhost.h | 4 +
include/uapi/linux/vhost_types.h | 6 +
10 files changed, 2221 insertions(+), 19 deletions(-)
create mode 100644 Documentation/compare-sqcq-results.sh
create mode 100644 Documentation/run-sqcq-compare.sh
create mode 100644 Documentation/virtio-sqcq-poll-spec.rst
create mode 100644 Documentation/virtio-sqcq-poll-virtio-spec.rst
create mode 100644 build-riscv-kernel.sh
diff --git a/Documentation/compare-sqcq-results.sh
b/Documentation/compare-sqcq-results.sh
new file mode 100644
index 0000000000000..85bb927fe91e5
--- /dev/null
+++ b/Documentation/compare-sqcq-results.sh
@@ -0,0 +1,239 @@
+#!/usr/bin/env python3
+"""compare-sqcq-results.py - Generate A/B comparison report for SQ/CQ polling
vs baseline.
+
+Usage:
+ python3 compare-sqcq-results.py <baseline_dir> <sqcq_dir>
+ python3 compare-sqcq-results.py # auto-detect latest results
+"""
+import json, sys, os, glob, re
+from collections import OrderedDict
+
+def find_latest_dirs():
+ """Find the most recent baseline and sqcq-poll result directories."""
+ base = "perf-results"
+ if not os.path.isdir(base):
+ print(f"Error: '{base}/' not found. Run run-sqcq-compare.sh first.")
+ sys.exit(1)
+
+ bl_dirs = sorted(glob.glob(os.path.join(base, "baseline_*")))
+ sq_dirs = sorted(glob.glob(os.path.join(base, "sqcq-poll_*")))
+
+ if not bl_dirs:
+ print("Error: No baseline_* directory found in perf-results/")
+ sys.exit(1)
+ if not sq_dirs:
+ print("Error: No sqcq-poll_* directory found in perf-results/")
+ sys.exit(1)
+
+ return bl_dirs[-1], sq_dirs[-1]
+
+
+def parse_fio_json(filepath):
+ """Extract key metrics from a fio JSON output file."""
+ with open(filepath) as f:
+ d = json.load(f)
+
+ jobs = d.get("jobs", [])
+ if not jobs:
+ return None
+
+ j = jobs[0]
+ rd = j.get("read", {})
+ wr = j.get("write", {})
+
+ # Determine which direction has data
+ if rd.get("iops", 0) > 0:
+ data = rd
+ else:
+ data = wr
+
+ iops = data.get("iops", 0)
+ bw_mib = data.get("bw", 0) / 1024 # KiB/s -> MiB/s
+
+ # Latency
+ lat_ns = data.get("lat_ns", {})
+ avg_us = lat_ns.get("mean", 0) / 1000
+ min_us = lat_ns.get("min", 0) / 1000
+ max_us = lat_ns.get("max", 0) / 1000
+ stddev_us = lat_ns.get("stddev", 0) / 1000
+
+ # Completion latency percentiles
+ clat = data.get("clat_ns", {})
+ pct = clat.get("percentile", {}) if isinstance(clat, dict) else {}
+ p50 = pct.get("50.000000", 0) / 1000
+ p95 = pct.get("95.000000", 0) / 1000
+ p99 = pct.get("99.000000", 0) / 1000
+ p999 = pct.get("99.900000", 0) / 1000
+
+ return {
+ "iops": iops,
+ "bw_mib": bw_mib,
+ "lat_avg_us": avg_us,
+ "lat_min_us": min_us,
+ "lat_max_us": max_us,
+ "lat_stddev_us": stddev_us,
+ "p50_us": p50,
+ "p95_us": p95,
+ "p99_us": p99,
+ "p999_us": p999,
+ }
+
+
+def pct_change(old, new):
+ """Calculate percentage change. Returns None if division by zero."""
+ if old == 0:
+ return None
+ return (new - old) / old * 100
+
+
+def fmt_pct(val):
+ """Format a percentage change value."""
+ if val is None:
+ return "N/A"
+ sign = "+" if val > 0 else ""
+ return f"{sign}{val:.1f}%"
+
+
+def fmt_int(val):
+ """Format integer with comma separators."""
+ return f"{val:,.0f}"
+
+
+def fmt_f1(val):
+ """Format float with 1 decimal."""
+ return f"{val:.1f}"
+
+
+def main():
+ if len(sys.argv) >= 3:
+ bl_dir = sys.argv[1]
+ sq_dir = sys.argv[2]
+ else:
+ bl_dir, sq_dir = find_latest_dirs()
+
+ print(f"Baseline: {bl_dir}")
+ print(f"SQ/CQ Poll: {sq_dir}")
+ print()
+
+ # Collect all test labels
+ bl_files = {os.path.basename(f).replace("_best.json", ""): f
+ for f in glob.glob(os.path.join(bl_dir, "*_best.json"))}
+ sq_files = {os.path.basename(f).replace("_best.json", ""): f
+ for f in glob.glob(os.path.join(sq_dir, "*_best.json"))}
+
+ all_labels = sorted(set(bl_files.keys()) | set(sq_files.keys()))
+
+ if not all_labels:
+ print("Error: No *_best.json files found.")
+ sys.exit(1)
+
+ # Define metric display columns
+ metrics = [
+ ("IOPS", "iops", fmt_int, True),
+ ("BW(MiB/s)", "bw_mib", fmt_f1, True),
+ ("Avg(us)", "lat_avg_us", fmt_f1, False),
+ ("P50(us)", "p50_us", fmt_f1, False),
+ ("P95(us)", "p95_us", fmt_f1, False),
+ ("P99(us)", "p99_us", fmt_f1, False),
+ ("P99.9(us)", "p999_us", fmt_f1, False),
+ ]
+
+ # Per-label comparison
+ results = []
+ for label in all_labels:
+ bl_data = parse_fio_json(bl_files[label]) if label in bl_files else
None
+ sq_data = parse_fio_json(sq_files[label]) if label in sq_files else
None
+
+ entry = {"label": label, "baseline": bl_data, "sqcq": sq_data}
+ results.append(entry)
+
+ # Print grouped by workload type
+ groups = OrderedDict()
+ for label in all_labels:
+ # Extract workload type from label: e.g. "4k-randread-od1-nj1" ->
"4k-randread"
+ parts = label.rsplit("-od", 1)
+ wl = parts[0] if len(parts) > 1 else label
+ if wl not in groups:
+ groups[wl] = []
+ groups[wl].append(label)
+
+ print("=" * 100)
+ print(" SQ/CQ POLL vs BASELINE COMPARISON REPORT")
+ print("=" * 100)
+
+ for wl, labels in groups.items():
+ print(f"\n{'─' * 100}")
+ print(f" Workload: {wl}")
+ print(f"{'─' * 100}")
+
+ for label in labels:
+ r = next(x for x in results if x["label"] == label)
+ bl = r["baseline"]
+ sq = r["sqcq"]
+
+ # Extract iodepth/numjobs from label
+ m = re.search(r'od(\d+)-nj(\d+)', label)
+ od = m.group(1) if m else "?"
+ nj = m.group(2) if m else "?"
+
+ print(f"\n iodepth={od}, numjobs={nj}")
+ print(f" {'Metric':<14} {'Baseline':>12} {'SQ/CQ Poll':>12}
{'Change':>10}")
+ print(f" {'─'*14} {'─'*12} {'─'*12} {'─'*10}")
+
+ for metric_name, metric_key, fmt_fn, higher_is_better in metrics:
+ bl_val = bl[metric_key] if bl else 0
+ sq_val = sq[metric_key] if sq else 0
+ change = pct_change(bl_val, sq_val)
+
+ # For latency, negative change is improvement
+ line = f" {metric_name:<14} {fmt_fn(bl_val):>12}
{fmt_fn(sq_val):>12}"
+
+ if change is not None:
+ marker = ""
+ if higher_is_better and change > 2:
+ marker = " *"
+ elif not higher_is_better and change < -2:
+ marker = " *"
+ line += f" {fmt_pct(change):>10}{marker}"
+ else:
+ line += f" {'N/A':>10}"
+
+ print(line)
+
+ # Summary table - just IOPS
+ print(f"\n{'=' * 100}")
+ print(" IOPS SUMMARY (all scenarios)")
+ print(f"{'=' * 100}")
+ print(f"\n {'Test':<30} {'Baseline':>10} {'SQ/CQ':>10} {'Change':>10}")
+ print(f" {'─'*30} {'─'*10} {'─'*10} {'─'*10}")
+
+ for r in results:
+ bl_iops = r["baseline"]["iops"] if r["baseline"] else 0
+ sq_iops = r["sqcq"]["iops"] if r["sqcq"] else 0
+ change = pct_change(bl_iops, sq_iops)
+
+ marker = " **" if change and change > 5 else ""
+ print(f" {r['label']:<30} {bl_iops:>10,.0f} {sq_iops:>10,.0f}
{fmt_pct(change):>10}{marker}")
+
+ # Latency summary
+ print(f"\n{'=' * 100}")
+ print(" P99 LATENCY SUMMARY (all scenarios)")
+ print(f"{'=' * 100}")
+ print(f"\n {'Test':<30} {'Base(p99)':>12} {'SQ/CQ(p99)':>12}
{'Change':>10}")
+ print(f" {'─'*30} {'─'*12} {'─'*12} {'─'*10}")
+
+ for r in results:
+ bl_p99 = r["baseline"]["p99_us"] if r["baseline"] else 0
+ sq_p99 = r["sqcq"]["p99_us"] if r["sqcq"] else 0
+ change = pct_change(bl_p99, sq_p99)
+
+ marker = " **" if change and change < -5 else ""
+ print(f" {r['label']:<30} {bl_p99:>12.1f} {sq_p99:>12.1f}
{fmt_pct(change):>10}{marker}")
+
+ print(f"\n * = notable improvement")
+ print(f" ** = significant improvement (>5% IOPS gain or >5% latency
reduction)")
+ print()
+
+
+if __name__ == "__main__":
+ main()
diff --git a/Documentation/run-sqcq-compare.sh
b/Documentation/run-sqcq-compare.sh
new file mode 100644
index 0000000000000..3a5928bfaef2a
--- /dev/null
+++ b/Documentation/run-sqcq-compare.sh
@@ -0,0 +1,182 @@
+#!/bin/bash
+# run-sqcq-compare.sh - A/B comparison for SQ/CQ polling vs baseline
+#
+# Usage:
+# 1. Boot Guest WITHOUT VIRTIO_F_SQCQ_POLL: ./run-sqcq-compare.sh baseline
+# 2. Boot Guest WITH VIRTIO_F_SQCQ_POLL: ./run-sqcq-compare.sh sqcq-poll
+#
+# Results saved to: perf-results/<mode>_<timestamp>/
+#
+# IMPORTANT: CPU pinning uses --cpus_allowed (NOT taskset).
+# Verify with: ps -eo pid,comm,psr | grep fio
+#
+set -euo pipefail
+
+MODE=${1:?"Usage: $0 <baseline|sqcq-poll>"}
+DEV=${2:-/dev/sda}
+PIN_CPU=7
+RUNTIME=60
+RUNS=3
+
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+OUTDIR="perf-results/${MODE}_${TIMESTAMP}"
+mkdir -p "$OUTDIR"
+
+# Verify CPU pinning mapping: PIN_CPU -> hwq -> single Host vq
+echo "=== CPU/mq mapping verification ==="
+echo "PIN_CPU=$PIN_CPU"
+for d in /sys/block/$(basename $DEV)/mq/*/; do
+ dir=$(basename "$d")
+ cpus=$(cat "$d/cpu_list" 2>/dev/null || echo "?")
+ echo " hwq $dir: CPU $cpus"
+done
+echo ""
+
+echo "=== Mode: $MODE | Device: $DEV | CPU pin: $PIN_CPU | Runs: $RUNS ==="
+echo "=== Output: $OUTDIR ==="
+echo ""
+
+# Collect Host dmesg marker (for correlating with Host-side vhost stats)
+dmesg > "$OUTDIR/guest-dmesg-before.log" 2>/dev/null || true
+
+# Test matrix: label rw bs iodepth numjobs
+TESTS=(
+ "4k-randread-od1-nj1 randread 4k 1 1"
+ "4k-randread-od32-nj1 randread 4k 32 1"
+ "4k-randread-od32-nj4 randread 4k 32 4"
+ "4k-randread-od32-nj8 randread 4k 32 8"
+ "4k-randwrite-od1-nj1 randwrite 4k 1 1"
+ "4k-randwrite-od32-nj1 randwrite 4k 32 1"
+ "4k-randwrite-od32-nj4 randwrite 4k 32 4"
+ "4k-randwrite-od32-nj8 randwrite 4k 32 8"
+ "128k-seqread-od1-nj1 read 128k 1 1"
+ "128k-seqread-od32-nj1 read 128k 32 1"
+ "128k-seqread-od32-nj4 read 128k 32 4"
+ "128k-seqread-od32-nj8 read 128k 32 8"
+ "128k-seqwrite-od1-nj1 write 128k 1 1"
+ "128k-seqwrite-od32-nj1 write 128k 32 1"
+ "128k-seqwrite-od32-nj4 write 128k 32 4"
+ "128k-seqwrite-od32-nj8 write 128k 32 8"
+)
+
+for entry in "${TESTS[@]}"; do
+ read -r LABEL RW BS OD NJ <<< "$entry"
+
+ echo "=== $LABEL (iodepth=$OD, numjobs=$NJ) ==="
+
+ BEST_IOPS=0
+ BEST_FILE=""
+
+ for run in $(seq 1 $RUNS); do
+ OUTFILE="${OUTDIR}/${LABEL}_run${run}.json"
+
+ # For numjobs=1, pin to single CPU; for numjobs>1, let fio distribute
+ PIN_ARGS=""
+ if [ "$NJ" -eq 1 ]; then
+ PIN_ARGS="--cpus_allowed=$PIN_CPU --cpus_allowed_policy=shared"
+ fi
+
+ echo " run $run/$RUNS ..."
+ fio --ioengine=libaio \
+ --direct=1 \
+ --rw="$RW" \
+ --bs="$BS" \
+ --iodepth=$OD \
+ --numjobs=$NJ \
+ $PIN_ARGS \
+ --time_based \
+ --runtime=$RUNTIME \
+ --group_reporting \
+ --percentile_list=50,95,99,99.9 \
+ --filename="$DEV" \
+ --name="$LABEL" \
+ --output-format=json \
+ --output="$OUTFILE"
+
+ # Extract IOPS from JSON
+ IOPS=$(python3 -c "
+import json, sys
+with open('$OUTFILE') as f:
+ d = json.load(f)
+jobs = d.get('jobs', [])
+if jobs:
+ rd = jobs[0].get('read', {})
+ wr = jobs[0].get('write', {})
+ iops = rd.get('iops', 0) + wr.get('iops', 0)
+ print(f'{iops:.0f}')
+else:
+ print('0')
+" 2>/dev/null || echo "0")
+
+ echo " IOPS: $IOPS"
+
+ # Track best run
+ if [ "$IOPS" -gt "$BEST_IOPS" ]; then
+ BEST_IOPS=$IOPS
+ BEST_FILE=$OUTFILE
+ fi
+ done
+
+ # Copy best run as the canonical result
+ if [ -n "$BEST_FILE" ]; then
+ cp "$BEST_FILE" "${OUTDIR}/${LABEL}_best.json"
+ fi
+
+ echo ""
+done
+
+# Collect dmesg after
+dmesg > "$OUTDIR/guest-dmesg-after.log" 2>/dev/null || true
+
+# Generate summary
+echo "=== Generating summary ==="
+python3 - <<'PYEOF' "$OUTDIR"
+import json, sys, os, glob
+
+outdir = sys.argv[1]
+print(f"\n{'='*80}")
+print(f" RESULTS: {os.path.basename(outdir)}")
+print(f"{'='*80}\n")
+
+header = f"{'Test':<30} {'IOPS':>10} {'p50(us)':>10} {'p99(us)':>10}
{'p99.9(us)':>10} {'BW(MiB/s)':>10} {'lat_avg(us)':>12}"
+print(header)
+print("-" * len(header))
+
+for best in sorted(glob.glob(os.path.join(outdir, "*_best.json"))):
+ label = os.path.basename(best).replace("_best.json", "")
+ try:
+ with open(best) as f:
+ d = json.load(f)
+ jobs = d.get("jobs", [])
+ if not jobs:
+ continue
+
+ j = jobs[0]
+ rd = j.get("read", {})
+ wr = j.get("write", {})
+ iops = rd.get("iops", 0) + wr.get("iops", 0)
+ bw = (rd.get("bw", 0) + wr.get("bw", 0)) / 1024 # KiB/s -> MiB/s
+
+ lat_ns = rd.get("lat_ns", {}) or wr.get("lat_ns", {})
+ avg_us = lat_ns.get("mean", 0) / 1000
+
+ # Percentiles from clat_ns (completion latency)
+ clat = rd.get("clat_ns", {}) or wr.get("clat_ns", {})
+ pct = clat.get("percentile", {}) if isinstance(clat, dict) else {}
+ p50 = pct.get("50.000000", 0) / 1000
+ p99 = pct.get("99.000000", 0) / 1000
+ p999 = pct.get("99.900000", 0) / 1000
+
+ print(f"{label:<30} {iops:>10,.0f} {p50:>10.1f} {p99:>10.1f}
{p999:>10.1f} {bw:>10.1f} {avg_us:>12.1f}")
+ except Exception as e:
+ print(f"{label:<30} ERROR: {e}")
+
+print()
+PYEOF
+
+echo ""
+echo "=== Done. Results in $OUTDIR ==="
+echo "=== After running both modes, compare with: ==="
+echo " diff <(python3 summarize.py perf-results/baseline_*) <(python3
summarize.py perf-results/sqcq-poll_*)"
+echo ""
+echo "=== Or paste both summary tables side-by-side for comparison ==="
diff --git a/Documentation/virtio-sqcq-poll-spec.rst
b/Documentation/virtio-sqcq-poll-spec.rst
new file mode 100644
index 0000000000000..523bb3e3a1963
--- /dev/null
+++ b/Documentation/virtio-sqcq-poll-spec.rst
@@ -0,0 +1,617 @@
+Virtio SQ/CQ Polling Mode — 技术规格书
+==========================================
+
+Host Kernel vhost 端已实现。Guest Kernel 端已实现。QEMU 端已实现。
+
+1. 概述
+-------
+
+传统 virtio 使用 MMIO kick (Guest→Device) 和 MSI-X 中断 (Device→Guest) 通知，
+每次涉及 VM exit。SQ/CQ polling 借鉴 io_uring SQPOLL 模型，在 split ring 旁引入
+SQ/CQ 门铃结构，由轮询线程检测变更替代中断，消除 VM exit。
+
+- Guest→Device：Guest 更新 SQ.idx，Device 轮询线程检测
+- Device→Guest：Device 更新 CQ.idx，Guest 轮询线程检测
+
+SQ/CQ 门铃仅做通知信号，实际数据仍通过 desc/avail/used ring 传递。
+通过 Feature Bit 协商，未协商时零开销。仅支持 split ring。
+
+1.1 地址传递架构
+~~~~~~~~~~~~~~~
+
+::
+
+ Guest Kernel QEMU (userspace) Host Kernel (vhost)
+ PCI config space GPA → HVA 转换 VHOST_SET_VRING_SQCQ_ADDR
+ (写入 GPA) ioctl (传入 HVA) (接收 HVA)
+
+1.2 通知路径
+~~~~~~~~~~~~
+
+::
+
+ 活跃路径（零开销）：
+ Guest 提交 → SQ.idx → Device 轮询检测（消除 VM exit）
+ Device 完成 → CQ.idx → Guest 轮询检测（消除 VM exit）
+
+ 休眠唤醒路径：
+ Guest 提交 → SQ.idx → MMIO kick → QEMU → eventfd → vhost 唤醒
+ Device 完成 → CQ.idx → eventfd_signal → QEMU → MSI-X → Guest 唤醒
+
+
+2. Feature 协商
+---------------
+
+``VIRTIO_F_SQCQ_POLL = 42``（``include/uapi/linux/virtio_config.h``）。
+
+``VIRTIO_TRANSPORT_F_END`` 从 42 更新为 43。
+该 bit 属于 transport feature range (28..43)，需同时被 virtio_ring 和 virtio_pci 接受。
+vhost 端将该 feature 加入 ``VHOST_FEATURES`` 宏，在 ``VHOST_GET_FEATURES`` ioctl 中对
Guest 可见。
+
+**协商流程：**
+
+1. Device 设置 bit 42 → Guest 接受 → finalize_features
+2. Guest 在 find_vqs 阶段分配 SQ/CQ 结构
+3. Guest 在 vp_active_vq 阶段将 SQ/CQ DMA 地址写入 PCI config space
+4. QEMU 读 GPA → 转 HVA → ``VHOST_SET_VRING_SQCQ_ADDR`` ioctl → vhost
+5. vhost 在 ``VHOST_SCSI_SET_ENDPOINT`` 时启动轮询线程
+
+任一方不支持则回退传统中断模式。
+
+
+3. 数据结构
+-----------
+
+3.1 SQ/CQ 门铃
+~~~~~~~~~~~~~~~
+
+定义于 ``include/uapi/linux/virtio_ring.h``：
+
+.. code-block:: c
+
+ struct vring_sq {
+ __virtio64 idx; /* Producer position: Guest 更新 */
+ __u8 sq_need_wakeup; /* 1 = Device 应被 kick 唤醒 */
+ __u8 reserved[55];
+ } __attribute__((aligned(128))); /* 128 bytes, cache-line 对齐 */
+
+ struct vring_cq {
+ __virtio64 idx; /* Producer position: Device 更新 */
+ __u8 cq_need_wakeup; /* 1 = Guest 应被中断唤醒 */
+ __u8 reserved[55];
+ } __attribute__((aligned(128))); /* 128 bytes, cache-line 对齐 */
+
+- ``sq->idx`` 对应 ``avail->idx``，``cq->idx`` 对应 ``used->idx``
+- 128 字节对齐覆盖 ARM Neoverse N2/V2、Apple M 系列等 128 字节 cache line 的 CPU，
+ 消除 SQ 与 CQ 之间的 false sharing
+- ``sq/cq_need_wakeup`` 为 8-bit 直接赋值字段，用于 NEED_WAKEUP 协议（见第 6 节）
+
+3.2 PCI Config Space 寄存器
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+追加到 ``struct virtio_pci_modern_common_cfg`` 末尾（offset 62 之后），
+定义于 ``include/uapi/linux/virtio_pci.h``：
+
+.. code-block:: c
+
+ #define VIRTIO_PCI_COMMON_SQE_LO 64
+ #define VIRTIO_PCI_COMMON_SQE_HI 68
+ #define VIRTIO_PCI_COMMON_CQE_LO 72
+ #define VIRTIO_PCI_COMMON_CQE_HI 76
+
+ __le32 sqe_ring_lo; /* offset 64: SQ DMA 地址低 32 位 */
+ __le32 sqe_ring_hi; /* offset 68: SQ DMA 地址高 32 位 */
+ __le32 cqe_ring_lo; /* offset 72: CQ DMA 地址低 32 位 */
+ __le32 cqe_ring_hi; /* offset 76: CQ DMA 地址高 32 位 */
+
+受 queue_select 约束。UAPI 已定义，Guest 已实现写入，QEMU 已实现读取。
+
+3.3 vhost UAPI
+~~~~~~~~~~~~~~~
+
+.. code-block:: c
+
+ // include/uapi/linux/vhost.h
+ #define VHOST_SET_VRING_SQCQ_ADDR _IOW(VHOST_VIRTIO, 0x27, \
+ struct vhost_vring_sqcq_addr)
+
+ // include/uapi/linux/vhost_types.h
+ struct vhost_vring_sqcq_addr {
+ unsigned int index;
+ __u64 sq_user_addr; /* SQ doorbell HVA */
+ __u64 cq_user_addr; /* CQ doorbell HVA */
+ };
+
+QEMU 调用时机：在 ``VHOST_SET_VRING_ADDR`` 之后、``VHOST_SCSI_SET_ENDPOINT`` 之前。
+
+3.4 写入时序
+~~~~~~~~~~~~
+
+Guest（vp_active_vq 中）：queue_select → desc/avail/used 地址 → sqe/cqe_ring_lo/hi 地址
+QEMU：组合 GPA → address_space_map 转 HVA → VHOST_SET_VRING_SQCQ_ADDR ioctl
+
+
+4. 字段读写所有权
+-----------------
+
+.. list-table::
+ :header-rows: 1
+ :widths: 20 20 20 25 15
+
+ * - 字段
+ - 写入方
+ - 读取方
+ - 内存序
+ - 对应
+
+ * - sq->idx
+ - Guest
+ - Device
+ - Guest: smp_store_release; Device: get_user + smp_rmb
+ - avail->idx
+
+ * - sq->sq_need_wakeup
+ - Device
+ - Guest
+ - Device: smp_wmb + put_user + smp_mb (double-check)
+ - NEED_WAKEUP
+
+ * - cq->idx
+ - Device
+ - Guest
+ - Device: smp_wmb before put_user; Guest: smp_load_acquire
+ - used->idx
+
+ * - cq->cq_need_wakeup
+ - Guest
+ - Device
+ - Guest: smp_store_release; Device: get_user + smp_rmb
+ - NEED_WAKEUP
+
+
+5. 核心协议
+-----------
+
+5.1 Guest 提交请求 (SQ 方向)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: c
+
+ // virtqueue_kick() 中 sqcq_poll 模式
+ smp_store_release(&vring->sq->idx, avail_idx_shadow);
+ need_wakeup = smp_load_acquire(&vring->sq->sq_need_wakeup);
+ if (need_wakeup)
+ virtqueue_notify(vq); // MMIO write，仅在 Device 休眠时
+
+**关键：** Guest 仍写 ``avail->idx``（``virtqueue_add_split()`` 完成）。
+SQ idx 是额外通知信号，让 Device 快速检测（读一个 cache line vs 读 avail ring 两次）。
+Device 的 ``handle_kick()`` 通过 ``avail->idx`` 定位 descriptor。
+
+**virtqueue_kick_prepare() 守卫：** sqcq_poll 模式下直接返回 ``true``，
+不检查 ``used->flags``（vhost 不维护该 flag）。
+
+5.2 Device 处理请求 (vhost 轮询线程)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+独立 kthread ``vhost_sqcq_poll_thread``。直接调用 ``handle_kick`` 处理提交，
+通过 ``poll_complete`` 回调处理完成，单线程闭环。
+
+主循环伪代码（drivers/vhost/vhost.c）：
+
+.. code-block:: c
+
+ while (!poll_stop):
+ // 0. 处理 pending completion
+ if (atomic_read(&vq->completion_pending) && vq->poll_complete):
+ atomic_set(&vq->completion_pending, 0)
+ vq->poll_complete(vq) // copy resp, add used, 写 cq->idx, 检查
cq_need_wakeup
+
+ // 1. 读 SQ idx
+ ret = get_user(sq_idx_val, &vq->sq->idx)
+ smp_rmb() // acquire
+ new_sq_idx = (u16)sq_idx_val
+
+ // 2. 有新提交：同步调用 handle_kick
+ if (new_sq_idx != vq->last_avail_idx):
+ vq->handle_kick(&vq->poll.work)
+
+ // 3. 自适应休眠（EMA 动态超时）
+ should_sleep = (last_avail_idx == last_used_idx)
+ if (!should_sleep && ema_ns > 0)
+ should_sleep = (active_io_vqs > 1) && (ema_us >
SQCQ_SLEEP_THRESHOLD_US)
+
+ if (should_sleep):
+ timeout = ema_based_or_fallback(SQCQ_IDLE_TIMEOUT_US_MAX)
+ // 设置 NEED_WAKEUP + double-check idx
+ smp_wmb(); put_user(1, &sq->sq_need_wakeup); smp_mb()
+ if (idx changed): put_user(0, &sq->sq_need_wakeup); continue
+ // 注册 kick eventfd，带超时等待
+ kick_register(vq)
+ wait_event_interruptible_timeout(poll_wait,
+ poll_stop || kick_received || completion_pending, timeout)
+ was_kicked = READ_ONCE(kick_received)
+ kick_unregister(vq)
+ put_user(0, &sq->sq_need_wakeup)
+ else:
+ cpu_relax()
+
+5.3 I/O 完成路径
+~~~~~~~~~~~~~~~~~
+
+Poll 模式下 completion 不经过 vhost_worker：
+
+1. **TCM 完成**（中断/workqueue 上下文）：``llist_add`` +
``atomic_set(completion_pending, 1)`` + 条件 ``wake_up``
+2. **Poll thread 步骤 0**（poll thread 上下文）：``poll_complete`` 回调
+3. **poll_complete 内部**：``llist_del_all`` → ``mutex_lock`` → copy response +
``vhost_add_used`` → ``mutex_unlock`` → ``put_user(cq->idx)`` → 检查
``cq_need_wakeup`` → ``eventfd_signal``
+
+设计要点：单线程闭环（handle_kick 和 poll_complete 顺序执行，共享 mutex 不会死锁）；
+CQ idx 单写者（仅 poll_complete）；条件 wake_up（仅在 poll thread 非 RUNNING 时）。
+
+5.4 vhost_enable/disable_notify 在 poll 模式下
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- ``vhost_enable_notify()`` → ``return false``
+- ``vhost_disable_notify()`` → ``return``
+
+通知控制由 sq_need_wakeup / cq_need_wakeup 承担，used ring flags 不再有效。
+Guest 端对称：``virtqueue_kick_prepare()`` 直接返回 true，``disable_cb/enable_cb`` 操作
avail->flags 但 vhost 忽略。
+
+5.5 Guest 接收完成 (CQ 方向)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Guest 轮询线程（``virtio_sqcq_poll_thread``），每 device 一个线程。
+
+主循环要点：
+
+1. ``while (cq_poll_has_work())`` drain 该 vq 所有完成，callback 后 ``do_softirq()``
+2. 有工作时重置 ``last_io_ts = ktime_get()``，不 yield，快速轮询
+3. 无工作时基于 ``last_io_ts`` 时间窗口自旋（``busy_spin_us`` = 1ms）
+4. 自旋中使用 ``need_resched()`` + ``cpu_relax()`` 协作让出（io_uring SQPOLL 模型）
+5. spin 窗口过期后设置所有 vq 的 ``cq_need_wakeup`` + double-check ``more_used()``
+6. 无新工作则 ``schedule_timeout(idle_timeout_us)`` 休眠（1ms）
+7. 醒来后清除所有 vq 的 NEED_WAKEUP，不重置 spin 窗口（超时唤醒不 re-arm spin）
+
+**CQ 完成检测：** 直接使用 ``more_used(vq)`` 检查 used ring，不再轮询 ``cq->idx``。
+``cq->idx`` 仅由 Device 写入，保留用于 NEED_WAKEUP 协议的 ``cq_need_wakeup`` 字段，
+不再作为完成检测的 fast path（消除了 cq->idx 与 used ring 之间的竞态）。
+
+**CQ 访问器函数** (virtio_ring.c)：
+
+- ``cq_poll_has_work(vq)``：``more_used(vq)`` — 直接检查 used ring
+- ``cq_set_need_wakeup(vq)``：``smp_store_release(&cq->cq_need_wakeup, 1)``
+- ``cq_clear_need_wakeup(vq)``：``WRITE_ONCE(cq->cq_need_wakeup, 0)``
+- ``cq_recheck(vq)``：``smp_mb()`` + ``more_used(vq)``
+
+5.6 Device 唤醒路径 (kick eventfd 桥接)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+::
+
+ Guest MMIO kick → QEMU eventfd write → eventfd wq →
+ vhost_sqcq_kick_wakeup() → kick_received=true + wake_up →
+ poll thread → clear sq_need_wakeup → process sq->idx
+
+休眠前通过 ``vhost_sqcq_kick_register()`` 注册到 kick eventfd wait queue。
+关键：``vfs_poll`` 可能因 stale counter 返回 EPOLLIN，必须验证 ``sq->idx``
+确实有新工作才触发回调，否则所有 poll thread busy-loop。
+
+QEMU 端无需额外操作，现有 kick eventfd 路径即可。
+
+
+6. NEED_WAKEUP 协议（核心，必须严格遵守）
+-----------------------------------------
+
+双向对称，SQ 和 CQ 各一套。避免死锁和丢失唤醒的核心机制。
+
+6.1 SQ 方向
+~~~~~~~~~~~
+
+**Device 休眠前：**
+
+.. code-block:: pseudo
+
+ smp_wmb()
+ sq->sq_need_wakeup = 1
+ smp_mb()
+ new_sq_idx = sq->idx // double-check
+ if (new_sq_idx != last_avail):
+ sq->sq_need_wakeup = 0 // 取消休眠
+ goto process
+ wait_event(kick_received)
+ sq->sq_need_wakeup = 0 // 醒来后清除
+
+**Guest 提交后：**
+
+.. code-block:: pseudo
+
+ smp_store_release(&sq->idx, new_idx)
+ need_wakeup = smp_load_acquire(&sq->sq_need_wakeup)
+ if (need_wakeup)
+ virtqueue_notify(vq) // MMIO kick
+
+6.2 CQ 方向
+~~~~~~~~~~~
+
+ **Guest 休眠前：**
+
+.. code-block:: pseudo
+
+ smp_store_release(&cq->cq_need_wakeup, 1)
+ smp_mb()
+ if (more_used(vq)): // double-check used ring
+ cq->cq_need_wakeup = 0 // 取消休眠
+ goto process
+ schedule_timeout(idle_timeout_us)
+ cq->cq_need_wakeup = 0 // 醒来后清除
+
+**Device 完成后（两阶段）：**
+
+1. release_cmd：``llist_add`` + ``atomic_set(completion_pending, 1)`` + 条件
``wake_up``
+2. poll_complete：copy resp → add used → ``put_user(cq->idx)`` → 检查
``cq_need_wakeup`` → ``eventfd_signal``
+
+6.3 为什么需要 Double-check
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+设置 NEED_WAKEUP → 重读 idx 之间存在竞态：Guest 可能在 Device 读 idx 后、
+sleep 前提交。Device 在 sleep 前重读 idx，如果变了则取消休眠。
+此时 Guest 的 kick 可能已到也可能未到，但 Device 已在处理新工作，不影响正确性。
+
+6.4 内存序总结
+~~~~~~~~~~~~~~
+
+- idx 更新：写入方 store-release（Guest: ``smp_store_release``；Device: ``smp_wmb`` +
``put_user``）
+- idx 读取：Guest 使用 ``more_used()`` 直接检查 used ring（不再轮询 ``cq->idx``）
+- sq_need_wakeup 读取：Guest 使用 ``smp_load_acquire``（替代 ``smp_mb`` +
``READ_ONCE``）
+- NEED_WAKEUP 设置后 ``smp_mb()``，然后重读 used ring / sq->idx
+- NEED_WAKEUP 清除在 sleep 唤醒后立即执行
+- x86 上屏障为 no-op（TSO），ARM/RISC-V 上生成实际屏障指令
+
+
+7. QEMU 实现指南
+----------------
+
+7.1 Feature 协商
+~~~~~~~~~~~~~~~~
+
+.. code-block:: c
+
+ virtio_add_feature(&vdev->host_features, VIRTIO_F_SQCQ_POLL);
+
+7.2 GPA → HVA 转换
+~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: c
+
+ hwaddr sq_gpa = (sqe_ring_hi << 32) | sqe_ring_lo;
+ void *sq_hva = address_space_map(&address_space_memory, sq_gpa,
sizeof(struct vring_sq), false);
+ // ... cq 类似
+ struct vhost_vring_sqcq_addr addr = { .index = idx, .sq_user_addr =
sq_hva, .cq_user_addr = cq_hva };
+ ioctl(vhost_fd, VHOST_SET_VRING_SQCQ_ADDR, &addr);
+
+7.3 ioctl 调用顺序
+~~~~~~~~~~~~~~~~~~
+
+启动：
+
+::
+
+ 1. VHOST_SET_VRING_ADDR (desc/avail/used)
+ 2. VHOST_SET_VRING_SQCQ_ADDR (sq/cq) — 新增
+ 3. VHOST_SET_VRING_KICK
+ 4. VHOST_SET_VRING_CALL
+ 5. VHOST_SCSI_SET_ENDPOINT — 触发轮询线程启动
+
+停止：
+
+::
+
+ 1. VHOST_SCSI_CLEAR_ENDPOINT — 自动停止轮询线程
+ 2. VHOST_SET_VRING_KICK (null_fd)
+ 3. VHOST_SET_VRING_CALL (null_fd)
+
+7.4 call eventfd 处理
+~~~~~~~~~~~~~~~~~~~~~
+
+收到 eventfd signal 说明 Guest 设置了 cq_need_wakeup，需注入 MSI-X 中断。
+使用现有 ``virtio_irq()`` / ``virtio_notify_irqfd()`` 即可。
+
+即使启用 polling，Device 仍需保留中断能力（CQ NEED_WAKEUP 唤醒、config change、fallback）。
+
+7.5 内存序
+~~~~~~~~~~
+
+QEMU 用户态需使用 ``atomic_store_release`` / ``atomic_load_acquire`` 或 C11
``stdatomic.h``。
+
+
+8. Host Kernel vhost 端实现
+----------------------------
+
+8.1 vhost_virtqueue 扩展
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. code-block:: c
+
+ struct vhost_virtqueue {
+ /* ... 现有字段 ... */
+ /* SQ/CQ polling mode */
+ bool sqcq_poll;
+ struct vring_sq __user *sq;
+ struct vring_cq __user *cq;
+ struct task_struct *poll_task;
+ bool poll_stop;
+ wait_queue_head_t poll_wait;
+ struct completion poll_started, poll_stopped;
+ bool kick_received;
+ atomic_t completion_pending;
+ void (*poll_complete)(struct vhost_virtqueue *vq);
+ wait_queue_head_t *kick_wqh;
+ wait_queue_entry_t kick_wait;
+ poll_table kick_pt;
+ /* 10s 统计窗口和 EMA */
+ u64 ema_latency_ns; /* EMA of cmd processing latency */
+ u64 ema_interval_ns; /* EMA of cmd-to-cmd interval */
+ /* ... lat_*/stat_* 统计字段 ... */
+ };
+
+8.2 ioctl 处理
+~~~~~~~~~~~~~~
+
+``vhost_vring_set_sqcq_addr()``：copy_from_user → 验证地址/对齐 → access_ok（backend
已配置时）→ 存储 sq/cq。
+另含自动启动逻辑：当 private_data && sq && cq && !sqcq_poll && poll_complete && feature
时调 ``vhost_sqcq_poll_start()``。
+
+8.3 独立轮询线程
+~~~~~~~~~~~~~~~~~
+
+kthread ``vhost-sqcq-<pid>-<vq_offset>``，通过 ``kthread_use_mm(dev->mm)`` 绑定
QEMU 地址空间。
+直接调用 ``handle_kick`` 处理提交，``poll_complete`` 处理完成，单线程闭环。
+
+API：``vhost_sqcq_poll_start(vq)`` / ``vhost_sqcq_poll_stop(vq)``。
+start 时先创建 kthread 并等待 ``poll_started`` completion，然后才设置 ``sqcq_poll = true``，
+确保 ``vhost_signal()`` 看到 sqcq_poll 时 ``poll_task`` 已有效（消除 NULL deref 窗口）。
+stop 后清除 poll_task、sqcq_poll、sq、cq，flush 残留 completion。
+
+8.4 策略常量与 CPU 绑核
+~~~~~~~~~~~~~~~~~~~~~~~
+
+策略常量（编译期固定，``#define``）：
+
+.. code-block:: c
+
+ #define SQCQ_IDLE_TIMEOUT_US_MAX 1000U /* sleep 时长上界 (us) */
+ #define SQCQ_SLEEP_EMA_MULT 2 /* sleep 时长 = N * ema_latency
*/
+ #define SQCQ_SLEEP_THRESHOLD_US 20U /* 快于此时长的后端选择 spin */
+ #define SQCQ_SPIN_EMA_MULT 2 /* inflight spin 窗口 = N *
ema_latency */
+ #define SQCQ_ACTIVE_TIMEOUT_NS 1000000U /* 无 inflight 固定自旋窗口 1ms */
+ #define SQCQ_STALE_TIMEOUT_NS 1000000000ULL /* sq_was_active 兜底清除 1s
*/
+
+**CPU 绑核**：round-robin（``sqcq_poll_cpu_rr`` 全局原子计数器 ``% num_online_cpus()``），
+分散到不同在线 CPU。计数器取模确保重启后不会溢出到 fallback（全绑 cpu0）。
+
+**协作式自旋**（io_uring SQPOLL 模型）：两条自旋路径均使用
+``need_resched()`` / ``cond_resched()`` 协作让出——自旋时若调度器标记需要让出，
+主动 ``cond_resched()``，避免饿死同核其他任务。线程保持 TASK_RUNNING（轮询
+``sq->idx``），不进入 sleep 路径、不设 ``sq_need_wakeup``，从而消除 Guest kick
+（VM exit）。仅在无 inflight 且超过 1ms 无新提交时才真正 sleep。
+
+**TCM 完成 CPU 导向**：``sqcq_poll_active_cpu_mask`` 记录忙 poll 线程占用的 CPU
+（跟着 ``sq_was_active`` 翻转）。``vhost_sqcq_pick_completion_cpu()`` 从空闲 CPU
+中选一个给 TCM 完成，避开忙 poll 核；全占满时回退到 next-CPU。
+
+8.5 vhost_signal / notify
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- ``vhost_signal()``：poll 模式下设 completion_pending + 条件 wake_up。SCSI completion
不经过此路径
+- ``vhost_enable_notify()``：poll 模式下 return false
+- ``vhost_disable_notify()``：poll 模式下 return
+- ``vhost_dev_stop()``：仅对 sqcq_poll vq 加锁停止
+
+8.6 vhost-scsi 集成
+~~~~~~~~~~~~~~~~~~~
+
+``set_endpoint`` 中对 I/O VQs（index >= VHOST_SCSI_VQ_IO）：如果 feature + sq + cq 则
start。
+``clear_endpoint`` 中先 stop 再 clear backend。
+
+8.7 call_ctx.ctx NULL 检查
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+``poll_complete`` 中 ``eventfd_signal(call_ctx.ctx)`` 前做 NULL 检查。
+QEMU 必须在 SET_ENDPOINT 之前完成 SET_VRING_CALL 设置。
+
+8.8 空闲策略与 EMA
+~~~~~~~~~~~~~~~~~~
+
+系统维护两个独立的 EMA：
+- ``ema_latency_ns``：cmd 处理延迟（submit → process_completions）的 EMA，每次
``poll_complete`` 后更新 ``old - old/8 + avg/8``（溢出安全）
+- ``ema_interval_ns``：cmd-to-cmd 间隔的 EMA，每次检测到 sq->idx 变化时更新 ``old - old/8 +
new/8``
+
+**无 inflight 自旋**（io_uring SQPOLL 模型）：固定 ``SQCQ_ACTIVE_TIMEOUT_NS``（1ms）
+窗口 + ``cond_resched()`` 协作让出。线程持续轮询 ``sq->idx`` 接住下一条提交，
+不进入 sleep 路径、不设 ``sq_need_wakeup`` → 消除 Guest kick（VM exit）。
+仅在超过 1ms 无新提交时才 sleep。
+
+**有 inflight 自旋**：``spin_ns = SQCQ_SPIN_EMA_MULT * ema_latency_ns``，
+``cond_resched()`` 协作让出。超出窗口或多 VQ + 高延迟时 sleep。
+
+Sleep 动态超时：``timeout = ema_latency / (1000 / SQCQ_SLEEP_EMA_MULT)``，clamp 到
[10us, SQCQ_IDLE_TIMEOUT_US_MAX]。
+多 VQ 休眠决策：有 inflight 时仅在 ``sqcq_active_io_vqs > 1`` 且 ``ema_latency_us >
SQCQ_SLEEP_THRESHOLD_US`` 时休眠。
+
+8.9 统计与诊断
+~~~~~~~~~~~~~~
+
+10s 统计窗口，打印
cq/fail/sq_hits/enters/wkick/wcomp/avg/kick_pct/wait_pct/cqw_pct/spin_pct/wake_avg/wake_n/interval/ema_lat/inflight
到 dmesg。
+Stuck 检测：wait_event 超时或窗口内无完成但有 inflight 时告警。
+
+9. Guest Kernel 端实现
+-----------------------
+
+9.1 SQ/CQ DMA 分配
+~~~~~~~~~~~~~~~~~~
+
+``virtio_alloc_sqcq()``：``dma_alloc_coherent()`` 一次分配 256 bytes（sq+cq 各 128
字节），
+在 ``vring_create_virtqueue_split()`` 中分配，``vring_free`` 时释放。
+
+9.2 地址传递
+~~~~~~~~~~~~
+
+``vp_modern_queue_address_sqcq()``：在 ``vp_active_vq()`` 中，通过 PCI common config
+的 queue_select + sqe/cqe_ring_lo/hi 写入 64 位 DMA 地址。
+
+9.3 提交路径
+~~~~~~~~~~~~
+
+``virtqueue_kick()``：``smp_store_release(&sq->idx, shadow)`` → 如果 NEED_WAKEUP
则 MMIO kick。
+``virtqueue_kick_prepare()``：sqqcq_poll 下直接 return true。
+
+9.4 CQ 轮询线程
+~~~~~~~~~~~~~~~~~
+
+独立 kthread ``virtio-sqcq/<device_name>``，每 device 一个线程（``struct
virtio_sqcq_poll_data``）。
+
+**CQ 访问器函数** (virtio_ring.c)：
+
+- ``cq_poll_has_work(vq)``：``smp_load_acquire(&cq->idx) != last_cq_head``
+- ``cq_poll_consume(vq)``：snapshot cq->idx（callback 前）
+- ``cq_set_need_wakeup(vq)``：``smp_store_release(&cq->cq_need_wakeup, 1)``
+- ``cq_clear_need_wakeup(vq)``：``WRITE_ONCE(cq->cq_need_wakeup, 0)``
+- ``cq_recheck(vq)``：``smp_mb()`` + ``smp_load_acquire(&cq->idx) !=
last_cq_head``
+
+**线程生命周期：** ``vp_modern_find_vqs()`` 中启动（``virtio_start_sqcq_poll(vdev, 1000,
1000)``），
+注册所有 poll 活跃的 vq。``vp_del_vqs()`` 中停止（``virtio_stop_sqcq_poll`` 内部释放所有 vq
node）。
+
+**启动参数：** ``idle_timeout_us=1000``（1ms sleep 超时），``busy_spin_us=1000``（1ms
spin 预算）。
+不绑核（调度器自由分配），无指数退避（固定 1ms sleep）。
+
+**config 空间大小检查：** ``vp_check_common_size()`` 验证 common config 至少 80 字节。
+
+
+10. 与现有机制的关系
+---------------------
+
+- **EVENT_IDX：** 可协商但对 polling vq 无效，被 sq/cq_need_wakeup 替代
+- **virtio-net NAPI：** 可共存，NAPI 可直接检查 CQ
+- **blk-mq iopoll：** 互补（iopoll 同步快路径，SQ/CQ 替代异步中断路径）
+- **Live Migration：** reset 时回退中断模式，迁移后 Guest 重新协商
+
+
+11. 文件清单
+------------
+
+**Guest Kernel：** ``virtio_config.h`` (feature bit), ``virtio_ring.h``
(vring_sq/cq),
+``virtio_pci.h`` (PCI offsets), ``virtio_ring.c`` (分配/SQ提交/CQ accessor),
+``virtio_sqcq_poll.c`` (CQ 轮询线程), ``virtio_pci_modern.c``
(feature/queue激活/线程生命周期),
+``virtio_pci_common.c`` (vp_del_vqs), ``virtio_pci_modern_dev.c`` (PCI config
映射)
+
+**Host Kernel vhost：** ``vhost.h`` (ioctl), ``vhost_types.h`` (sqcq_addr
struct),
+``vhost.h/vhost.c`` (轮询线程/ioctl/signal/生命周期), ``scsi.c`` (vhost-scsi 集成)
+
+**QEMU：** ``virtio-pci.c`` (feature/config), ``virtio-pci-modern.c``
(BAR/GPA→HVA)
+
+
+12. 验证方案
+------------
+
+**功能：** QEMU 带 VIRTIO_F_SQCQ_POLL + virtio-scsi → Guest 读写验证正确性。
+
+**性能：** fio 对比中断模式 vs polling，观察 IOPS/延迟/VM exit 次数。
+
+**正确性：** 24h+ stress test；idle/wakeup 验证；NEED_WAKEUP 边界条件；ARM/RISC-V 内存序验证。
diff --git a/Documentation/virtio-sqcq-poll-virtio-spec.rst
b/Documentation/virtio-sqcq-poll-virtio-spec.rst
new file mode 100644
index 0000000000000..26ae41860a8e5
--- /dev/null
+++ b/Documentation/virtio-sqcq-poll-virtio-spec.rst
@@ -0,0 +1,355 @@
+SQ/CQ Doorbell Polling
+======================
+
+This section describes the SQ/CQ doorbell polling mechanism
+(``VIRTIO_F_SQCQ_POLL``), which allows the driver and device to use
+shared-memory doorbells instead of MMIO writes (notifications) and
+interrupts, eliminating VM exits on both submission and completion paths.
+
+This feature is only supported with split virtqueues. A driver or
+device that negotiates this feature MUST NOT also negotiate
+``VIRTIO_F_RING_PACKED``.
+
+
+Feature bit
+-----------
+
+``VIRTIO_F_SQCQ_POLL (42)`` — SQ/CQ Doorbell Polling.
+
+This feature is in the transport feature range (28..43). Both
+``virtio_ring`` and ``virtio_pci`` MUST accept this feature for it
+to be negotiated.
+
+
+Data Structures
+---------------
+
+SQ Doorbell
+~~~~~~~~~~~
+
+The SQ (Submission Queue) doorbell is a cache-line aligned structure
+written by the driver and read by the device:
+
+.. code-block:: c
+
+ struct vring_sq {
+ le64 idx;
+ u8 sq_need_wakeup;
+ u8 reserved[55];
+ };
+
+The structure MUST be aligned to the cache line size of the platform
+(minimum 64 bytes; 128 bytes recommended for platforms with 128-byte
+cache lines such as ARM Neoverse N2/V2).
+
+``idx`` corresponds to ``avail->idx`` and is written by the driver
+using ``smp_store_release()``.
+
+``sq_need_wakeup`` is a single-byte flag written by the device,
+indicating whether the device's poll thread is sleeping and needs to
+be woken by an MMIO write (kick).
+
+CQ Doorbell
+~~~~~~~~~~~
+
+The CQ (Completion Queue) doorbell is a cache-line aligned structure
+written by the device and read by the driver:
+
+.. code-block:: c
+
+ struct vring_cq {
+ le64 idx;
+ u8 cq_need_wakeup;
+ u8 reserved[55];
+ };
+
+The structure MUST be aligned to the same alignment as ``vring_sq``.
+
+``idx`` corresponds to ``used->idx`` and is written by the device.
+
+``cq_need_wakeup`` is a single-byte flag written by the driver,
+indicating whether the driver's poll thread is sleeping and needs to
+be woken by an interrupt.
+
+
+PCI Configuration
+-----------------
+
+The SQ and CQ doorbell DMA addresses are passed through the PCI
+common configuration space. Four new registers are appended to
+``struct virtio_pci_modern_common_cfg``:
+
+============== ======== ========================================
+Offset Field Description
+============== ======== ========================================
+64 sqe_ring_lo SQ doorbell DMA address, low 32 bits
+68 sqe_ring_hi SQ doorbell DMA address, high 32 bits
+72 cqe_ring_lo CQ doorbell DMA address, low 32 bits
+76 cqe_ring_hi CQ doorbell DMA address, high 32 bits
+============== ======== ========================================
+
+These registers are scoped by ``queue_select``. The driver writes
+the SQ and CQ doorbell DMA addresses for each virtqueue during
+``vp_active_vq``.
+
+The device (QEMU) translates the guest physical address to a host
+virtual address and passes it to the vhost backend via
+``VHOST_SET_VRING_SQCQ_ADDR`` ioctl.
+
+
+Allocation
+----------
+
+The driver allocates SQ and CQ doorbells as a single
+``dma_alloc_coherent()`` call (contiguous physical memory):
+
+.. code-block:: c
+
+ size_t total = sizeof(struct vring_sq) + sizeof(struct vring_cq);
+ void *buf = dma_alloc_coherent(dev, total, &dma_addr, GFP_KERNEL);
+ vq->sq = buf;
+ vq->cq = buf + sizeof(struct vring_sq);
+
+Control and event virtqueues MUST NOT use SQ/CQ polling. Only request
+virtqueues (index >= VIRTIO_SCSI_VQ_BASE for virtio-scsi) negotiate
+this feature.
+
+
+Submission Path (Driver → Device)
+---------------------------------
+
+When the driver has new requests available:
+
+1. The driver writes descriptors to the avail ring as usual
+2. The driver updates ``sq->idx`` using ``smp_store_release()``
+3. The driver reads ``sq_need_wakeup`` using ``smp_load_acquire()``
+
+If ``sq_need_wakeup`` is 0, the device's poll thread is actively
+spinning and will detect the new ``sq->idx`` value without any
+further action.
+
+If ``sq_need_wakeup`` is 1, the device's poll thread is sleeping.
+The driver MUST perform a traditional MMIO write (kick) to wake it.
+
+.. code-block:: pseudo
+
+ smp_store_release(&sq->idx, avail_idx_shadow)
+ need_wakeup = smp_load_acquire(&sq->sq_need_wakeup)
+ if (need_wakeup):
+ mmio_kick(vq)
+
+The ``virtqueue_kick_prepare()`` function MUST return ``true``
+unconditionally when SQ/CQ polling is active, bypassing the
+``used->flags`` check.
+
+
+Completion Path (Device → Driver)
+---------------------------------
+
+Device Side
+~~~~~~~~~~~
+
+When the device processes completions:
+
+1. The device writes completion entries to the used ring as usual
+2. The device's poll thread writes ``cq->idx`` to signal completion
+
+The device's poll thread is a dedicated kernel thread that processes
+both submissions (via ``handle_kick``) and completions (via
+``poll_complete`` callback) in a single-threaded loop.
+
+Driver Side
+~~~~~~~~~~~
+
+The driver's poll thread (one per device) detects completions by
+checking ``more_used()`` — directly comparing ``used->idx`` with
+``last_used_idx``:
+
+.. code-block:: pseudo
+
+ while more_used(vq):
+ consume_and_callback(vq)
+ do_softirq()
+
+Note: The driver polls ``used->idx`` directly rather than ``cq->idx``,
+because ``cq->idx`` may lag behind ``used->idx`` (the device writes
+the used ring entries before updating ``cq->idx``). Polling
+``used->idx`` ensures the lowest-latency completion detection.
+
+``cq->idx`` is retained for the ``cq_need_wakeup`` sleep/wake
+protocol (see below) but is not polled as a fast-path completion
+detector.
+
+
+NEED_WAKEUP Protocol
+--------------------
+
+The NEED_WAKEUP protocol prevents lost wakeups when either side
+transitions from polling to sleeping. It is symmetric: SQ uses
+``sq_need_wakeup`` and CQ uses ``cq_need_wakeup``.
+
+SQ NEED_WAKEUP (Device sleeping)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before the device's poll thread sleeps:
+
+1. Set ``sq_need_wakeup = 1`` (via ``put_user``)
+2. Execute ``smp_mb()``
+3. Re-read ``sq->idx`` — if it changed, cancel sleep
+
+After waking (via eventfd or timeout):
+- Clear ``sq_need_wakeup = 0``
+
+The driver checks ``sq_need_wakeup`` after writing ``sq->idx`` and
+performs an MMIO kick only if the flag is set.
+
+CQ NEED_WAKEUP (Driver sleeping)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Before the driver's poll thread sleeps:
+
+1. Set ``cq_need_wakeup = 1`` (via ``smp_store_release``)
+2. Execute ``smp_mb()``
+3. Re-check ``more_used()`` — if work found, cancel sleep
+
+After waking (via eventfd/interrupt or timeout):
+- Clear ``cq_need_wakeup = 0``
+
+The device checks ``cq_need_wakeup`` after writing ``cq->idx`` and
+sends an eventfd signal only if the flag is set.
+
+
+Memory Ordering
+---------------
+
+The following memory ordering rules MUST be observed:
+
+==================== ============================
============================
+Field Writer Reader
+==================== ============================
============================
+``sq->idx`` ``smp_store_release`` ``get_user`` + ``smp_rmb``
+``sq_need_wakeup`` ``put_user`` + ``smp_wmb`` ``smp_load_acquire``
+``cq->idx`` ``put_user`` (with ``smp_wmb``) Not polled (see note)
+``cq_need_wakeup`` ``smp_store_release`` ``get_user`` + ``smp_rmb``
+``used->idx`` Device (via ``vhost_add_used``) ``more_used()`` (plain
read)
+==================== ============================
============================
+
+Note: ``cq->idx`` is written by the device but is not polled by the
+driver's fast path. The driver uses ``more_used()`` (checking
+``used->idx``) for completion detection, which is the authoritative
+source of completions.
+
+On x86 (TSO memory model), all barriers are no-ops. On ARM64 and
+RISC-V, ``smp_store_release`` generates ``stlr`` and
+``smp_load_acquire`` generates ``ldar``, which are lighter than full
+memory barriers (``dmb ish``).
+
+
+Poll Thread Behavior
+--------------------
+
+Device Poll Thread (vhost)
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+One kernel thread per virtqueue. Each thread:
+
+1. Polls ``sq->idx`` for new submissions and ``completion_pending``
+ for completions
+2. When active: busy-spins using ``need_resched()`` + ``cpu_relax()``
+ for cooperative yielding (io_uring SQPOLL pattern)
+3. Spin budget: fixed 1ms (``SQCQ_ACTIVE_TIMEOUT_NS``) for no-inflight,
+ or ``2 × ema_latency`` for has-inflight
+4. After spin budget expires: sets ``sq_need_wakeup``, sleeps via
+ ``wait_event_interruptible_timeout``
+5. Sleep timeout: adaptive, based on EMA latency, clamped to [10us, 1ms]
+
+CPU affinity: round-robin across online CPUs
+(``atomic_counter % num_online_cpus()``).
+
+Driver Poll Thread
+~~~~~~~~~~~~~~~~~~
+
+One kernel thread per device (serves all registered virtqueues). The
+thread:
+
+1. Iterates all registered virtqueues, polls ``more_used()`` for each
+2. When active: busy-spins within a time budget (``busy_spin_us = 1ms``)
+ from ``last_io_ts``, using ``need_resched()`` + ``cpu_relax()``
+3. After spin budget expires: sets ``cq_need_wakeup`` on all VQs,
+ double-checks ``more_used()``, sleeps via ``schedule_timeout``
+4. Sleep timeout: fixed 1ms (``idle_timeout_us = 1000``)
+5. ``do_softirq()`` called after each completion callback (no hardware
+ interrupts means softirqs must be explicitly flushed)
+
+Parameters (configurable at thread creation):
+
+- ``idle_timeout_us``: sleep timeout in microseconds (default: 1000)
+- ``busy_spin_us``: spin budget in microseconds (default: 1000)
+
+
+Feature Negotiation
+-------------------
+
+1. The device offers ``VIRTIO_F_SQCQ_POLL`` in its feature bits
+2. The driver accepts the feature during ``finalize_features``
+3. The driver allocates SQ/CQ doorbells in ``find_vqs``
+4. The driver writes SQ/CQ DMA addresses to PCI config space
+5. The device (vhost) receives addresses and starts poll threads
+6. ``virtio_features_ok()`` MUST reject
+ ``VIRTIO_F_SQCQ_POLL | VIRTIO_F_RING_PACKED``
+
+If either side does not support the feature, traditional MMIO kick +
+MSI-X interrupt mode is used with zero overhead.
+
+
+Mutual Exclusion
+----------------
+
+``VIRTIO_F_SQCQ_POLL`` and ``VIRTIO_F_RING_PACKED`` MUST NOT be
+negotiated simultaneously. The driver's ``virtio_features_ok()``
+function rejects this combination.
+
+This restriction exists because the SQ/CQ doorbell implementation
+accesses split-ring-specific fields (``avail_idx_shadow``,
+``vring->used->idx``) unconditionally in ``virtqueue_notify()``.
+Packed ring support is a future enhancement.
+
+
+Relationship to Existing Mechanisms
+-----------------------------------
+
+- **EVENT_IDX** (``VIRTIO_RING_F_EVENT_IDX``): may be negotiated but
+ has no effect on polling virtqueues. Notification suppression is
+ handled by ``sq_need_wakeup`` / ``cq_need_wakeup`` instead.
+
+- **blk-mq iopoll**: complementary. iopoll handles synchronous
+ polling from the submitting task; SQ/CQ replaces the asynchronous
+ interrupt path.
+
+- **vhost-vDPA**: vDPA provides its own doorbell mmap mechanism.
+ SQ/CQ polling is specific to the vhost-scsi kernel backend and
+ does not require vDPA hardware support.
+
+
+Performance Characteristics
+---------------------------
+
+Benchmark results (fio, 4K random I/O, arm64 with 8 vCPUs):
+
+============================ ============ =========== =========
+Test Baseline SQ/CQ Poll Change
+============================ ============ =========== =========
+randread od1 nj1 22,427 IOPS 28,289 IOPS +26.1%
+randread od32 nj1 89,910 IOPS 75,665 IOPS -15.8%
+randread od32 nj4 186,763 IOPS 379,549 IOPS +103.2%
+randread od32 nj8 199,967 IOPS 550,633 IOPS +175.4%
+randwrite od1 nj1 21,912 IOPS 27,261 IOPS +24.4%
+randwrite od32 nj1 85,349 IOPS 81,389 IOPS -4.6%
+randwrite od32 nj4 190,443 IOPS 355,811 IOPS +86.8%
+randwrite od32 nj8 196,552 IOPS 566,640 IOPS +188.3%
+============================ ============ =========== =========
+
+Multi-queue workloads (nj4/nj8) see 87-188% improvement from
+eliminated VM exits on both submission and completion paths.
+Single-queue high-queue-depth workloads (od32 nj1) show a
+minor regression due to polling overhead vs. the VM exit savings.
diff --git a/build-riscv-kernel.sh b/build-riscv-kernel.sh
new file mode 100644
index 0000000000000..2ace46278369e
--- /dev/null
+++ b/build-riscv-kernel.sh
@@ -0,0 +1,26 @@
+#!/bin/bash
+
+# 设置变量
+export ARCH=riscv
+export CROSS_COMPILE=riscv64-linux-gnu-
+export KERNEL_DIR=$(pwd)
+export OUTPUT_DIR="${KERNEL_DIR}/output"
+
+# 清理
+make mrproper
+
+# 配置
+make defconfig
+# 或使用特定配置
+# cp ${CONFIG_FILE} .config
+
+# 编译内核
+echo "开始编译内核..."
+make -j$(nproc) || exit 1
+
+# 编译设备树
+echo "编译设备树..."
+make dtbs || exit 1
+
+echo "编译完成！"
+echo "内核镜像: ${KERNEL_DIR}/arch/riscv/boot/Image"
diff --git a/drivers/vhost/scsi.c b/drivers/vhost/scsi.c
index 1c22880e72267..4d65205fcb23e 100644
--- a/drivers/vhost/scsi.c
+++ b/drivers/vhost/scsi.c
@@ -142,6 +142,7 @@ struct vhost_scsi_cmd {
struct llist_node tvc_completion_list;
/* Used to track inflight cmd */
struct vhost_scsi_inflight *inflight;
+ ktime_t tvc_submit_ts;
};
struct vhost_scsi_nexus {
@@ -234,6 +235,7 @@ struct vhost_scsi_virtqueue {
struct vhost_work completion_work;
struct llist_head completion_list;
+ ktime_t stats_window;
};
struct vhost_scsi {
@@ -490,9 +492,18 @@ static void vhost_scsi_release_cmd(struct se_cmd *se_cmd)
struct vhost_scsi_virtqueue *svq = container_of(cmd->tvc_vq,
struct vhost_scsi_virtqueue, vq);
- llist_add(&cmd->tvc_completion_list, &svq->completion_list);
- if (!vhost_vq_work_queue(&svq->vq, &svq->completion_work))
- vhost_scsi_drop_cmds(svq);
+ llist_add(&cmd->tvc_completion_list,
&svq->completion_list);
+ if (READ_ONCE(svq->vq.sqcq_poll)) {
+ smp_wmb();
+ atomic_set(&svq->vq.completion_pending, 1);
+ smp_mb();
+ if (!READ_ONCE(svq->vq.poll_stop) &&
+ READ_ONCE(svq->vq.poll_task) &&
+ !task_is_running(svq->vq.poll_task))
+ wake_up(&svq->vq.poll_wait);
+ } else if (!vhost_vq_work_queue(&svq->vq,
&svq->completion_work)) {
+ vhost_scsi_drop_cmds(svq);
+ }
}
}
@@ -678,15 +689,11 @@ static int vhost_scsi_copy_sgl_to_iov(struct
vhost_scsi_cmd *cmd)
return 0;
}
-/* Fill in status and signal that we are done processing this command
- *
- * This is scheduled in the vhost work queue so we are called with the owner
- * process mm and can access the vring.
+/* Process completion_list: copy responses, add used entries, release cmds.
+ * Returns true if any completions were signalled.
*/
-static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
+static bool vhost_scsi_process_completions(struct vhost_scsi_virtqueue *svq)
{
- struct vhost_scsi_virtqueue *svq = container_of(work,
- struct vhost_scsi_virtqueue, completion_work);
struct virtio_scsi_cmd_resp v_rsp;
struct vhost_scsi_cmd *cmd, *t;
struct llist_node *llnode;
@@ -726,6 +733,9 @@ static void vhost_scsi_complete_cmd_work(struct vhost_work
*work)
signal = true;
vhost_add_used(cmd->tvc_vq, cmd->tvc_vq_desc, 0);
+ cmd->tvc_vq->lat_scsi_ns +=
+ ktime_to_ns(ktime_sub(ktime_get(),
cmd->tvc_submit_ts));
+ cmd->tvc_vq->stat_completions++;
} else
pr_err("Faulted on virtio_scsi_cmd_resp\n");
@@ -737,10 +747,70 @@ static void vhost_scsi_complete_cmd_work(struct
vhost_work *work)
mutex_unlock(&svq->vq.mutex);
+ return signal;
+}
+
+/* Worker-path completion (non-poll mode only) */
+static void vhost_scsi_complete_cmd_work(struct vhost_work *work)
+{
+ struct vhost_scsi_virtqueue *svq = container_of(work,
+ struct vhost_scsi_virtqueue, completion_work);
+ struct vhost_virtqueue *vq = &svq->vq;
+ bool signal;
+
+ signal = vhost_scsi_process_completions(svq);
+
if (signal)
vhost_signal(&svq->vs->dev, &svq->vq);
+
+ if (vq->stat_completions > 0) {
+ u64 now = ktime_get_ns();
+
+ if (!svq->stats_window)
+ svq->stats_window = ns_to_ktime(now);
+
+ if (now - ktime_to_ns(svq->stats_window) >= 10 * NSEC_PER_SEC) {
+ int vi;
+
+ for (vi = 0; vi < svq->vs->dev.nvqs; vi++)
+ if (svq->vs->dev.vqs[vi] == vq)
+ break;
+ pr_info("vhost-baseline: vq[%d] cmd=%llu
avg_lat=%lluns\n",
+ vi, vq->stat_completions,
+ div64_u64(vq->lat_scsi_ns,
vq->stat_completions));
+ svq->stats_window = ns_to_ktime(now);
+ vq->lat_scsi_ns = 0;
+ vq->stat_completions = 0;
+ }
+ }
}
+/* Poll-thread-path completion: write cq->idx and signal Guest directly */
+static void vhost_scsi_poll_complete(struct vhost_virtqueue *vq)
+{
+ struct vhost_scsi_virtqueue *svq = container_of(vq,
+ struct vhost_scsi_virtqueue, vq);
+ bool signal;
+
+ signal = vhost_scsi_process_completions(svq);
+
+ if (signal && svq->vq.cq) {
+ __virtio64 cq_idx_val;
+ __u8 cq_need_wakeup;
+
+ smp_mb();
+ cq_idx_val = (__force
__virtio64)READ_ONCE(svq->vq.last_used_idx);
+ put_user(cq_idx_val, &svq->vq.cq->idx);
+ smp_mb(); /* Ensure cq->idx is visible before reading
cq_need_wakeup */
+ if (!get_user(cq_need_wakeup, &svq->vq.cq->cq_need_wakeup)) {
+ smp_rmb();
+ if (cq_need_wakeup && svq->vq.call_ctx.ctx)
+ eventfd_signal(svq->vq.call_ctx.ctx);
+ }
+ }
+}
+
+
static struct vhost_scsi_cmd *
vhost_scsi_get_cmd(struct vhost_virtqueue *vq, u64 scsi_tag)
{
@@ -1060,9 +1130,25 @@ static void vhost_scsi_target_queue_cmd(struct
vhost_scsi_nexus *nexus,
}
se_cmd->tag = 0;
+ cmd->tvc_submit_ts = ktime_get();
target_init_cmd(se_cmd, nexus->tvn_se_sess, &cmd->tvc_sense_buf[0],
lun, exp_data_len, vhost_scsi_to_tcm_attr(task_attr),
- data_dir, TARGET_SCF_ACK_KREF);
+ data_dir,
+ TARGET_SCF_ACK_KREF |
+ (READ_ONCE(cmd->tvc_vq->sqcq_poll) ?
+ TARGET_SCF_USE_CPUID : 0));
+
+ /* Poll mode: steer TCM completion to next CPU to avoid
+ * starvation behind poll thread. Skip if fabric has
+ * custom cmd_compl_affinity (-2=UNBOUND or specific CPU).
+ */
+ if (READ_ONCE(cmd->tvc_vq->sqcq_poll) && num_online_cpus() > 1) {
+ struct se_wwn *wwn = se_cmd->se_sess->se_tpg->se_tpg_wwn;
+
+ if (!wwn || wwn->cmd_compl_affinity == SE_COMPL_AFFINITY_CPUID)
+ se_cmd->cpuid =
+
vhost_sqcq_pick_completion_cpu(smp_processor_id());
+ }
if (target_submit_prep(se_cmd, cdb, sg_ptr,
cmd->tvc_sgl_count, NULL, 0, sg_prot_ptr,
@@ -1346,7 +1432,6 @@ vhost_scsi_handle_vq(struct vhost_scsi *vs, struct
vhost_virtqueue *vq)
ret = vhost_scsi_chk_size(vq, &vc);
if (ret)
goto err;
-
ret = vhost_scsi_get_req(vq, &vc, &tpg);
if (ret)
goto err;
@@ -2070,6 +2155,15 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
mutex_lock(&vq->mutex);
vhost_vq_set_backend(vq, vs_tpg);
vhost_vq_init_access(vq);
+ if (i >= VHOST_SCSI_VQ_IO &&
+ vhost_has_feature(vq, VIRTIO_F_SQCQ_POLL) &&
+ vq->sq && vq->cq) {
+ ret = vhost_sqcq_poll_start(vq);
+ if (ret) {
+ mutex_unlock(&vq->mutex);
+ goto stop_poll_threads;
+ }
+ }
mutex_unlock(&vq->mutex);
}
ret = 0;
@@ -2086,6 +2180,19 @@ vhost_scsi_set_endpoint(struct vhost_scsi *vs,
vs->vs_tpg = vs_tpg;
goto out;
+stop_poll_threads:
+ /* VQ at index i failed to start. Stop poll threads for all
+ * previously started VQs (indices i-1 down to VHOST_SCSI_VQ_IO).
+ */
+ for (i = i - 1; i >= VHOST_SCSI_VQ_IO; i--) {
+ vq = &vs->vqs[i].vq;
+ mutex_lock(&vq->mutex);
+ vhost_sqcq_poll_stop(vq);
+ vhost_vq_set_backend(vq, NULL);
+ mutex_unlock(&vq->mutex);
+ }
+ /* Reset i for full vq cmds cleanup */
+ i = vs->dev.nvqs;
destroy_vq_cmds:
for (i--; i >= VHOST_SCSI_VQ_IO; i--) {
if (!vhost_vq_get_backend(&vs->vqs[i].vq))
@@ -2163,9 +2270,17 @@ vhost_scsi_clear_endpoint(struct vhost_scsi *vs,
/* Prevent new cmds from starting and accessing the tpgs/sessions */
for (i = 0; i < vs->dev.nvqs; i++) {
vq = &vs->vqs[i].vq;
- mutex_lock(&vq->mutex);
- vhost_vq_set_backend(vq, NULL);
- mutex_unlock(&vq->mutex);
+ if (!READ_ONCE(vq->sqcq_poll)) {
+ mutex_lock(&vq->mutex);
+ vhost_vq_set_backend(vq, NULL);
+ mutex_unlock(&vq->mutex);
+ } else {
+ mutex_lock(&vq->mutex);
+ vhost_sqcq_poll_stop(vq);
+ vhost_vq_set_backend(vq, NULL);
+ mutex_unlock(&vq->mutex);
+ vhost_sqcq_poll_flush(vq);
+ }
}
/* Make sure cmds are not running before tearing them down. */
vhost_scsi_flush(vs);
@@ -2221,6 +2336,7 @@ static int vhost_scsi_set_features(struct vhost_scsi *vs,
u64 features)
bool is_log, was_log;
int i;
+
if (features & ~VHOST_SCSI_FEATURES)
return -EOPNOTSUPP;
@@ -2321,6 +2437,7 @@ static int vhost_scsi_open(struct inode *inode, struct
file *f)
vhost_work_init(&svq->completion_work,
vhost_scsi_complete_cmd_work);
svq->vq.handle_kick = vhost_scsi_handle_kick;
+ svq->vq.poll_complete = vhost_scsi_poll_complete;
}
vhost_dev_init(&vs->dev, vqs, nvqs, UIO_MAXIOV,
VHOST_SCSI_WEIGHT, 0, true, NULL);
@@ -2415,6 +2532,45 @@ vhost_scsi_ioctl(struct file *f,
if (copy_from_user(&features, featurep, sizeof features))
return -EFAULT;
return vhost_scsi_set_features(vs, features);
+ case VHOST_GET_FEATURES_ARRAY: {
+ u64 count, copied;
+ u64 farr[VIRTIO_FEATURES_U64S] = {};
+
+ if (get_user(count, (u64 __user *)argp))
+ return -EFAULT;
+ copied = min_t(u64, count, VIRTIO_FEATURES_U64S);
+ farr[0] = VHOST_SCSI_FEATURES;
+ if (copy_to_user((u64 __user *)(argp + sizeof(u64)),
+ farr, copied * sizeof(u64)))
+ return -EFAULT;
+ if (count > copied &&
+ clear_user((u64 __user *)(argp + sizeof(u64) +
+ copied * sizeof(u64)),
+ (count - copied) * sizeof(u64)))
+ return -EFAULT;
+ return 0;
+ }
+ case VHOST_SET_FEATURES_ARRAY: {
+ u64 count, copied, all_features[VIRTIO_FEATURES_U64S] = {};
+ int i;
+
+ if (get_user(count, (u64 __user *)argp))
+ return -EFAULT;
+ copied = min_t(u64, count, VIRTIO_FEATURES_U64S);
+ if (copy_from_user(all_features,
+ (u64 __user *)(argp + sizeof(u64)),
+ copied * sizeof(u64)))
+ return -EFAULT;
+ for (i = copied; i < count; i++) {
+ u64 tmp;
+ if (get_user(tmp, (u64 __user *)(argp +
+ sizeof(u64) + i * sizeof(u64))))
+ return -EFAULT;
+ if (tmp)
+ return -EOPNOTSUPP;
+ }
+ return vhost_scsi_set_features(vs, all_features[0]);
+ }
case VHOST_NEW_WORKER:
case VHOST_FREE_WORKER:
case VHOST_ATTACH_VRING_WORKER:
@@ -2424,6 +2580,7 @@ vhost_scsi_ioctl(struct file *f,
mutex_unlock(&vs->dev.mutex);
return r;
default:
+
mutex_lock(&vs->dev.mutex);
r = vhost_dev_ioctl(&vs->dev, ioctl, argp);
/* TODO: flush backend after dev ioctl. */
diff --git a/drivers/vhost/vhost.c b/drivers/vhost/vhost.c
index 2f2c45d208832..68fb53c40b2e6 100644
--- a/drivers/vhost/vhost.c
+++ b/drivers/vhost/vhost.c
@@ -42,6 +42,20 @@ static int max_iotlb_entries = 2048;
module_param(max_iotlb_entries, int, 0444);
MODULE_PARM_DESC(max_iotlb_entries,
"Maximum number of iotlb entries. (default: 2048)");
+#define SQCQ_IDLE_TIMEOUT_US_MAX 1000U
+#define SQCQ_SLEEP_EMA_MULT 2
+#define SQCQ_SPIN_EMA_MULT 2
+#define SQCQ_ACTIVE_TIMEOUT_NS 1000000U /* cap active-VQ clear
at 1 ms */
+#define SQCQ_STALE_TIMEOUT_NS 1000000000ULL /* backstop EMA/active
clear at 1 s */
+
+static atomic_t sqcq_poll_cpu_rr = ATOMIC_INIT(-1);
+static atomic_t sqcq_active_io_vqs = ATOMIC_INIT(0);
+/* CPUs with a BUSY poll thread (actively processing cmds); TCM completion
+ * steers away. Idle (sleeping) poll threads don't occupy their CPU, so
+ * their bit is cleared via the sq_was_active transitions in the poll loop.
+ */
+static cpumask_t sqcq_poll_active_cpu_mask;
+
static bool fork_from_owner_default = VHOST_FORK_OWNER_TASK;
#ifdef CONFIG_VHOST_ENABLE_FORK_OWNER_CONTROL
@@ -390,6 +404,20 @@ static void vhost_vq_reset(struct vhost_dev *dev,
vhost_disable_cross_endian(vq);
vhost_reset_is_le(vq);
vq->busyloop_timeout = 0;
+ WRITE_ONCE(vq->sqcq_poll, false);
+ vq->sq = NULL;
+ vq->cq = NULL;
+ vq->poll_task = NULL;
+ WRITE_ONCE(vq->poll_stop, false);
+ init_waitqueue_head(&vq->poll_wait);
+ init_completion(&vq->poll_started);
+ init_completion(&vq->poll_stopped);
+ WRITE_ONCE(vq->kick_received, false);
+ atomic_set(&vq->completion_pending, 0);
+ vq->kick_wqh = NULL;
+ vq->stat_completions = 0;
+ vq->lat_scsi_ns = 0;
+ vq->poll_cpu = -1;
vq->umem = NULL;
vq->iotlb = NULL;
rcu_assign_pointer(vq->worker, NULL);
@@ -1168,6 +1196,15 @@ void vhost_dev_stop(struct vhost_dev *dev)
{
int i;
+ for (i = 0; i < dev->nvqs; ++i) {
+ if (!READ_ONCE(dev->vqs[i]->sqcq_poll))
+ continue;
+ mutex_lock(&dev->vqs[i]->mutex);
+ vhost_sqcq_poll_stop(dev->vqs[i]);
+ mutex_unlock(&dev->vqs[i]->mutex);
+ vhost_sqcq_poll_flush(dev->vqs[i]);
+ }
+
for (i = 0; i < dev->nvqs; ++i) {
if (dev->vqs[i]->kick && dev->vqs[i]->handle_kick)
vhost_poll_stop(&dev->vqs[i]->poll);
@@ -2051,6 +2088,535 @@ static long vhost_vring_set_num(struct vhost_dev *d,
return 0;
}
+static int vhost_sqcq_kick_wakeup(wait_queue_entry_t *wait, unsigned int mode,
+ int sync, void *key)
+{
+ struct vhost_virtqueue *vq = container_of(wait,
+ struct vhost_virtqueue, kick_wait);
+
+ if (!(key_to_poll(key) & EPOLLIN))
+ return 0;
+
+ WRITE_ONCE(vq->kick_received, true);
+ wake_up(&vq->poll_wait);
+ return 0;
+}
+
+static void vhost_sqcq_kick_ptable_func(struct file *file,
+ wait_queue_head_t *wqh,
+ poll_table *pt)
+{
+ struct vhost_virtqueue *vq = container_of(pt,
+ struct vhost_virtqueue, kick_pt);
+
+ vq->kick_wqh = wqh;
+ add_wait_queue(wqh, &vq->kick_wait);
+}
+
+static void vhost_sqcq_kick_register(struct vhost_virtqueue *vq)
+{
+ init_waitqueue_func_entry(&vq->kick_wait, vhost_sqcq_kick_wakeup);
+ vq->kick_wqh = NULL;
+
+ if (vq->kick) {
+ __poll_t mask;
+
+ init_poll_funcptr(&vq->kick_pt, vhost_sqcq_kick_ptable_func);
+ mask = vfs_poll(vq->kick, &vq->kick_pt);
+ /* eventfd counter is never consumed; verify stale EPOLLIN
+ * against sq->idx before acting; real kicks wake via
+ * the registered wait queue.
+ */
+ if (mask & EPOLLIN) {
+ __virtio64 sq_val;
+
+ if (vq->sq && !get_user(sq_val, &vq->sq->idx)) {
+ smp_rmb();
+ if ((__force u16)sq_val != vq->last_avail_idx)
+ vhost_sqcq_kick_wakeup(
+ &vq->kick_wait, 0, 0,
+ poll_to_key(mask));
+ }
+ }
+ }
+}
+
+static void vhost_sqcq_kick_unregister(struct vhost_virtqueue *vq)
+{
+ if (vq->kick_wqh) {
+ remove_wait_queue(vq->kick_wqh, &vq->kick_wait);
+ vq->kick_wqh = NULL;
+ }
+ WRITE_ONCE(vq->kick_received, false);
+}
+
+static int vhost_sqcq_poll_thread(void *data)
+{
+ struct vhost_virtqueue *vq = data;
+ struct vhost_dev *dev = vq->dev;
+ u16 new_sq_idx;
+ __virtio64 sq_idx_val;
+ __u8 sq_need_wakeup_val = 0;
+ ktime_t ts2;
+ ktime_t last_io_ts = 0;
+ ktime_t stats_window = ktime_get();
+ u64 stats_cq = 0;
+ u64 stats_lat = 0;
+ u64 stats_inflight_sum = 0;
+ u64 stats_inflight_n = 0;
+ u64 stats_multi_cnt = 0;
+ u64 stats_sleep_cnt = 0;
+ u64 stats_wake_kick = 0;
+ u64 stats_wake_complete = 0;
+ u64 stats_wake_timeout = 0;
+ u64 ema_ns_val;
+ u64 spin_ns_val;
+ unsigned long _timeout_jiffies;
+ u64 _timeout_us;
+
+ kthread_use_mm(dev->mm);
+ complete(&vq->poll_started);
+
+ vq->stat_completions = 0;
+ vq->lat_scsi_ns = 0;
+ vq->ema_latency_ns = 0;
+ vq->ema_interval_ns = 0;
+ ts2 = ktime_get();
+
+ while (!READ_ONCE(vq->poll_stop)) {
+
+ stats_inflight_sum += (u16)(vq->last_avail_idx -
vq->last_used_idx);
+ stats_inflight_n++;
+ stats_multi_cnt += ((atomic_read(&sqcq_active_io_vqs) > 1) ? 1
: 0);
+
+ // 10's output windows.
+ if (ktime_to_ns(ktime_sub(ktime_get(), stats_window)) >=
+ (s64)10 * NSEC_PER_SEC) {
+ if (stats_cq > 0) {
+ int vi;
+
+ for (vi = 0; vi < dev->nvqs; vi++)
+ if (dev->vqs[vi] == vq)
+ break;
+ pr_info("vhost-sqcq: vq[%d] cq=%llu avg=%lluns
ema_lat=%lluns interval=%lluns avg_inflight=%llu multi=%llu%% sleep=%llu
wk_kick=%llu%% wk_cmp=%llu%% wk_to=%llu%%\n",
+ vi, stats_cq,
+ div64_u64(stats_lat, stats_cq),
+ vq->ema_latency_ns,
+ vq->ema_interval_ns,
+ stats_inflight_n > 0 ?
+ div64_u64(stats_inflight_sum,
stats_inflight_n) : 0,
+ div64_u64(stats_multi_cnt*100,
stats_inflight_n),
+ stats_sleep_cnt,
+ stats_sleep_cnt > 0 ?
+ div64_u64(stats_wake_kick*100,
stats_sleep_cnt) : 0,
+ stats_sleep_cnt > 0 ?
+ div64_u64(stats_wake_complete*100,
stats_sleep_cnt) : 0,
+ stats_sleep_cnt > 0 ?
+ div64_u64(stats_wake_timeout*100,
stats_sleep_cnt) : 0);
+ }
+ stats_window = ktime_get();
+ stats_cq = 0;
+ stats_lat = 0;
+ stats_inflight_sum = 0;
+ stats_inflight_n = 0;
+ stats_multi_cnt = 0;
+ stats_sleep_cnt = 0;
+ stats_wake_kick = 0;
+ stats_wake_complete = 0;
+ stats_wake_timeout = 0;
+ }
+
+ /* Backstop cleanup: drop sq_was_active and reset EMAs once a VQ
+ * has been idle (no submission) for a fixed interval.
+ */
+ if (vq->sq_was_active && last_io_ts > 0 &&
+ ktime_to_ns(ktime_sub(ktime_get(), last_io_ts)) >
+ (s64)SQCQ_STALE_TIMEOUT_NS) {
+ vq->sq_was_active = false;
+ atomic_dec(&sqcq_active_io_vqs);
+ if (vq->poll_cpu >= 0)
+ cpumask_clear_cpu(vq->poll_cpu,
&sqcq_poll_active_cpu_mask);
+ WRITE_ONCE(vq->ema_interval_ns, 0);
+ WRITE_ONCE(vq->ema_latency_ns, 0);
+ }
+
+ /* complete*/
+ if (atomic_read(&vq->completion_pending) && vq->poll_complete) {
+ smp_rmb();
+ atomic_set(&vq->completion_pending, 0);
+ vq->poll_complete(vq);
+ if (vq->stat_completions > 0) {
+ u64 avg = div64_u64(vq->lat_scsi_ns,
+ vq->stat_completions);
+
+ if (vq->ema_latency_ns == 0)
+ vq->ema_latency_ns = avg;
+ else
+ vq->ema_latency_ns =
+ vq->ema_latency_ns -
+ vq->ema_latency_ns / 8 +
+ avg / 8;
+ stats_cq += vq->stat_completions;
+ stats_lat += vq->lat_scsi_ns;
+ vq->lat_scsi_ns = 0;
+ vq->stat_completions = 0;
+ }
+ }
+
+ if (get_user(sq_idx_val, &vq->sq->idx)) {
+ int fault_retries = 3;
+
+ while (fault_retries-- > 0) {
+ if (!get_user(sq_idx_val, &vq->sq->idx))
+ break;
+ schedule_timeout_idle(1);
+ }
+ if (fault_retries < 0)
+ schedule_timeout_idle(HZ / 100);
+ continue;
+ }
+ /* Acquire: see all prior Guest writes (descriptors). */
+ smp_rmb();
+ new_sq_idx = (__force u16)sq_idx_val;
+
+ if (new_sq_idx != vq->last_avail_idx) {
+ vq->handle_kick(&vq->poll.work);
+ ts2 = ktime_get();
+ if (last_io_ts > 0) {
+ u64 _interval = ktime_to_ns(ktime_sub(ts2,
last_io_ts));
+ if (vq->ema_interval_ns == 0)
+ vq->ema_interval_ns = _interval;
+ else
+ vq->ema_interval_ns =
+ vq->ema_interval_ns -
+ vq->ema_interval_ns / 8 +
+ _interval / 8;
+ }
+ last_io_ts = ts2;
+ if (!vq->sq_was_active) {
+ vq->sq_was_active = true;
+ atomic_inc(&sqcq_active_io_vqs);
+ if (vq->poll_cpu >= 0)
+ cpumask_set_cpu(vq->poll_cpu,
&sqcq_poll_active_cpu_mask);
+ }
+ sq_need_wakeup_val = 0;
+ smp_mb();
+ put_user(sq_need_wakeup_val, &vq->sq->sq_need_wakeup);
+ }
+
+
+ if (vq->last_avail_idx == vq->last_used_idx) {
+ /* Spin for a fixed window to catch next submission via
+ * sq->idx polling; need_resched+cond_resched prevents
+ * starving other tasks on this CPU.
+ */
+ spin_ns_val = SQCQ_ACTIVE_TIMEOUT_NS;
+ if (last_io_ts > 0 && ktime_to_ns(ktime_sub(
+ ktime_get(), last_io_ts)) < spin_ns_val) {
+ if (need_resched())
+ cond_resched();
+ cpu_relax();
+ continue;
+ }
+ /* Clear sq_was_active to keep active_io_vqs and
+ * cpu mask fresh. Has-inflight path keeps the
+ * flag for TCM completion steering.
+ */
+ if (vq->sq_was_active) {
+ vq->sq_was_active = false;
+ atomic_dec(&sqcq_active_io_vqs);
+ if (vq->poll_cpu >= 0)
+ cpumask_clear_cpu(vq->poll_cpu,
&sqcq_poll_active_cpu_mask);
+ }
+ } else {
+ /* Spin within SQCQ_SPIN_EMA_MULT * ema_latency (with
+ * cooperative yield), then sleep. cond_resched handles
+ * multi-VQ fairness without a separate yield heuristic.
+ */
+ ema_ns_val = READ_ONCE(vq->ema_latency_ns);
+ spin_ns_val = ema_ns_val > 0 ?
+ (u64)SQCQ_SPIN_EMA_MULT * ema_ns_val :
+ 2 * NSEC_PER_USEC; /* fallback when EMA
unknown */
+
+ if (ktime_to_ns(ktime_sub(
+ ktime_get(), last_io_ts)) < spin_ns_val) {
+ if (need_resched())
+ cond_resched();
+ cpu_relax();
+ continue;
+ }
+ }
+
+ ema_ns_val = READ_ONCE(vq->ema_latency_ns);
+
+ if (ema_ns_val == 0) {
+ _timeout_jiffies = usecs_to_jiffies(
+ SQCQ_IDLE_TIMEOUT_US_MAX);
+ } else {
+ _timeout_us = div64_u64(
+ ema_ns_val,
+ (u64)NSEC_PER_USEC / SQCQ_SLEEP_EMA_MULT);
+
+ _timeout_us = clamp(_timeout_us,
+ 10ULL,
+ (u64)SQCQ_IDLE_TIMEOUT_US_MAX);
+ _timeout_jiffies = usecs_to_jiffies(
+ _timeout_us);
+ if (_timeout_jiffies == 0)
+ _timeout_jiffies = 1;
+ }
+
+ /* Set NEED_WAKEUP, then double-check sq->idx before sleeping.
*/
+ smp_wmb();
+ sq_need_wakeup_val = 1;
+ put_user(sq_need_wakeup_val,
+ &vq->sq->sq_need_wakeup);
+ smp_mb();
+
+ if (get_user(sq_idx_val, &vq->sq->idx)) {
+ int fault_retries = 3;
+
+ while (fault_retries-- > 0) {
+ if (!get_user(sq_idx_val, &vq->sq->idx))
+ break;
+ schedule_timeout_idle(1);
+ }
+ if (fault_retries < 0)
+ schedule_timeout_idle(HZ / 100);
+ continue;
+ }
+ smp_rmb();
+ new_sq_idx = (__force u16)sq_idx_val;
+
+ if (new_sq_idx != vq->last_avail_idx) {
+ sq_need_wakeup_val = 0;
+ smp_mb();
+ put_user(sq_need_wakeup_val,
+ &vq->sq->sq_need_wakeup);
+ continue;
+ }
+
+ ++stats_sleep_cnt;
+ vhost_sqcq_kick_register(vq);
+ wait_event_interruptible_timeout(
+ vq->poll_wait,
+ READ_ONCE(vq->poll_stop) ||
+ READ_ONCE(vq->kick_received) ||
+ atomic_read(&vq->completion_pending),
+ _timeout_jiffies);
+ /* Wake reason stats: snapshot before unregister clears
kick_received */
+ {
+ bool _kicked = READ_ONCE(vq->kick_received);
+ bool _completed = atomic_read(&vq->completion_pending);
+
+ if (_kicked)
+ stats_wake_kick++;
+ if (_completed)
+ stats_wake_complete++;
+ if (!_kicked && !_completed)
+ stats_wake_timeout++;
+ }
+ vhost_sqcq_kick_unregister(vq);
+
+ sq_need_wakeup_val = 0;
+ put_user(sq_need_wakeup_val,
+ &vq->sq->sq_need_wakeup);
+ }
+
+ complete(&vq->poll_stopped);
+ kthread_unuse_mm(dev->mm);
+ return 0;
+}
+
+int vhost_sqcq_poll_start(struct vhost_virtqueue *vq)
+{
+ if (!vq->sq || !vq->cq || !vq->handle_kick)
+ return -EINVAL;
+
+ vhost_poll_stop(&vq->poll);
+
+ WRITE_ONCE(vq->poll_stop, false);
+ reinit_completion(&vq->poll_started);
+ reinit_completion(&vq->poll_stopped);
+
+ int vi;
+
+ for (vi = 0; vi < vq->dev->nvqs; vi++)
+ if (vq->dev->vqs[vi] == vq)
+ break;
+ vq->poll_task = kthread_run(vhost_sqcq_poll_thread, vq,
+ "vhost-sqcq-%d-%d",
+ task_pid_nr(current),
+ vi);
+ if (IS_ERR(vq->poll_task)) {
+ int ret = PTR_ERR(vq->poll_task);
+
+ vq->poll_task = NULL;
+ if (vq->kick)
+ vhost_poll_start(&vq->poll, vq->kick);
+ return ret;
+ }
+
+ wait_for_completion(&vq->poll_started);
+
+ /* Enable sqcq mode only after poll_task is valid and the
+ * thread has signaled readiness. Before this point,
+ * vhost_signal() routes completions via eventfd (old path).
+ */
+ smp_mb();
+ WRITE_ONCE(vq->sqcq_poll, true);
+
+ if (vq->poll_task) {
+ int cpu;
+ int start = atomic_inc_return(&sqcq_poll_cpu_rr);
+ int target_cpu = -1;
+ int online = num_online_cpus();
+
+ if (online > 0)
+ start = start % online;
+ else
+ start = 0;
+
+ for_each_online_cpu(cpu) {
+ if (start-- <= 0) {
+ target_cpu = cpu;
+ break;
+ }
+ }
+ if (target_cpu < 0) {
+ for_each_online_cpu(cpu) {
+ target_cpu = cpu;
+ break;
+ }
+ }
+
+ if (target_cpu >= 0 && target_cpu < nr_cpu_ids &&
+ cpu_online(target_cpu)) {
+ int vi;
+
+ set_cpus_allowed_ptr(vq->poll_task,
cpumask_of(target_cpu));
+ for (vi = 0; vi < vq->dev->nvqs; vi++)
+ if (vq->dev->vqs[vi] == vq)
+ break;
+ pr_info("vhost-sqcq: vq[%d] poll thread bound to
cpu%d\n",
+ vi, target_cpu);
+ vq->poll_cpu = target_cpu;
+ }
+ }
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(vhost_sqcq_poll_start);
+
+void vhost_sqcq_poll_stop(struct vhost_virtqueue *vq)
+{
+ if (!vq->poll_task)
+ return;
+
+ WRITE_ONCE(vq->poll_stop, true);
+ smp_mb();
+ wake_up_process(vq->poll_task);
+ wake_up(&vq->poll_wait);
+ wait_for_completion(&vq->poll_stopped);
+ vq->poll_task = NULL;
+ WRITE_ONCE(vq->sqcq_poll, false);
+ if (vq->poll_cpu >= 0) {
+ cpumask_clear_cpu(vq->poll_cpu, &sqcq_poll_active_cpu_mask);
+ vq->poll_cpu = -1;
+ }
+ smp_mb();
+
+ if (vq->kick)
+ vhost_poll_start(&vq->poll, vq->kick);
+ vq->sq = NULL;
+ vq->cq = NULL;
+}
+EXPORT_SYMBOL_GPL(vhost_sqcq_poll_stop);
+
+/* Flush residual completions after stop. Must be called outside vq->mutex. */
+void vhost_sqcq_poll_flush(struct vhost_virtqueue *vq)
+{
+ if (WARN_ON_ONCE(!vq->poll_complete))
+ return;
+
+ atomic_set(&vq->completion_pending, 0);
+ vq->poll_complete(vq);
+}
+EXPORT_SYMBOL_GPL(vhost_sqcq_poll_flush);
+
+/* Pick a CPU for TCM completion: prefer a free CPU (no poll thread),
+ * searching from hint_cpu+1 with wrap. Fallback to next-online
+ * if every CPU has a poll thread.
+ */
+int vhost_sqcq_pick_completion_cpu(int hint_cpu)
+{
+ int first, cpu, target = -1;
+
+ first = cpumask_next(hint_cpu, cpu_online_mask);
+ if (first >= nr_cpu_ids)
+ first = cpumask_first(cpu_online_mask);
+
+ cpu = first;
+ do {
+ if (!cpumask_test_cpu(cpu, &sqcq_poll_active_cpu_mask)) {
+ target = cpu;
+ break;
+ }
+ cpu = cpumask_next(cpu, cpu_online_mask);
+ if (cpu >= nr_cpu_ids)
+ cpu = cpumask_first(cpu_online_mask);
+ } while (cpu != first);
+
+ if (target < 0)
+ target = first;
+
+ return target;
+}
+EXPORT_SYMBOL_GPL(vhost_sqcq_pick_completion_cpu);
+
+static long vhost_vring_set_sqcq_addr(struct vhost_dev *d,
+ struct vhost_virtqueue *vq,
+ void __user *argp)
+{
+ struct vhost_vring_sqcq_addr a;
+
+ if (copy_from_user(&a, argp, sizeof(a)))
+ return -EFAULT;
+
+ if ((u64)(unsigned long)a.sq_user_addr != a.sq_user_addr ||
+ (u64)(unsigned long)a.cq_user_addr != a.cq_user_addr)
+ return -EFAULT;
+
+ if (a.sq_user_addr & 7 || a.cq_user_addr & 7)
+ return -EINVAL;
+
+ if (vq->private_data) {
+ if (!access_ok((void __user *)(unsigned long)a.sq_user_addr,
+ sizeof(struct vring_sq)) ||
+ !access_ok((void __user *)(unsigned long)a.cq_user_addr,
+ sizeof(struct vring_cq)))
+ return -EINVAL;
+ }
+
+ vq->sq = (struct vring_sq __user *)(unsigned long)a.sq_user_addr;
+ vq->cq = (struct vring_cq __user *)(unsigned long)a.cq_user_addr;
+
+ /* SET_ENDPOINT may run before SET_FEATURES/SQCQ_ADDR, so poll-start
+ * can fail there; this is the canonical trigger. Skip VQs without
+ * poll_complete (ctrl/event VQs) — they stay interrupt-driven.
+ */
+ if (vq->private_data && vq->sq && vq->cq &&
+ !READ_ONCE(vq->sqcq_poll) &&
+ vq->poll_complete &&
+ vhost_has_feature(vq, VIRTIO_F_SQCQ_POLL)) {
+ long ret = vhost_sqcq_poll_start(vq);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
static long vhost_vring_set_addr(struct vhost_dev *d,
struct vhost_virtqueue *vq,
void __user *argp)
@@ -2244,6 +2810,9 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int
ioctl, void __user *arg
if (copy_to_user(argp, &s, sizeof(s)))
r = -EFAULT;
break;
+ case VHOST_SET_VRING_SQCQ_ADDR:
+ r = vhost_vring_set_sqcq_addr(d, vq, argp);
+ break;
default:
r = -ENOIOCTLCMD;
}
@@ -2256,7 +2825,7 @@ long vhost_vring_ioctl(struct vhost_dev *d, unsigned int
ioctl, void __user *arg
if (filep)
fput(filep);
- if (pollstart && vq->handle_kick)
+ if (pollstart && vq->handle_kick && !READ_ONCE(vq->sqcq_poll))
r = vhost_poll_start(&vq->poll, vq->kick);
mutex_unlock(&vq->mutex);
@@ -2938,8 +3507,12 @@ int vhost_get_vq_desc_n(struct vhost_virtqueue *vq,
*ndesc = c;
/* Assume notifications from guest are disabled at this point,
- * if they aren't we would need to update avail_event index. */
- BUG_ON(!(vq->used_flags & VRING_USED_F_NO_NOTIFY));
+ * if they aren't we would need to update avail_event index.
+ * In SQCQ poll mode, vhost_disable_notify is a no-op so
+ * VRING_USED_F_NO_NOTIFY is never set; skip the check.
+ */
+ WARN_ON_ONCE(!vq->sqcq_poll &&
+ !(vq->used_flags & VRING_USED_F_NO_NOTIFY));
return head;
}
EXPORT_SYMBOL_GPL(vhost_get_vq_desc_n);
@@ -3167,6 +3740,16 @@ static bool vhost_notify(struct vhost_dev *dev, struct
vhost_virtqueue *vq)
/* This actually signals the guest, using eventfd. */
void vhost_signal(struct vhost_dev *dev, struct vhost_virtqueue *vq)
{
+ if (READ_ONCE(vq->sqcq_poll) && vq->cq) {
+ atomic_set(&vq->completion_pending, 1);
+ if (!READ_ONCE(vq->poll_stop)) {
+ smp_mb();
+ if (!task_is_running(vq->poll_task))
+ wake_up(&vq->poll_wait);
+ }
+ return;
+ }
+
/* Signal the Guest tell them we used something up. */
if (vq->call_ctx.ctx && vhost_notify(dev, vq))
eventfd_signal(vq->call_ctx.ctx);
@@ -3213,6 +3796,9 @@ EXPORT_SYMBOL_GPL(vhost_vq_avail_empty);
/* OK, now we need to know about added descriptors. */
bool vhost_enable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
{
+ if (READ_ONCE(vq->sqcq_poll))
+ return false;
+
int r;
if (!(vq->used_flags & VRING_USED_F_NO_NOTIFY))
@@ -3249,6 +3835,9 @@ EXPORT_SYMBOL_GPL(vhost_enable_notify);
/* We don't need to be notified again. */
void vhost_disable_notify(struct vhost_dev *dev, struct vhost_virtqueue *vq)
{
+ if (READ_ONCE(vq->sqcq_poll))
+ return;
+
int r;
if (vq->used_flags & VRING_USED_F_NO_NOTIFY)
diff --git a/drivers/vhost/vhost.h b/drivers/vhost/vhost.h
index 4fe99765c5c73..b9c15961f5a18 100644
--- a/drivers/vhost/vhost.h
+++ b/drivers/vhost/vhost.h
@@ -111,6 +111,7 @@ struct vhost_virtqueue {
/* The routine to call when the Guest pings us, or timeout. */
vhost_work_fn_t handle_kick;
+ void (*poll_complete)(struct vhost_virtqueue *vq);
/* Last available index we saw.
* Values are limited to 0x7fff, and the high bit is used as
@@ -164,6 +165,27 @@ struct vhost_virtqueue {
bool user_be;
#endif
u32 busyloop_timeout;
+
+ /* SQ/CQ polling mode */
+ bool sqcq_poll;
+ struct vring_sq __user *sq;
+ struct vring_cq __user *cq;
+ struct task_struct *poll_task;
+ bool poll_stop;
+ wait_queue_head_t poll_wait;
+ struct completion poll_started;
+ struct completion poll_stopped;
+ bool kick_received; /* Set when kick eventfd wakes us */
+ atomic_t completion_pending; /* Set by vhost_signal, poll thread
atomically clears */
+ wait_queue_head_t *kick_wqh; /* kick eventfd wait queue head */
+ wait_queue_entry_t kick_wait; /* Wait entry on kick eventfd wq */
+ poll_table kick_pt; /* poll_table for kick eventfd */
+ u64 stat_completions;
+ u64 lat_scsi_ns;
+ u64 ema_interval_ns;
+ u64 ema_latency_ns;
+ bool sq_was_active;
+ int poll_cpu; /* CPU this VQ's poll thread is pinned
to, -1 = unpinned */
};
struct vhost_msg_node {
@@ -256,6 +278,10 @@ void vhost_add_used_and_signal_n(struct vhost_dev *,
struct vhost_virtqueue *,
void vhost_signal(struct vhost_dev *, struct vhost_virtqueue *);
void vhost_disable_notify(struct vhost_dev *, struct vhost_virtqueue *);
bool vhost_vq_avail_empty(struct vhost_dev *, struct vhost_virtqueue *);
+int vhost_sqcq_poll_start(struct vhost_virtqueue *vq);
+void vhost_sqcq_poll_stop(struct vhost_virtqueue *vq);
+void vhost_sqcq_poll_flush(struct vhost_virtqueue *vq);
+int vhost_sqcq_pick_completion_cpu(int hint_cpu);
bool vhost_enable_notify(struct vhost_dev *, struct vhost_virtqueue *);
int vhost_log_write(struct vhost_virtqueue *vq, struct vhost_log *log,
@@ -294,7 +320,8 @@ void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
VIRTIO_RING_F_EVENT_IDX, \
VHOST_F_LOG_ALL, \
VIRTIO_F_ANY_LAYOUT, \
- VIRTIO_F_VERSION_1
+ VIRTIO_F_VERSION_1, \
+ VIRTIO_F_SQCQ_POLL
static inline u64 vhost_features_u64(const int *features, int size, int idx)
{
diff --git a/include/uapi/linux/vhost.h b/include/uapi/linux/vhost.h
index c57674a6aa0db..dacb0e1fdc4bc 100644
--- a/include/uapi/linux/vhost.h
+++ b/include/uapi/linux/vhost.h
@@ -123,6 +123,10 @@
#define VHOST_SET_BACKEND_FEATURES _IOW(VHOST_VIRTIO, 0x25, __u64)
#define VHOST_GET_BACKEND_FEATURES _IOR(VHOST_VIRTIO, 0x26, __u64)
+/* Set SQ/CQ doorbell addresses for polling mode */
+#define VHOST_SET_VRING_SQCQ_ADDR _IOW(VHOST_VIRTIO, 0x27, \
+ struct vhost_vring_sqcq_addr)
+
/* VHOST_NET specific defines */
/* Attach virtio net ring to a raw socket, or tap device.
diff --git a/include/uapi/linux/vhost_types.h b/include/uapi/linux/vhost_types.h
index 1c39cc5f5a31b..6514c1b5be20b 100644
--- a/include/uapi/linux/vhost_types.h
+++ b/include/uapi/linux/vhost_types.h
@@ -47,6 +47,12 @@ struct vhost_vring_addr {
__u64 log_guest_addr;
};
+struct vhost_vring_sqcq_addr {
+ unsigned int index;
+ __u64 sq_user_addr; /* SQ doorbell HVA */
+ __u64 cq_user_addr; /* CQ doorbell HVA */
+};
+
struct vhost_worker_state {
/*
* For VHOST_NEW_WORKER the kernel will return the new vhost_worker id.
--
2.34.1
