From: Patrick Fu <patrick...@intel.com> Vhost async enqueue assumes that all async copies should break at packet boundary. i.e. if a packet is splited into multiple copy segments, the async engine should always report copy completion when entire packet is finished. This patch removes the assumption.
Fixes: cd6760da1076 ("vhost: introduce async enqueue for split ring") Signed-off-by: Patrick Fu <patrick...@intel.com> --- v2: - fix an issue that can stuck async poll when packets buffer is full - rename a local variable to better reflect its usage lib/librte_vhost/vhost.h | 3 +++ lib/librte_vhost/virtio_net.c | 17 ++++++++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/librte_vhost/vhost.h b/lib/librte_vhost/vhost.h index 8c01cee42..0f7212f88 100644 --- a/lib/librte_vhost/vhost.h +++ b/lib/librte_vhost/vhost.h @@ -46,6 +46,8 @@ #define MAX_PKT_BURST 32 +#define ASYNC_MAX_POLL_SEG 255 + #define VHOST_MAX_ASYNC_IT (MAX_PKT_BURST * 2) #define VHOST_MAX_ASYNC_VEC (BUF_VECTOR_MAX * 2) @@ -225,6 +227,7 @@ struct vhost_virtqueue { uint64_t *async_pending_info; uint16_t async_pkts_idx; uint16_t async_pkts_inflight_n; + uint16_t async_last_seg_n; /* vq async features */ bool async_inorder; diff --git a/lib/librte_vhost/virtio_net.c b/lib/librte_vhost/virtio_net.c index 1d0be3dd4..17808ab29 100644 --- a/lib/librte_vhost/virtio_net.c +++ b/lib/librte_vhost/virtio_net.c @@ -1633,6 +1633,7 @@ uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id, struct vhost_virtqueue *vq; uint16_t n_pkts_cpl, n_pkts_put = 0, n_descs = 0; uint16_t start_idx, pkts_idx, vq_size; + uint16_t n_inflight; uint64_t *async_pending_info; VHOST_LOG_DATA(DEBUG, "(%d) %s\n", dev->vid, __func__); @@ -1646,28 +1647,32 @@ uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id, rte_spinlock_lock(&vq->access_lock); + n_inflight = vq->async_pkts_inflight_n; pkts_idx = vq->async_pkts_idx; async_pending_info = vq->async_pending_info; vq_size = vq->size; start_idx = virtio_dev_rx_async_get_info_idx(pkts_idx, vq_size, vq->async_pkts_inflight_n); - n_pkts_cpl = - vq->async_ops.check_completed_copies(vid, queue_id, 0, count); + n_pkts_cpl = vq->async_ops.check_completed_copies(vid, queue_id, + 0, ASYNC_MAX_POLL_SEG - vq->async_last_seg_n) + + vq->async_last_seg_n; rte_smp_wmb(); - while (likely(((start_idx + n_pkts_put) & (vq_size - 1)) != pkts_idx)) { + while (likely((n_pkts_put < count) && n_inflight)) { uint64_t info = async_pending_info[ (start_idx + n_pkts_put) & (vq_size - 1)]; uint64_t n_segs; n_pkts_put++; + n_inflight--; n_descs += info & ASYNC_PENDING_INFO_N_MSK; n_segs = info >> ASYNC_PENDING_INFO_N_SFT; if (n_segs) { - if (!n_pkts_cpl || n_pkts_cpl < n_segs) { + if (unlikely(n_pkts_cpl < n_segs)) { n_pkts_put--; + n_inflight++; n_descs -= info & ASYNC_PENDING_INFO_N_MSK; if (n_pkts_cpl) { async_pending_info[ @@ -1684,8 +1689,10 @@ uint16_t rte_vhost_poll_enqueue_completed(int vid, uint16_t queue_id, } } + vq->async_last_seg_n = n_pkts_cpl; + if (n_pkts_put) { - vq->async_pkts_inflight_n -= n_pkts_put; + vq->async_pkts_inflight_n = n_inflight; __atomic_add_fetch(&vq->used->idx, n_descs, __ATOMIC_RELEASE); vhost_vring_call_split(dev, vq); -- 2.18.4