Add the logic to process vDPA notify relay.
Signed-off-by: Chaoyong He <chaoyong...@corigine.com>
Signed-off-by: Shujing Dong <shujing.d...@corigine.com>
Reviewed-by: Long Wu <long...@corigine.com>
Reviewed-by: Peng Zhang <peng.zh...@corigine.com>
---
drivers/vdpa/nfp/nfp_vdpa.c | 155 +++++++++++++++++++++++++++++++
drivers/vdpa/nfp/nfp_vdpa_core.c | 61 ++++++++++++
drivers/vdpa/nfp/nfp_vdpa_core.h | 4 +
3 files changed, 220 insertions(+)
diff --git a/drivers/vdpa/nfp/nfp_vdpa.c b/drivers/vdpa/nfp/nfp_vdpa.c
index 465ee4841d..01a59e2cfd 100644
--- a/drivers/vdpa/nfp/nfp_vdpa.c
+++ b/drivers/vdpa/nfp/nfp_vdpa.c
@@ -4,7 +4,9 @@
*/
#include <pthread.h>
+#include <sys/epoll.h>
#include <sys/ioctl.h>
+#include <unistd.h>
#include <nfp_common_pci.h>
#include <nfp_dev.h>
@@ -29,6 +31,9 @@ struct nfp_vdpa_dev {
int vfio_dev_fd;
int iommu_group;
+ pthread_t tid; /**< Thread for notify relay */
+ int epoll_fd;
+
int vid;
uint16_t max_queues;
uint32_t started;
@@ -368,6 +373,148 @@ nfp_vdpa_disable_vfio_intr(struct nfp_vdpa_dev *device)
return 0;
}
+static void
+nfp_vdpa_read_kickfd(int kickfd)
+{
+ int bytes;
+ uint64_t buf;
+
+ for (;;) {
+ bytes = read(kickfd, &buf, 8);
+ if (bytes >= 0)
+ break;
+
+ if (errno != EINTR && errno != EWOULDBLOCK &&
+ errno != EAGAIN) {
+ DRV_VDPA_LOG(ERR, "Error reading kickfd");
+ break;
+ }
+ }
+}
+
+static int
+nfp_vdpa_notify_epoll_ctl(uint32_t queue_num,
+ struct nfp_vdpa_dev *device)
+{
+ int ret;
+ uint32_t qid;
+
+ for (qid = 0; qid < queue_num; qid++) {
+ struct epoll_event ev;
+ struct rte_vhost_vring vring;
+
+ ev.events = EPOLLIN | EPOLLPRI;
+ rte_vhost_get_vhost_vring(device->vid, qid, &vring);
+ ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
+ ret = epoll_ctl(device->epoll_fd, EPOLL_CTL_ADD, vring.kickfd,
&ev);
+ if (ret < 0) {
+ DRV_VDPA_LOG(ERR, "Epoll add error for queue %d", qid);
+ return ret;
+ }
+ }
+
+ return 0;
+}
+
+static int
+nfp_vdpa_notify_epoll_wait(uint32_t queue_num,
+ struct nfp_vdpa_dev *device)
+{
+ int i;
+ int fds;
+ int kickfd;
+ uint32_t qid;
+ struct epoll_event events[NFP_VDPA_MAX_QUEUES * 2];
+
+ for (;;) {
+ fds = epoll_wait(device->epoll_fd, events, queue_num, -1);
+ if (fds < 0) {
+ if (errno == EINTR)
+ continue;
+
+ DRV_VDPA_LOG(ERR, "Epoll wait fail");
+ return -EACCES;
+ }
+
+ for (i = 0; i < fds; i++) {
+ qid = events[i].data.u32;
+ kickfd = (uint32_t)(events[i].data.u64 >> 32);
+
+ nfp_vdpa_read_kickfd(kickfd);
+ nfp_vdpa_notify_queue(&device->hw, qid);
+ }
+ }
+
+ return 0;
+}
+
+static void *
+nfp_vdpa_notify_relay(void *arg)
+{
+ int ret;
+ int epoll_fd;
+ uint32_t queue_num;
+ struct nfp_vdpa_dev *device = arg;
+
+ epoll_fd = epoll_create(NFP_VDPA_MAX_QUEUES * 2);
+ if (epoll_fd < 0) {
+ DRV_VDPA_LOG(ERR, "failed to create epoll instance.");
+ return NULL;
+ }
+
+ device->epoll_fd = epoll_fd;
+
+ queue_num = rte_vhost_get_vring_num(device->vid);
+
+ ret = nfp_vdpa_notify_epoll_ctl(queue_num, device);
+ if (ret != 0)
+ goto notify_exit;
+
+ ret = nfp_vdpa_notify_epoll_wait(queue_num, device);
+ if (ret != 0)
+ goto notify_exit;
+
+ return 0;
+
+notify_exit:
+ close(device->epoll_fd);
+ device->epoll_fd = -1;
+
+ return NULL;
+}
+
+static int
+nfp_vdpa_setup_notify_relay(struct nfp_vdpa_dev *device)
+{
+ int ret;
+
+ ret = pthread_create(&device->tid, NULL, nfp_vdpa_notify_relay,
+ (void *)device);
+ if (ret != 0) {
+ DRV_VDPA_LOG(ERR, "Failed to create notify relay pthread.");
+ return -1;
+ }
+
+ return 0;
+}
+
+static void
+nfp_vdpa_unset_notify_relay(struct nfp_vdpa_dev *device)
+{
+ void *status;
+
+ if (device->tid != 0) {
+ pthread_cancel(device->tid);
+ pthread_join(device->tid, &status);
+ device->tid = 0;
+ }
+
+ if (device->epoll_fd >= 0) {
+ close(device->epoll_fd);
+ device->epoll_fd = -1;
+ }
+}
+
static int
update_datapath(struct nfp_vdpa_dev *device)
{
@@ -390,11 +537,17 @@ update_datapath(struct nfp_vdpa_dev *device)
if (ret != 0)
goto disable_vfio_intr;
+ ret = nfp_vdpa_setup_notify_relay(device);
+ if (ret != 0)
+ goto vdpa_stop;
+
__atomic_store_n(&device->running, 1, __ATOMIC_RELAXED);
} else if ((__atomic_load_n(&device->running, __ATOMIC_RELAXED) != 0) &&
((__atomic_load_n(&device->started, __ATOMIC_RELAXED)
!= 0) ||
(__atomic_load_n(&device->dev_attached,
__ATOMIC_RELAXED) != 0))) {
+ nfp_vdpa_unset_notify_relay(device);
+
nfp_vdpa_stop(device);
ret = nfp_vdpa_disable_vfio_intr(device);
@@ -411,6 +564,8 @@ update_datapath(struct nfp_vdpa_dev *device)
rte_spinlock_unlock(&device->lock);
return 0;
+vdpa_stop:
+ nfp_vdpa_stop(device);
disable_vfio_intr:
nfp_vdpa_disable_vfio_intr(device);
dma_map_rollback:
diff --git a/drivers/vdpa/nfp/nfp_vdpa_core.c b/drivers/vdpa/nfp/nfp_vdpa_core.c
index db9b8462b4..e2a6253ae5 100644
--- a/drivers/vdpa/nfp/nfp_vdpa_core.c
+++ b/drivers/vdpa/nfp/nfp_vdpa_core.c
@@ -15,6 +15,41 @@
#define VIRTIO_F_IN_ORDER 35
#endif
+#define NFP_QCP_NOTIFY_MAX_ADD 0x7f
+
+enum nfp_qcp_notify_ptr {
+ NFP_QCP_NOTIFY_WRITE_PTR = 0,
+ NFP_QCP_NOTIFY_READ_PTR
+};
+
+/**
+ * Add the value to the selected pointer of a queue
+ *
+ * @param queue
+ * Base address for queue structure
+ * @param ptr
+ * Add to the Read or Write pointer
+ * @param val
+ * Value to add to the queue pointer
+ */
+static inline void
+nfp_qcp_notify_ptr_add(uint8_t *q,
+ enum nfp_qcp_notify_ptr ptr,
+ uint32_t val)
+{
+ uint32_t off;
+
+ if (ptr == NFP_QCP_NOTIFY_WRITE_PTR)
+ off = NFP_QCP_QUEUE_ADD_RPTR;
+ else
+ off = NFP_QCP_QUEUE_ADD_WPTR;
+
+ for (; val > NFP_QCP_NOTIFY_MAX_ADD; val -= NFP_QCP_NOTIFY_MAX_ADD)
+ nn_writel(rte_cpu_to_le_32(NFP_QCP_NOTIFY_MAX_ADD), q + off);
+
+ nn_writel(rte_cpu_to_le_32(val), q + off);
+}
+
int
nfp_vdpa_hw_init(struct nfp_vdpa_hw *vdpa_hw,
struct rte_pci_device *pci_dev)
@@ -130,3 +165,29 @@ nfp_vdpa_hw_stop(struct nfp_vdpa_hw *vdpa_hw)
{
nfp_disable_queues(&vdpa_hw->super);
}
+
+/*
+ * This offset is used for mmaping the notify area. It implies it needs
+ * to be a multiple of PAGE_SIZE.
+ * For debugging, using notify region 0 with an offset of 4K. This should
+ * point to the conf bar.
+ */
+uint64_t
+nfp_vdpa_get_queue_notify_offset(struct nfp_vdpa_hw *vdpa_hw __rte_unused,
+ int qid)
+{
+ return NFP_VDPA_NOTIFY_ADDR_BASE + (qid *
NFP_VDPA_NOTIFY_ADDR_INTERVAL);
+}
+
+/*
+ * With just one queue the increment is 0, which does not
+ * incremente the counter but will raise a queue event due
+ * to queue configured for watermark events.
+ */
+void
+nfp_vdpa_notify_queue(struct nfp_vdpa_hw *vdpa_hw,
+ uint16_t qid)
+{
+ nfp_qcp_notify_ptr_add(vdpa_hw->notify_addr[qid],
+ NFP_QCP_NOTIFY_WRITE_PTR, qid);
+}
diff --git a/drivers/vdpa/nfp/nfp_vdpa_core.h b/drivers/vdpa/nfp/nfp_vdpa_core.h
index a88de768dd..a8e0d6dd70 100644
--- a/drivers/vdpa/nfp/nfp_vdpa_core.h
+++ b/drivers/vdpa/nfp/nfp_vdpa_core.h
@@ -44,4 +44,8 @@ int nfp_vdpa_hw_start(struct nfp_vdpa_hw *vdpa_hw, int vid);
void nfp_vdpa_hw_stop(struct nfp_vdpa_hw *vdpa_hw);
+void nfp_vdpa_notify_queue(struct nfp_vdpa_hw *vdpa_hw, uint16_t qid);
+
+uint64_t nfp_vdpa_get_queue_notify_offset(struct nfp_vdpa_hw *vdpa_hw, int
qid);
+
#endif /* __NFP_VDPA_CORE_H__ */
--
2.39.1
