> -----Original Message-----
> From: Maxime Coquelin <maxime.coque...@redhat.com>
> Sent: Wednesday, January 20, 2021 5:25 AM
> To: dev@dpdk.org; Xia, Chenbo <chenbo....@intel.com>; olivier.m...@6wind.com;
> amore...@redhat.com; david.march...@redhat.com
> Cc: Maxime Coquelin <maxime.coque...@redhat.com>
> Subject: [PATCH v2 30/44] net/virtio: add Virtio-user vring setting ops
>
> This patch introduces new callbacks for setting
> and getting vring state.
>
> Signed-off-by: Maxime Coquelin <maxime.coque...@redhat.com>
> ---
> drivers/net/virtio/virtio_user/vhost.h | 3 +
> drivers/net/virtio/virtio_user/vhost_kernel.c | 49 +++++++-
> drivers/net/virtio/virtio_user/vhost_user.c | 113 +++++++++++++-----
> drivers/net/virtio/virtio_user/vhost_vdpa.c | 41 +++++--
> .../net/virtio/virtio_user/virtio_user_dev.c | 9 +-
> 5 files changed, 165 insertions(+), 50 deletions(-)
>
> diff --git a/drivers/net/virtio/virtio_user/vhost.h
> b/drivers/net/virtio/virtio_user/vhost.h
> index 463801f563..2d433d5966 100644
> --- a/drivers/net/virtio/virtio_user/vhost.h
> +++ b/drivers/net/virtio/virtio_user/vhost.h
> @@ -115,6 +115,9 @@ struct virtio_user_backend_ops {
> int (*get_protocol_features)(struct virtio_user_dev *dev, uint64_t
> *features);
> int (*set_protocol_features)(struct virtio_user_dev *dev, uint64_t
> features);
> int (*set_memory_table)(struct virtio_user_dev *dev);
> + int (*set_vring_num)(struct virtio_user_dev *dev, struct
> vhost_vring_state *state);
> + int (*set_vring_base)(struct virtio_user_dev *dev, struct
> vhost_vring_state *state);
> + int (*get_vring_base)(struct virtio_user_dev *dev, struct
> vhost_vring_state *state);
> int (*send_request)(struct virtio_user_dev *dev,
> enum vhost_user_request req,
> void *arg);
> diff --git a/drivers/net/virtio/virtio_user/vhost_kernel.c
> b/drivers/net/virtio/virtio_user/vhost_kernel.c
> index 2d30f572b6..e12fe74f46 100644
> --- a/drivers/net/virtio/virtio_user/vhost_kernel.c
> +++ b/drivers/net/virtio/virtio_user/vhost_kernel.c
> @@ -219,12 +219,49 @@ vhost_kernel_set_memory_table(struct virtio_user_dev
> *dev)
> return -1;
> }
>
> +static int
> +vhost_kernel_set_vring(struct virtio_user_dev *dev, uint64_t req, struct
> vhost_vring_state *state)
> +{
> + int ret, fd;
> + unsigned int index = state->index;
> +
> + /* Convert from queue index to queue-pair & offset */
> + fd = dev->vhostfds[state->index / 2];
> + state->index %= 2;
> +
> + ret = vhost_kernel_ioctl(fd, req, state);
> + if (ret < 0) {
> + PMD_DRV_LOG(ERR, "Failed to set vring (request %" PRIu64 ")",
> req);
> + return -1;
> + }
> +
> + /* restore index back to queue index */
> + state->index = index;
> +
> + return 0;
> +}
> +
> +static int
> +vhost_kernel_set_vring_num(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_kernel_set_vring(dev, VHOST_SET_VRING_NUM, state);
> +}
> +
> +static int
> +vhost_kernel_set_vring_base(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_kernel_set_vring(dev, VHOST_SET_VRING_BASE, state);
> +}
> +
> +static int
> +vhost_kernel_get_vring_base(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_kernel_set_vring(dev, VHOST_GET_VRING_BASE, state);
> +}
> +
> static uint64_t vhost_req_user_to_kernel[] = {
> [VHOST_USER_RESET_OWNER] = VHOST_RESET_OWNER,
> [VHOST_USER_SET_VRING_CALL] = VHOST_SET_VRING_CALL,
> - [VHOST_USER_SET_VRING_NUM] = VHOST_SET_VRING_NUM,
> - [VHOST_USER_SET_VRING_BASE] = VHOST_SET_VRING_BASE,
> - [VHOST_USER_GET_VRING_BASE] = VHOST_GET_VRING_BASE,
> [VHOST_USER_SET_VRING_ADDR] = VHOST_SET_VRING_ADDR,
> [VHOST_USER_SET_VRING_KICK] = VHOST_SET_VRING_KICK,
> };
> @@ -245,10 +282,7 @@ vhost_kernel_send_request(struct virtio_user_dev *dev,
> req_kernel = vhost_req_user_to_kernel[req];
>
> switch (req_kernel) {
> - case VHOST_SET_VRING_NUM:
> case VHOST_SET_VRING_ADDR:
> - case VHOST_SET_VRING_BASE:
> - case VHOST_GET_VRING_BASE:
> case VHOST_SET_VRING_KICK:
> case VHOST_SET_VRING_CALL:
> queue_sel = *(unsigned int *)arg;
> @@ -403,6 +437,9 @@ struct virtio_user_backend_ops virtio_ops_kernel = {
> .get_features = vhost_kernel_get_features,
> .set_features = vhost_kernel_set_features,
> .set_memory_table = vhost_kernel_set_memory_table,
> + .set_vring_num = vhost_kernel_set_vring_num,
> + .set_vring_base = vhost_kernel_set_vring_base,
> + .get_vring_base = vhost_kernel_get_vring_base,
> .send_request = vhost_kernel_send_request,
> .enable_qp = vhost_kernel_enable_queue_pair
> };
> diff --git a/drivers/net/virtio/virtio_user/vhost_user.c
> b/drivers/net/virtio/virtio_user/vhost_user.c
> index 94533e31ce..c39be24ce4 100644
> --- a/drivers/net/virtio/virtio_user/vhost_user.c
> +++ b/drivers/net/virtio/virtio_user/vhost_user.c
> @@ -417,17 +417,94 @@ vhost_user_set_memory_table(struct virtio_user_dev *dev)
> return -1;
> }
>
> +static int
> +vhost_user_set_vring(struct virtio_user_dev *dev, enum vhost_user_request
> req,
> + struct vhost_vring_state *state)
> +{
> + int ret;
> + struct vhost_user_msg msg = {
> + .request = req,
> + .flags = VHOST_USER_VERSION,
> + .size = sizeof(*state),
> + .payload.state = *state,
> + };
> +
> + ret = vhost_user_write(dev->vhostfd, &msg, NULL, 0);
> + if (ret < 0) {
> + PMD_DRV_LOG(ERR, "Failed to set vring state (request %d)", req);
> + return -1;
> + }
> +
> + return 0;
> +}
> +
> +static int
> +vhost_user_set_vring_enable(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_ENABLE, state);
> +}
> +
> +static int
> +vhost_user_set_vring_num(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_NUM, state);
> +}
> +
> +static int
> +vhost_user_set_vring_base(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_user_set_vring(dev, VHOST_USER_SET_VRING_BASE, state);
> +}
> +
> +static int
> +vhost_user_get_vring_base(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + int ret;
> + struct vhost_user_msg msg;
> + unsigned int index = state->index;
> +
> + ret = vhost_user_set_vring(dev, VHOST_USER_GET_VRING_BASE, state);
> + if (ret < 0) {
> + PMD_DRV_LOG(ERR, "Failed to send request");
> + goto err;
> + }
> +
> + ret = vhost_user_read(dev->vhostfd, &msg);
> + if (ret < 0) {
> + PMD_DRV_LOG(ERR, "Failed to read reply");
> + goto err;
> + }
> +
> + if (msg.request != VHOST_USER_GET_VRING_BASE) {
> + PMD_DRV_LOG(ERR, "Unexpected request type (%d)", msg.request);
> + goto err;
> + }
> +
> + if (msg.size != sizeof(*state)) {
> + PMD_DRV_LOG(ERR, "Unexpected payload size (%u)", msg.size);
> + goto err;
> + }
> +
> + if (msg.payload.state.index != index) {
> + PMD_DRV_LOG(ERR, "Unexpected ring index (%u)", state->index);
> + goto err;
> + }
> +
> + *state = msg.payload.state;
> +
> + return 0;
> +err:
> + PMD_DRV_LOG(ERR, "Failed to get vring base");
> + return -1;
> +}
> +
> static struct vhost_user_msg m;
>
> const char * const vhost_msg_strings[] = {
> [VHOST_USER_RESET_OWNER] = "VHOST_RESET_OWNER",
> [VHOST_USER_SET_VRING_CALL] = "VHOST_SET_VRING_CALL",
> - [VHOST_USER_SET_VRING_NUM] = "VHOST_SET_VRING_NUM",
> - [VHOST_USER_SET_VRING_BASE] = "VHOST_SET_VRING_BASE",
> - [VHOST_USER_GET_VRING_BASE] = "VHOST_GET_VRING_BASE",
> [VHOST_USER_SET_VRING_ADDR] = "VHOST_SET_VRING_ADDR",
> [VHOST_USER_SET_VRING_KICK] = "VHOST_SET_VRING_KICK",
> - [VHOST_USER_SET_VRING_ENABLE] = "VHOST_SET_VRING_ENABLE",
> [VHOST_USER_SET_STATUS] = "VHOST_SET_STATUS",
> [VHOST_USER_GET_STATUS] = "VHOST_GET_STATUS",
> };
> @@ -495,19 +572,6 @@ vhost_user_sock(struct virtio_user_dev *dev,
> fds[fd_num++] = *((int *)arg);
> break;
>
> - case VHOST_USER_SET_VRING_NUM:
> - case VHOST_USER_SET_VRING_BASE:
> - case VHOST_USER_SET_VRING_ENABLE:
> - memcpy(&msg.payload.state, arg, sizeof(msg.payload.state));
> - msg.size = sizeof(m.payload.state);
> - break;
> -
> - case VHOST_USER_GET_VRING_BASE:
> - memcpy(&msg.payload.state, arg, sizeof(msg.payload.state));
> - msg.size = sizeof(m.payload.state);
> - need_reply = 1;
> - break;
> -
> case VHOST_USER_SET_VRING_ADDR:
> memcpy(&msg.payload.addr, arg, sizeof(msg.payload.addr));
> msg.size = sizeof(m.payload.addr);
> @@ -556,14 +620,6 @@ vhost_user_sock(struct virtio_user_dev *dev,
> }
> *((__u64 *)arg) = msg.payload.u64;
> break;
> - case VHOST_USER_GET_VRING_BASE:
> - if (msg.size != sizeof(m.payload.state)) {
> - PMD_DRV_LOG(ERR, "Received bad msg size");
> - return -1;
> - }
> - memcpy(arg, &msg.payload.state,
> - sizeof(struct vhost_vring_state));
> - break;
> default:
> /* Reply-ack handling */
> if (msg.size != sizeof(m.payload.u64)) {
> @@ -671,10 +727,10 @@ vhost_user_enable_queue_pair(struct virtio_user_dev
> *dev,
> for (i = 0; i < 2; ++i) {
> struct vhost_vring_state state = {
> .index = pair_idx * 2 + i,
> - .num = enable,
> + .num = enable,
> };
>
> - if (vhost_user_sock(dev, VHOST_USER_SET_VRING_ENABLE, &state))
> + if (vhost_user_set_vring_enable(dev, &state))
> return -1;
> }
>
> @@ -690,6 +746,9 @@ struct virtio_user_backend_ops virtio_ops_user = {
> .get_protocol_features = vhost_user_get_protocol_features,
> .set_protocol_features = vhost_user_set_protocol_features,
> .set_memory_table = vhost_user_set_memory_table,
> + .set_vring_num = vhost_user_set_vring_num,
> + .set_vring_base = vhost_user_set_vring_base,
> + .get_vring_base = vhost_user_get_vring_base,
> .send_request = vhost_user_sock,
> .enable_qp = vhost_user_enable_queue_pair
> };
> diff --git a/drivers/net/virtio/virtio_user/vhost_vdpa.c
> b/drivers/net/virtio/virtio_user/vhost_vdpa.c
> index bd7daebf0c..323751093a 100644
> --- a/drivers/net/virtio/virtio_user/vhost_vdpa.c
> +++ b/drivers/net/virtio/virtio_user/vhost_vdpa.c
> @@ -32,24 +32,17 @@
> #define VHOST_VDPA_GET_DEVICE_ID _IOR(VHOST_VIRTIO, 0x70, __u32)
> #define VHOST_VDPA_GET_STATUS _IOR(VHOST_VIRTIO, 0x71, __u8)
> #define VHOST_VDPA_SET_STATUS _IOW(VHOST_VIRTIO, 0x72, __u8)
> -#define VHOST_VDPA_SET_VRING_ENABLE _IOW(VHOST_VIRTIO, 0x75, \
> - struct vhost_vring_state)
> +#define VHOST_VDPA_SET_VRING_ENABLE _IOW(VHOST_VIRTIO, 0x75, struct
> vhost_vring_state)
> #define VHOST_SET_BACKEND_FEATURES _IOW(VHOST_VIRTIO, 0x25, __u64)
> #define VHOST_GET_BACKEND_FEATURES _IOR(VHOST_VIRTIO, 0x26, __u64)
>
> static uint64_t vhost_req_user_to_vdpa[] = {
> [VHOST_USER_RESET_OWNER] = VHOST_RESET_OWNER,
> [VHOST_USER_SET_VRING_CALL] = VHOST_SET_VRING_CALL,
> - [VHOST_USER_SET_VRING_NUM] = VHOST_SET_VRING_NUM,
> - [VHOST_USER_SET_VRING_BASE] = VHOST_SET_VRING_BASE,
> - [VHOST_USER_GET_VRING_BASE] = VHOST_GET_VRING_BASE,
> [VHOST_USER_SET_VRING_ADDR] = VHOST_SET_VRING_ADDR,
> [VHOST_USER_SET_VRING_KICK] = VHOST_SET_VRING_KICK,
> [VHOST_USER_SET_STATUS] = VHOST_VDPA_SET_STATUS,
> [VHOST_USER_GET_STATUS] = VHOST_VDPA_GET_STATUS,
> - [VHOST_USER_SET_VRING_ENABLE] = VHOST_VDPA_SET_VRING_ENABLE,
> - [VHOST_USER_GET_PROTOCOL_FEATURES] = VHOST_GET_BACKEND_FEATURES,
> - [VHOST_USER_SET_PROTOCOL_FEATURES] = VHOST_SET_BACKEND_FEATURES,
> };
>
> /* no alignment requirement */
> @@ -347,6 +340,30 @@ vhost_vdpa_set_memory_table(struct virtio_user_dev *dev)
> return ret;
> }
>
> +static int
> +vhost_vdpa_set_vring_enable(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_vdpa_ioctl(dev->vhostfd, VHOST_VDPA_SET_VRING_ENABLE,
> state);
> +}
> +
> +static int
> +vhost_vdpa_set_vring_num(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_NUM, state);
> +}
> +
> +static int
> +vhost_vdpa_set_vring_base(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_vdpa_ioctl(dev->vhostfd, VHOST_SET_VRING_BASE, state);
> +}
> +
> +static int
> +vhost_vdpa_get_vring_base(struct virtio_user_dev *dev, struct
> vhost_vring_state *state)
> +{
> + return vhost_vdpa_ioctl(dev->vhostfd, VHOST_GET_VRING_BASE, state);
> +}
> +
> /* with below features, vhost vdpa does not need to do the checksum and TSO,
> * these info will be passed to virtio_user through virtio net header.
> */
> @@ -375,10 +392,7 @@ vhost_vdpa_send_request(struct virtio_user_dev *dev,
> req_vdpa = vhost_req_user_to_vdpa[req];
>
> switch (req_vdpa) {
> - case VHOST_SET_VRING_NUM:
> case VHOST_SET_VRING_ADDR:
> - case VHOST_SET_VRING_BASE:
> - case VHOST_GET_VRING_BASE:
> case VHOST_SET_VRING_KICK:
> case VHOST_SET_VRING_CALL:
> PMD_DRV_LOG(DEBUG, "vhostfd=%d, index=%u",
> @@ -440,7 +454,7 @@ vhost_vdpa_enable_queue_pair(struct virtio_user_dev *dev,
> .num = enable,
> };
>
> - if (vhost_vdpa_send_request(dev, VHOST_USER_SET_VRING_ENABLE,
> &state))
> + if (vhost_vdpa_set_vring_enable(dev, &state))
> return -1;
> }
>
> @@ -457,6 +471,9 @@ struct virtio_user_backend_ops virtio_ops_vdpa = {
> .get_protocol_features = vhost_vdpa_get_backend_features,
> .set_protocol_features = vhost_vdpa_set_backend_features,
> .set_memory_table = vhost_vdpa_set_memory_table,
> + .set_vring_num = vhost_vdpa_set_vring_num,
> + .set_vring_base = vhost_vdpa_set_vring_base,
> + .get_vring_base = vhost_vdpa_get_vring_base,
> .send_request = vhost_vdpa_send_request,
> .enable_qp = vhost_vdpa_enable_queue_pair,
> .dma_map = vhost_vdpa_dma_map_batch,
> diff --git a/drivers/net/virtio/virtio_user/virtio_user_dev.c
> b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> index cb8f8a872c..0b30c10016 100644
> --- a/drivers/net/virtio/virtio_user/virtio_user_dev.c
> +++ b/drivers/net/virtio/virtio_user/virtio_user_dev.c
> @@ -73,13 +73,13 @@ virtio_user_kick_queue(struct virtio_user_dev *dev,
> uint32_t queue_sel)
>
> state.index = queue_sel;
> state.num = vring->num;
> - dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state);
> + dev->ops->set_vring_num(dev, &state);
>
> state.index = queue_sel;
> state.num = 0; /* no reservation */
> if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
> state.num |= (1 << 15);
> - dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state);
> + dev->ops->set_vring_base(dev, &state);
>
> dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr);
>
> @@ -218,9 +218,8 @@ int virtio_user_stop_device(struct virtio_user_dev *dev)
> /* Stop the backend. */
> for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
> state.index = i;
> - if (dev->ops->send_request(dev, VHOST_USER_GET_VRING_BASE,
> - &state) < 0) {
> - PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u\n",
> + if (dev->ops->get_vring_base(dev, &state) < 0) {
> + PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u",
> i);
> error = -1;
> goto out;
> --
> 2.29.2
Reviewed-by: Chenbo Xia <chenbo....@intel.com>
