On Mon, 8 Nov 2021 16:46:35 -0800
John Johnson <john.g.john...@oracle.com> wrote:
> Signed-off-by: John G Johnson <john.g.john...@oracle.com>
> Signed-off-by: Elena Ufimtseva <elena.ufimts...@oracle.com>
> Signed-off-by: Jagannathan Raman <jag.ra...@oracle.com>
> ---
> hw/vfio/user.h | 78 +++++++++++++++++++
> include/hw/vfio/vfio-common.h | 2 +
> hw/vfio/pci.c | 20 +++++
> hw/vfio/user.c | 170
> ++++++++++++++++++++++++++++++++++++++++++
> MAINTAINERS | 4 +
> hw/vfio/meson.build | 1 +
> 6 files changed, 275 insertions(+)
> create mode 100644 hw/vfio/user.h
> create mode 100644 hw/vfio/user.c
>
> diff --git a/hw/vfio/user.h b/hw/vfio/user.h
> new file mode 100644
> index 0000000..301ef6a
> --- /dev/null
> +++ b/hw/vfio/user.h
> @@ -0,0 +1,78 @@
> +#ifndef VFIO_USER_H
> +#define VFIO_USER_H
> +
> +/*
> + * vfio protocol over a UNIX socket.
> + *
> + * Copyright © 2018, 2021 Oracle and/or its affiliates.
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2. See
> + * the COPYING file in the top-level directory.
> + *
> + */
> +
> +typedef struct {
> + int send_fds;
> + int recv_fds;
> + int *fds;
> +} VFIOUserFDs;
> +
> +enum msg_type {
> + VFIO_MSG_NONE,
> + VFIO_MSG_ASYNC,
> + VFIO_MSG_WAIT,
> + VFIO_MSG_NOWAIT,
> + VFIO_MSG_REQ,
> +};
> +
> +typedef struct VFIOUserMsg {
> + QTAILQ_ENTRY(VFIOUserMsg) next;
> + VFIOUserFDs *fds;
> + uint32_t rsize;
> + uint32_t id;
> + QemuCond cv;
> + bool complete;
> + enum msg_type type;
> +} VFIOUserMsg;
> +
> +
> +enum proxy_state {
> + VFIO_PROXY_CONNECTED = 1,
> + VFIO_PROXY_ERROR = 2,
> + VFIO_PROXY_CLOSING = 3,
> + VFIO_PROXY_CLOSED = 4,
> +};
> +
> +typedef QTAILQ_HEAD(VFIOUserMsgQ, VFIOUserMsg) VFIOUserMsgQ;
> +
> +typedef struct VFIOProxy {
> + QLIST_ENTRY(VFIOProxy) next;
> + char *sockname;
> + struct QIOChannel *ioc;
> + void (*request)(void *opaque, VFIOUserMsg *msg);
> + void *req_arg;
> + int flags;
> + QemuCond close_cv;
> + AioContext *ctx;
> + QEMUBH *req_bh;
> +
> + /*
> + * above only changed when BQL is held
> + * below are protected by per-proxy lock
> + */
> + QemuMutex lock;
> + VFIOUserMsgQ free;
> + VFIOUserMsgQ pending;
> + VFIOUserMsgQ incoming;
> + VFIOUserMsgQ outgoing;
> + VFIOUserMsg *last_nowait;
> + enum proxy_state state;
> +} VFIOProxy;
> +
> +/* VFIOProxy flags */
> +#define VFIO_PROXY_CLIENT 0x1
> +
> +VFIOProxy *vfio_user_connect_dev(SocketAddress *addr, Error **errp);
> +void vfio_user_disconnect(VFIOProxy *proxy);
> +
> +#endif /* VFIO_USER_H */
> diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
> index c0dbbfb..224dbf8 100644
> --- a/include/hw/vfio/vfio-common.h
> +++ b/include/hw/vfio/vfio-common.h
> @@ -76,6 +76,7 @@ typedef struct VFIOAddressSpace {
>
> struct VFIOGroup;
> typedef struct VFIOContIO VFIOContIO;
> +typedef struct VFIOProxy VFIOProxy;
>
> typedef struct VFIOContainer {
> VFIOAddressSpace *space;
> @@ -150,6 +151,7 @@ typedef struct VFIODevice {
> Error *migration_blocker;
> OnOffAuto pre_copy_dirty_page_tracking;
> struct vfio_region_info **regions;
> + VFIOProxy *proxy;
> } VFIODevice;
>
> struct VFIODeviceOps {
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index fa3e028..ebfabb1 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -43,6 +43,7 @@
> #include "qapi/error.h"
> #include "migration/blocker.h"
> #include "migration/qemu-file.h"
> +#include "hw/vfio/user.h"
>
> #define TYPE_VFIO_PCI_NOHOTPLUG "vfio-pci-nohotplug"
>
> @@ -3476,6 +3477,9 @@ static void vfio_user_pci_realize(PCIDevice *pdev,
> Error **errp)
> VFIOUserPCIDevice *udev = VFIO_USER_PCI(pdev);
> VFIOPCIDevice *vdev = VFIO_PCI_BASE(pdev);
> VFIODevice *vbasedev = &vdev->vbasedev;
> + SocketAddress addr;
> + VFIOProxy *proxy;
> + Error *err = NULL;
>
> /*
> * TODO: make option parser understand SocketAddress
> @@ -3488,6 +3492,16 @@ static void vfio_user_pci_realize(PCIDevice *pdev,
> Error **errp)
> return;
> }
>
> + memset(&addr, 0, sizeof(addr));
> + addr.type = SOCKET_ADDRESS_TYPE_UNIX;
> + addr.u.q_unix.path = udev->sock_name;
> + proxy = vfio_user_connect_dev(&addr, &err);
> + if (!proxy) {
> + error_setg(errp, "Remote proxy not found");
> + return;
> + }
> + vbasedev->proxy = proxy;
> +
> vbasedev->name = g_strdup_printf("VFIO user <%s>", udev->sock_name);
> vbasedev->dev = DEVICE(vdev);
> vbasedev->fd = -1;
> @@ -3500,6 +3514,12 @@ static void vfio_user_pci_realize(PCIDevice *pdev,
> Error **errp)
>
> static void vfio_user_instance_finalize(Object *obj)
> {
> + VFIOPCIDevice *vdev = VFIO_PCI_BASE(obj);
> + VFIODevice *vbasedev = &vdev->vbasedev;
> +
> + vfio_put_device(vdev);
This looks suspiciously like the initial function in the previous patch
should not have been empty. Thanks,
Alex
> +
> + vfio_user_disconnect(vbasedev->proxy);
> }
>
> static Property vfio_user_pci_dev_properties[] = {
> diff --git a/hw/vfio/user.c b/hw/vfio/user.c
> new file mode 100644
> index 0000000..92d4e03
> --- /dev/null
> +++ b/hw/vfio/user.c
> @@ -0,0 +1,170 @@
> +/*
> + * vfio protocol over a UNIX socket.
> + *
> + * Copyright © 2018, 2021 Oracle and/or its affiliates.
> + *
> + * This work is licensed under the terms of the GNU GPL, version 2 or later.
> + * See the COPYING file in the top-level directory.
> + *
> + */
> +
> +#include "qemu/osdep.h"
> +#include <linux/vfio.h>
> +#include <sys/ioctl.h>
> +
> +#include "qemu/error-report.h"
> +#include "qapi/error.h"
> +#include "qemu/main-loop.h"
> +#include "hw/hw.h"
> +#include "hw/vfio/vfio-common.h"
> +#include "hw/vfio/vfio.h"
> +#include "qemu/sockets.h"
> +#include "io/channel.h"
> +#include "io/channel-socket.h"
> +#include "io/channel-util.h"
> +#include "sysemu/iothread.h"
> +#include "user.h"
> +
> +static IOThread *vfio_user_iothread;
> +static void vfio_user_shutdown(VFIOProxy *proxy);
> +
> +
> +/*
> + * Functions called by main, CPU, or iothread threads
> + */
> +
> +static void vfio_user_shutdown(VFIOProxy *proxy)
> +{
> + qio_channel_shutdown(proxy->ioc, QIO_CHANNEL_SHUTDOWN_READ, NULL);
> + qio_channel_set_aio_fd_handler(proxy->ioc, proxy->ctx, NULL, NULL, NULL);
> +}
> +
> +
> +/*
> + * Functions only called by iothread
> + */
> +
> +static void vfio_user_cb(void *opaque)
> +{
> + VFIOProxy *proxy = opaque;
> +
> + QEMU_LOCK_GUARD(&proxy->lock);
> +
> + proxy->state = VFIO_PROXY_CLOSED;
> + qemu_cond_signal(&proxy->close_cv);
> +}
> +
> +
> +/*
> + * Functions called by main or CPU threads
> + */
> +
> +static QLIST_HEAD(, VFIOProxy) vfio_user_sockets =
> + QLIST_HEAD_INITIALIZER(vfio_user_sockets);
> +
> +VFIOProxy *vfio_user_connect_dev(SocketAddress *addr, Error **errp)
> +{
> + VFIOProxy *proxy;
> + QIOChannelSocket *sioc;
> + QIOChannel *ioc;
> + char *sockname;
> +
> + if (addr->type != SOCKET_ADDRESS_TYPE_UNIX) {
> + error_setg(errp, "vfio_user_connect - bad address family");
> + return NULL;
> + }
> + sockname = addr->u.q_unix.path;
> +
> + sioc = qio_channel_socket_new();
> + ioc = QIO_CHANNEL(sioc);
> + if (qio_channel_socket_connect_sync(sioc, addr, errp)) {
> + object_unref(OBJECT(ioc));
> + return NULL;
> + }
> + qio_channel_set_blocking(ioc, false, NULL);
> +
> + proxy = g_malloc0(sizeof(VFIOProxy));
> + proxy->sockname = g_strdup_printf("unix:%s", sockname);
> + proxy->ioc = ioc;
> + proxy->flags = VFIO_PROXY_CLIENT;
> + proxy->state = VFIO_PROXY_CONNECTED;
> +
> + qemu_mutex_init(&proxy->lock);
> + qemu_cond_init(&proxy->close_cv);
> +
> + if (vfio_user_iothread == NULL) {
> + vfio_user_iothread = iothread_create("VFIO user", errp);
> + }
> +
> + proxy->ctx = iothread_get_aio_context(vfio_user_iothread);
> +
> + QTAILQ_INIT(&proxy->outgoing);
> + QTAILQ_INIT(&proxy->incoming);
> + QTAILQ_INIT(&proxy->free);
> + QTAILQ_INIT(&proxy->pending);
> + QLIST_INSERT_HEAD(&vfio_user_sockets, proxy, next);
> +
> + return proxy;
> +}
> +
> +void vfio_user_disconnect(VFIOProxy *proxy)
> +{
> + VFIOUserMsg *r1, *r2;
> +
> + qemu_mutex_lock(&proxy->lock);
> +
> + /* our side is quitting */
> + if (proxy->state == VFIO_PROXY_CONNECTED) {
> + vfio_user_shutdown(proxy);
> + if (!QTAILQ_EMPTY(&proxy->pending)) {
> + error_printf("vfio_user_disconnect: outstanding requests\n");
> + }
> + }
> + object_unref(OBJECT(proxy->ioc));
> + proxy->ioc = NULL;
> +
> + proxy->state = VFIO_PROXY_CLOSING;
> + QTAILQ_FOREACH_SAFE(r1, &proxy->outgoing, next, r2) {
> + qemu_cond_destroy(&r1->cv);
> + QTAILQ_REMOVE(&proxy->pending, r1, next);
> + g_free(r1);
> + }
> + QTAILQ_FOREACH_SAFE(r1, &proxy->incoming, next, r2) {
> + qemu_cond_destroy(&r1->cv);
> + QTAILQ_REMOVE(&proxy->pending, r1, next);
> + g_free(r1);
> + }
> + QTAILQ_FOREACH_SAFE(r1, &proxy->pending, next, r2) {
> + qemu_cond_destroy(&r1->cv);
> + QTAILQ_REMOVE(&proxy->pending, r1, next);
> + g_free(r1);
> + }
> + QTAILQ_FOREACH_SAFE(r1, &proxy->free, next, r2) {
> + qemu_cond_destroy(&r1->cv);
> + QTAILQ_REMOVE(&proxy->free, r1, next);
> + g_free(r1);
> + }
> +
> + /*
> + * Make sure the iothread isn't blocking anywhere
> + * with a ref to this proxy by waiting for a BH
> + * handler to run after the proxy fd handlers were
> + * deleted above.
> + */
> + aio_bh_schedule_oneshot(proxy->ctx, vfio_user_cb, proxy);
> + qemu_cond_wait(&proxy->close_cv, &proxy->lock);
> +
> + /* we now hold the only ref to proxy */
> + qemu_mutex_unlock(&proxy->lock);
> + qemu_cond_destroy(&proxy->close_cv);
> + qemu_mutex_destroy(&proxy->lock);
> +
> + QLIST_REMOVE(proxy, next);
> + if (QLIST_EMPTY(&vfio_user_sockets)) {
> + iothread_destroy(vfio_user_iothread);
> + vfio_user_iothread = NULL;
> + }
> +
> + g_free(proxy->sockname);
> + g_free(proxy);
> +}
> diff --git a/MAINTAINERS b/MAINTAINERS
> index d838b9e..f429bab 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -1882,8 +1882,12 @@ L: qemu-s3...@nongnu.org
> vfio-user
> M: John G Johnson <john.g.john...@oracle.com>
> M: Thanos Makatos <thanos.maka...@nutanix.com>
> +M: Elena Ufimtseva <elena.ufimts...@oracle.com>
> +M: Jagannathan Raman <jag.ra...@oracle.com>
> S: Supported
> F: docs/devel/vfio-user.rst
> +F: hw/vfio/user.c
> +F: hw/vfio/user.h
>
> vhost
> M: Michael S. Tsirkin <m...@redhat.com>
> diff --git a/hw/vfio/meson.build b/hw/vfio/meson.build
> index da9af29..2f86f72 100644
> --- a/hw/vfio/meson.build
> +++ b/hw/vfio/meson.build
> @@ -9,6 +9,7 @@ vfio_ss.add(when: 'CONFIG_VFIO_PCI', if_true: files(
> 'pci-quirks.c',
> 'pci.c',
> ))
> +vfio_ss.add(when: 'CONFIG_VFIO_USER', if_true: files('user.c'))
> vfio_ss.add(when: 'CONFIG_VFIO_CCW', if_true: files('ccw.c'))
> vfio_ss.add(when: 'CONFIG_VFIO_PLATFORM', if_true: files('platform.c'))
> vfio_ss.add(when: 'CONFIG_VFIO_XGMAC', if_true: files('calxeda-xgmac.c'))
