On Mon, May 8, 2023 at 8:28 AM Jason Wang <jasow...@redhat.com> wrote:
>
> On Mon, May 8, 2023 at 1:25 PM Jason Wang <jasow...@redhat.com> wrote:
> >
> > On Mon, May 1, 2023 at 10:02 AM Viktor Prutyanov <vik...@daynix.com> wrote:
> > >
> > > The guest can disable or never enable Device-TLB. In these cases,
> > > it can't be used even if enabled in QEMU. So, check Device-TLB state
> > > before registering IOMMU notifier and select unmap flag depending on
> > > that. Also, implement a way to change IOMMU notifier flag if Device-TLB
> > > state is changed.
> > >
> > > Buglink: https://bugzilla.redhat.com/show_bug.cgi?id=2001312
> > > Signed-off-by: Viktor Prutyanov <vik...@daynix.com>
> > > ---
> > > hw/virtio/vhost-backend.c | 6 ++++++
> > > hw/virtio/vhost.c | 26 ++++++++++++++++++++++++--
> > > include/hw/virtio/vhost-backend.h | 4 ++++
> > > include/hw/virtio/vhost.h | 1 +
> > > 4 files changed, 35 insertions(+), 2 deletions(-)
> > >
> > > diff --git a/hw/virtio/vhost-backend.c b/hw/virtio/vhost-backend.c
> > > index 8e581575c9..30eb71fb83 100644
> > > --- a/hw/virtio/vhost-backend.c
> > > +++ b/hw/virtio/vhost-backend.c
> > > @@ -297,6 +297,11 @@ static void vhost_kernel_set_iotlb_callback(struct
> > > vhost_dev *dev,
> > > qemu_set_fd_handler((uintptr_t)dev->opaque, NULL, NULL, NULL);
> > > }
> > >
> > > +static void vhost_kernel_toggle_device_iotlb(struct vhost_dev *dev, int
> > > enable)
> > > +{
> > > + vhost_toggle_device_iotlb(dev, enable);
> > > +}
> > > +
> > > const VhostOps kernel_ops = {
> > > .backend_type = VHOST_BACKEND_TYPE_KERNEL,
> > > .vhost_backend_init = vhost_kernel_init,
> > > @@ -328,6 +333,7 @@ const VhostOps kernel_ops = {
> > > .vhost_vsock_set_running = vhost_kernel_vsock_set_running,
> > > .vhost_set_iotlb_callback = vhost_kernel_set_iotlb_callback,
> > > .vhost_send_device_iotlb_msg =
> > > vhost_kernel_send_device_iotlb_msg,
> > > + .vhost_toggle_device_iotlb = vhost_kernel_toggle_device_iotlb,
> > > };
> > > #endif
> > >
> > > diff --git a/hw/virtio/vhost.c b/hw/virtio/vhost.c
> > > index a266396576..1bfcc6d263 100644
> > > --- a/hw/virtio/vhost.c
> > > +++ b/hw/virtio/vhost.c
> > > @@ -796,7 +796,9 @@ static void vhost_iommu_region_add(MemoryListener
> > > *listener,
> > > iommu_idx = memory_region_iommu_attrs_to_index(iommu_mr,
> > >
> > > MEMTXATTRS_UNSPECIFIED);
> > > iommu_notifier_init(&iommu->n, vhost_iommu_unmap_notify,
> > > - IOMMU_NOTIFIER_DEVIOTLB_UNMAP,
> > > + dev->vdev->device_iotlb_enabled ?
> > > + IOMMU_NOTIFIER_DEVIOTLB_UNMAP :
> > > + IOMMU_NOTIFIER_UNMAP,
> > > section->offset_within_region,
> > > int128_get64(end),
> > > iommu_idx);
> > > @@ -804,7 +806,8 @@ static void vhost_iommu_region_add(MemoryListener
> > > *listener,
> > > iommu->iommu_offset = section->offset_within_address_space -
> > > section->offset_within_region;
> > > iommu->hdev = dev;
> > > - ret = memory_region_register_iommu_notifier(section->mr, &iommu->n,
> > > NULL);
> > > + ret = memory_region_register_iommu_notifier(section->mr, &iommu->n,
> > > + dev->vdev->device_iotlb_enabled ? NULL : &error_fatal);
> > > if (ret) {
> > > /*
> > > * Some vIOMMUs do not support dev-iotlb yet. If so, try to use
> > > the
> > > @@ -841,6 +844,25 @@ static void vhost_iommu_region_del(MemoryListener
> > > *listener,
> > > }
> > > }
> > >
> > > +void vhost_toggle_device_iotlb(struct vhost_dev *dev, bool enable)
> > > +{
> > > + struct vhost_iommu *iommu;
> > > + int ret;
> > > +
> > > + QLIST_FOREACH(iommu, &dev->iommu_list, iommu_next) {
> > > + memory_region_unregister_iommu_notifier(iommu->mr, &iommu->n);
> > > + iommu->n.notifier_flags = enable ?
> > > + IOMMU_NOTIFIER_DEVIOTLB_UNMAP : IOMMU_NOTIFIER_UNMAP;
> > > + ret = memory_region_register_iommu_notifier(iommu->mr, &iommu->n,
> > > + enable ? NULL : &error_fatal);
> > > + if (ret) {
> > > + iommu->n.notifier_flags = IOMMU_NOTIFIER_UNMAP;
> > > + memory_region_register_iommu_notifier(iommu->mr, &iommu->n,
> > > + &error_fatal);
> >
> > I think it's better to tweak the code to avoid doing IOMMU_NOTIFIER_UNMAP
> > twice.
> >
> > The rest looks good.
>
> Btw, it might worth to add comment to explain why we need this fallback.
>
> Actually, I'm not sure I understand the logic.
>
> E.g if guest tries to enable ATS it means it knows there's a vIOMMU
> that support device IOTLB. If we use UNMAP notifier, we will lose the
> device device IOTLB event here?
Yes. So, the fallback is not really needed anymore. It can't help if
the guest is going to use Device-TLB (by enabling ATS) but it isn't
available in emulated IOMMU.
Thanks,
Viktor Prutyanov
>
> Thanks
>
> >
> > Thanks
> >
> > > + }
> > > + }
> > > +}
> > > +
> > > static int vhost_virtqueue_set_addr(struct vhost_dev *dev,
> > > struct vhost_virtqueue *vq,
> > > unsigned idx, bool enable_log)
> > > diff --git a/include/hw/virtio/vhost-backend.h
> > > b/include/hw/virtio/vhost-backend.h
> > > index ec3fbae58d..f8e9660a96 100644
> > > --- a/include/hw/virtio/vhost-backend.h
> > > +++ b/include/hw/virtio/vhost-backend.h
> > > @@ -133,6 +133,9 @@ typedef int (*vhost_set_config_call_op)(struct
> > > vhost_dev *dev,
> > >
> > > typedef void (*vhost_reset_status_op)(struct vhost_dev *dev);
> > >
> > > +typedef void (*vhost_toggle_device_iotlb_op)(struct vhost_dev *dev,
> > > + int enabled);
> > > +
> > > typedef struct VhostOps {
> > > VhostBackendType backend_type;
> > > vhost_backend_init vhost_backend_init;
> > > @@ -181,6 +184,7 @@ typedef struct VhostOps {
> > > vhost_force_iommu_op vhost_force_iommu;
> > > vhost_set_config_call_op vhost_set_config_call;
> > > vhost_reset_status_op vhost_reset_status;
> > > + vhost_toggle_device_iotlb_op vhost_toggle_device_iotlb;
> > > } VhostOps;
> > >
> > > int vhost_backend_update_device_iotlb(struct vhost_dev *dev,
> > > diff --git a/include/hw/virtio/vhost.h b/include/hw/virtio/vhost.h
> > > index a52f273347..b3f585c6cd 100644
> > > --- a/include/hw/virtio/vhost.h
> > > +++ b/include/hw/virtio/vhost.h
> > > @@ -320,6 +320,7 @@ bool vhost_has_free_slot(void);
> > > int vhost_net_set_backend(struct vhost_dev *hdev,
> > > struct vhost_vring_file *file);
> > >
> > > +void vhost_toggle_device_iotlb(struct vhost_dev *dev, bool enable);
> > > int vhost_device_iotlb_miss(struct vhost_dev *dev, uint64_t iova, int
> > > write);
> > >
> > > int vhost_virtqueue_start(struct vhost_dev *dev, struct VirtIODevice
> > > *vdev,
> > > --
> > > 2.35.1
> > >
>
