On Sat, 21 May 2016 19:19:50 +0300
"Aviv B.D" <bd.a...@gmail.com> wrote:
> From: "Aviv Ben-David" <bd.a...@gmail.com>
>
Some commentary about the changes necessary to achieve $SUBJECT would
be nice here.
> Signed-off-by: Aviv Ben-David <bd.a...@gmail.com>
> ---
> hw/i386/intel_iommu.c | 69
> ++++++++++++++++++++++++++++++++++++++++--
> hw/i386/intel_iommu_internal.h | 2 ++
> hw/vfio/common.c | 11 +++++--
> include/hw/i386/intel_iommu.h | 4 +++
> include/hw/vfio/vfio-common.h | 1 +
> 5 files changed, 81 insertions(+), 6 deletions(-)
>
> diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
> index 410f810..128ec7c 100644
> --- a/hw/i386/intel_iommu.c
> +++ b/hw/i386/intel_iommu.c
> @@ -43,6 +43,9 @@ static int vtd_dbgflags = VTD_DBGBIT(GENERAL) |
> VTD_DBGBIT(CSR);
> #define VTD_DPRINTF(what, fmt, ...) do {} while (0)
> #endif
>
> +static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num,
> + uint8_t devfn, VTDContextEntry *ce);
> +
> static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val,
> uint64_t wmask, uint64_t w1cmask)
> {
> @@ -126,6 +129,22 @@ static uint32_t vtd_set_clear_mask_long(IntelIOMMUState
> *s, hwaddr addr,
> return new_val;
> }
>
> +static int vtd_get_did_dev(IntelIOMMUState *s, uint8_t bus_num, uint8_t
> devfn, uint16_t * domain_id)
> +{
> + VTDContextEntry ce;
> + int ret_fr;
> +
> + assert(domain_id);
> +
> + ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce);
> + if (ret_fr){
> + return -1;
> + }
> +
> + *domain_id = VTD_CONTEXT_ENTRY_DID(ce.hi);
> + return 0;
> +}
> +
> static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr,
> uint64_t clear, uint64_t mask)
> {
> @@ -724,9 +743,6 @@ static int vtd_dev_to_context_entry(IntelIOMMUState *s,
> uint8_t bus_num,
> }
>
> if (!vtd_context_entry_present(ce)) {
> - VTD_DPRINTF(GENERAL,
> - "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") "
> - "is not present", devfn, bus_num);
> return -VTD_FR_CONTEXT_ENTRY_P;
> } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) ||
> (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) {
> @@ -1033,18 +1049,58 @@ static void
> vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id)
> &domain_id);
> }
>
> +static void vtd_iotlb_page_invalidate_vfio(IntelIOMMUState *s, uint16_t
> domain_id,
> + hwaddr addr, uint8_t am)
> +{
> + VFIOGuestIOMMU * giommu;
> +
VT-d parsing VFIO private data structures, nope this is not a good idea.
> + QLIST_FOREACH(giommu, &(s->giommu_list), iommu_next){
> + VTDAddressSpace *vtd_as = container_of(giommu->iommu,
> VTDAddressSpace, iommu);
VT-d needs to keep track of its own address spaces and call the iommu
notifier, it should have no visibility whatsoever that there are vfio
devices attached.
> + uint16_t vfio_domain_id;
> + int ret = vtd_get_did_dev(s, pci_bus_num(vtd_as->bus),
> vtd_as->devfn, &vfio_domain_id);
> + int i=0;
> + if (!ret && domain_id == vfio_domain_id){
> + IOMMUTLBEntry entry;
> +
> + /* do vfio unmap */
> + VTD_DPRINTF(GENERAL, "Remove addr 0x%"PRIx64 " mask %d", addr,
> am);
> + entry.target_as = NULL;
> + entry.iova = addr & VTD_PAGE_MASK_4K;
> + entry.translated_addr = 0;
> + entry.addr_mask = ~VTD_PAGE_MASK(VTD_PAGE_SHIFT_4K + am);
> + entry.perm = IOMMU_NONE;
> + memory_region_notify_iommu(giommu->iommu, entry);
> +
> + /* do vfio map */
> + VTD_DPRINTF(GENERAL, "add addr 0x%"PRIx64 " mask %d", addr, am);
> + /* call to vtd_iommu_translate */
> + for (i = 0; i < (1 << am); i++, addr+=(1 << VTD_PAGE_SHIFT_4K)){
> + IOMMUTLBEntry entry = s->iommu_ops.translate(giommu->iommu,
> addr, IOMMU_NO_FAIL);
> + if (entry.perm != IOMMU_NONE){
> + memory_region_notify_iommu(giommu->iommu, entry);
> + }
> + }
> + }
> + }
> +}
> +
> static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id,
> hwaddr addr, uint8_t am)
> {
> VTDIOTLBPageInvInfo info;
>
> assert(am <= VTD_MAMV);
> +
> info.domain_id = domain_id;
> info.addr = addr;
> info.mask = ~((1 << am) - 1);
> +
> g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info);
> +
> + vtd_iotlb_page_invalidate_vfio(s, domain_id, addr, am);
> }
>
> +
> /* Flush IOTLB
> * Returns the IOTLB Actual Invalidation Granularity.
> * @val: the content of the IOTLB_REG
> @@ -1912,6 +1968,13 @@ static Property vtd_properties[] = {
> DEFINE_PROP_END_OF_LIST(),
> };
>
> +void vtd_register_giommu(VFIOGuestIOMMU * giommu)
> +{
> + VTDAddressSpace *vtd_as = container_of(giommu->iommu, VTDAddressSpace,
> iommu);
> + IntelIOMMUState *s = vtd_as->iommu_state;
> +
> + QLIST_INSERT_HEAD(&s->giommu_list, giommu, iommu_next);
> +}
>
> VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn)
> {
> diff --git a/hw/i386/intel_iommu_internal.h b/hw/i386/intel_iommu_internal.h
> index ae40f73..102e9a5 100644
> --- a/hw/i386/intel_iommu_internal.h
> +++ b/hw/i386/intel_iommu_internal.h
> @@ -339,6 +339,8 @@ typedef struct VTDIOTLBPageInvInfo VTDIOTLBPageInvInfo;
> #define VTD_PAGE_SHIFT_1G 30
> #define VTD_PAGE_MASK_1G (~((1ULL << VTD_PAGE_SHIFT_1G) - 1))
>
> +#define VTD_PAGE_MASK(shift) (~((1ULL << (shift)) - 1))
> +
> struct VTDRootEntry {
> uint64_t val;
> uint64_t rsvd;
> diff --git a/hw/vfio/common.c b/hw/vfio/common.c
> index 88154a1..54fc8bc 100644
> --- a/hw/vfio/common.c
> +++ b/hw/vfio/common.c
> @@ -35,6 +35,9 @@
> #endif
> #include "trace.h"
>
> +#include "hw/sysbus.h"
> +#include "hw/i386/intel_iommu.h"
> +
> struct vfio_group_head vfio_group_list =
> QLIST_HEAD_INITIALIZER(vfio_group_list);
> struct vfio_as_head vfio_address_spaces =
> @@ -315,12 +318,12 @@ static void vfio_iommu_map_notify(Notifier *n, void
> *data)
> out:
> rcu_read_unlock();
> }
> -
> +#if 0
> static hwaddr vfio_container_granularity(VFIOContainer *container)
> {
> return (hwaddr)1 << ctz64(container->iova_pgsizes);
> }
> -
> +#endif
Clearly this is unacceptable, the code has a purpose.
> static void vfio_listener_region_add(MemoryListener *listener,
> MemoryRegionSection *section)
> {
> @@ -384,11 +387,13 @@ static void vfio_listener_region_add(MemoryListener
> *listener,
> giommu->n.notify = vfio_iommu_map_notify;
> QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
>
> + vtd_register_giommu(giommu);
vfio will not assume VT-d, this is why we register the notifier below.
> memory_region_register_iommu_notifier(giommu->iommu, &giommu->n);
> +#if 0
> memory_region_iommu_replay(giommu->iommu, &giommu->n,
> vfio_container_granularity(container),
> false);
> -
> +#endif
Clearly this also has a purpose.
> return;
> }
>
> diff --git a/include/hw/i386/intel_iommu.h b/include/hw/i386/intel_iommu.h
> index b024ffa..22f3f83 100644
> --- a/include/hw/i386/intel_iommu.h
> +++ b/include/hw/i386/intel_iommu.h
> @@ -23,6 +23,7 @@
> #define INTEL_IOMMU_H
> #include "hw/qdev.h"
> #include "sysemu/dma.h"
> +#include "hw/vfio/vfio-common.h"
No. This header probably should not have been put under include, VT-d
has no business walking our guest IOMMU list.
>
> #define TYPE_INTEL_IOMMU_DEVICE "intel-iommu"
> #define INTEL_IOMMU_DEVICE(obj) \
> @@ -123,6 +124,8 @@ struct IntelIOMMUState {
> MemoryRegionIOMMUOps iommu_ops;
> GHashTable *vtd_as_by_busptr; /* VTDBus objects indexed by PCIBus*
> reference */
> VTDBus *vtd_as_by_bus_num[VTD_PCI_BUS_MAX]; /* VTDBus objects indexed by
> bus number */
> +
> + QLIST_HEAD(, VFIOGuestIOMMU) giommu_list;
> };
>
> /* Find the VTD Address space associated with the given bus pointer,
> @@ -130,4 +133,5 @@ struct IntelIOMMUState {
> */
> VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn);
>
> +void vtd_register_giommu(VFIOGuestIOMMU * giommu);
> #endif
> diff --git a/include/hw/vfio/vfio-common.h b/include/hw/vfio/vfio-common.h
> index eb0e1b0..bf56a1d 100644
> --- a/include/hw/vfio/vfio-common.h
> +++ b/include/hw/vfio/vfio-common.h
> @@ -92,6 +92,7 @@ typedef struct VFIOGuestIOMMU {
> MemoryRegion *iommu;
> Notifier n;
> QLIST_ENTRY(VFIOGuestIOMMU) giommu_next;
> + QLIST_ENTRY(VFIOGuestIOMMU) iommu_next;
No. Use the existing interfaces, create your own address space
tracking in VT-d, we are not going to host a list for VT-d to use.
Also note that there's no consideration of hot-unplug support in these
changes. vfio already works with guest iommus on powerpc, so any
change to vfio needs to be justified and generalized to a common
guest iommu api. Thanks,
Alex
> } VFIOGuestIOMMU;
>
> typedef struct VFIODeviceOps VFIODeviceOps;
