Hi,
Apologize for failing to make this email in short. Log is too long.
If the following statements failed to make me understood, you
are welcomed to ping me.
1. Problem description:
Recently, I find a strange thing with vIOMMU in QEMU. It looks like
g_hash_table_lookup() is not 100% give same result when the key
is the same. And this results in redundant memory allocation.
I'm not sure if it is expected. Has anyone else encountered it?
This potential issue is in vtd_find_add_as(), it uses the input PCIBus
pointer to lookup hash table and get a vtd_bus back. If no hit,
allocate one and inset it to hash_table. In my understanding, a pci
bus only requires a single vtd_bus structure. But what I observed
denied it.
2. Command to start guest:
x86_64-softmmu/qemu-system-x86_64 -boot c
-hda /home/sky/vms/vm-for-svm/svm-disk.img
-m 5120 -enable-kvm -machine type=q35 -device intel-iommu
-net nic -net tap,ifname=tap0, script=no,downscript=no -device
vfio-pci,host=00:02.0,id=hostdev0,addr=0x6
3. Log:
a) First enter of vtd_find_add_as(), no corresponding vtd_bus in
s->vtd_as_by_busptr, so allocate one, this is quite reasonable.
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0x0
lookup result: no vtd_bus, allocate one
vtd_bus: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
lookup hash_table again, vtd_bus: 0x56067457a620
b) Second enter of vtd_find_add_as(), again no corresponding
vtd_bus in s->vtd_as_by_busptr, so allocate one, this is strange
since devfn: 0x30 is actually also under pci bus 0.
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0x30
lookup result: no vtd_bus, allocate one
vtd_bus: 0x56067598fef0
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067598fef0
lookup hash_table again, vtd_bus: 0x56067598fef0
c) Third enter of vtd_find_add_as(),no corresponding vtd_bus
in s->vtd_as_by_busptr, so allocate one, this also strange since
there should have a vtd_bus in hash table for pci bus 0.
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0x0
lookup result: no vtd_bus, allocate one
vtd_bus: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067598fef0
lookup hash_table again, vtd_bus: 0x5606746aa400
d) reset seems to be correct, however, there is totally 3 vtd_bus
structure for pci bus 0. this is a waste of memory in my
understanding.
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0x8
lookup result: got vtd_bus
vtd_bus: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067598fef0
lookup hash_table again, vtd_bus: 0x5606746aa400
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0x10
lookup result: got vtd_bus
vtd_bus: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067598fef0
lookup hash_table again, vtd_bus: 0x5606746aa400
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0x30
lookup result: got vtd_bus
vtd_bus: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067598fef0
lookup hash_table again, vtd_bus: 0x5606746aa400
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0xf8
lookup result: got vtd_bus
vtd_bus: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067598fef0
lookup hash_table again, vtd_bus: 0x5606746aa400
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0xfa
lookup result: got vtd_bus
vtd_bus: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067598fef0
lookup hash_table again, vtd_bus: 0x5606746aa400
------------------------------
YiLiu - vtd_find_add_as()
bus: 0x5606747f99c0
s: 0x560675a2e000
s->vtd_as_by_busptr: 0x560675c521e0, devfn: 0xfb
lookup result: got vtd_bus
vtd_bus: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x5606746aa400
vtd_bus in s->vtd_as_by_busptr: 0x56067457a620
vtd_bus in s->vtd_as_by_busptr: 0x56067598fef0
lookup hash_table again, vtd_bus: 0x5606746aa400
4. QEMU version:
commit 48f592118ab42f83a1a7561c4bfd2b72a100f241
Author: Ed Maste <ema...@freebsd.org>
Date: Tue Oct 4 16:02:49 2016 -0400
bsd-user: fix FreeBSD build after d148d90e
Signed-off-by: Ed Maste <ema...@freebsd.org>
Message-id: 1475611369-74971-1-git-send-email-ema...@freebsd.org
Signed-off-by: Peter Maydell <peter.mayd...@linaro.org>
5. Debug code:
diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
index 9f4e64a..537d2d4 100644
--- a/hw/i386/intel_iommu.c
+++ b/hw/i386/intel_iommu.c
@@ -1986,7 +1986,7 @@ static void vtd_iommu_notify_flag_changed(MemoryRegion
*iommu,
"intel-iommu emulation",
vtd_as->bus->qbus.name, PCI_SLOT(vtd_as->devfn),
PCI_FUNC(vtd_as->devfn));
- exit(1);
+ // exit(1);//Yi: comment out for test purpose
}
}
@@ -2310,15 +2310,33 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s,
PCIBus *bus, int devfn)
VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
VTDAddressSpace *vtd_dev_as;
+ //Yi: add for debug
+ printf("------------------------------\n"
+ "YiLiu - %s()\n"
+ " bus: 0x%llx\n"
+ " s: 0x%llx\n"
+ " s->vtd_as_by_busptr: 0x%llx, "
+ "devfn: 0x%x\n",
+ __FUNCTION__,
+ (unsigned long long int) bus,
+ (unsigned long long int) s,
+ (unsigned long long int) s->vtd_as_by_busptr,
+ (unsigned int) devfn);
+
if (!vtd_bus) {
+ printf(" lookup result: no vtd_bus, allocate one\n");
/* No corresponding free() */
vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * \
X86_IOMMU_PCI_DEVFN_MAX);
vtd_bus->bus = bus;
key = (uintptr_t)bus;
g_hash_table_insert(s->vtd_as_by_busptr, &key, vtd_bus);
+ } else{
+ printf(" lookup result: got vtd_bus\n");
}
+ printf(" vtd_bus: 0x%llx\n", (unsigned long long int) vtd_bus);
+
vtd_dev_as = vtd_bus->dev_as[devfn];
if (!vtd_dev_as) {
@@ -2338,6 +2356,26 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s,
PCIBus *bus, int devfn)
address_space_init(&vtd_dev_as->as,
&vtd_dev_as->iommu, "intel_iommu");
}
+
+ {
+ //Yi: code snippet for debug, dump the vtd_as_by_buspt
+ // r to see if all the allocated vtd_bus are still there
+ GHashTableIter iter;
+ g_hash_table_iter_init(&iter, s->vtd_as_by_busptr);
+ while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) {
+ if (pci_bus_num(vtd_bus->bus) == pci_bus_num(bus)) {
+ printf(" vtd_bus in s->vtd_as_by_busptr: 0x%llx\n",
+ (unsigned long long int) vtd_bus);
+ }
+ }
+ // look up hash table again see if the result is the same with the
+ // beginning one.
+ vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key);
+ printf(" lookup hash_table again"
+ ", vtd_bus: 0x%llx\n",
+ (unsigned long long int) vtd_bus);
+ }
+
return vtd_dev_as;
}
diff --git a/hw/pci/pcie.c b/hw/pci/pcie.c
index 99cfb45..6c83fe9 100644
--- a/hw/pci/pcie.c
+++ b/hw/pci/pcie.c
@@ -683,7 +683,7 @@ void pcie_add_capability(PCIDevice *dev,
in the linked list */
next = pcie_find_capability_list(dev, 0, &prev);
- assert(prev >= PCI_CONFIG_SPACE_SIZE);
+ //assert(prev >= PCI_CONFIG_SPACE_SIZE);
assert(next == 0);
pcie_ext_cap_set_next(dev, prev, offset);
}
diff --git a/hw/vfio/common.c b/hw/vfio/common.c
index 29188a1..3e52d39 100644
--- a/hw/vfio/common.c
+++ b/hw/vfio/common.c
@@ -457,7 +457,7 @@ static void vfio_listener_region_add(MemoryListener
*listener,
QLIST_INSERT_HEAD(&container->giommu_list, giommu, giommu_next);
memory_region_register_iommu_notifier(giommu->iommu, &giommu->n);
- memory_region_iommu_replay(giommu->iommu, &giommu->n, false);
+ //memory_region_iommu_replay(giommu->iommu, &giommu->n, false);
return;
}
Best Wishes,
Yi Liu
