The vf's unmasked msix vectors will be enable one by one in
migraiton resume phase, VFIO_DEVICE_SET_IRQS will be called
for each vector, it's a bit expensive if the vf has more
vectors.
We can call VFIO_DEVICE_SET_IRQS once outside the loop of set
vector notifiers to reduce the cost.
The test VM has 128 vcpus and 8 VF (with 65 vectors enabled),
we mesure the cost of the vfio_msix_enable for each one, and
we can see 10% costs can be reduced.
Origin Apply this patch
1st 8 4
2nd 15 11
3rd 22 18
4th 24 25
5th 36 33
6th 44 40
7th 51 47
8th 58 54
Total 258ms 232ms
Signed-off-by: Longpeng(Mike) <longpe...@huawei.com>
---
hw/vfio/pci.c | 22 ++++++++++++++++++++++
hw/vfio/pci.h | 1 +
2 files changed, 23 insertions(+)
diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
index 7cc43fe..ca37fb7 100644
--- a/hw/vfio/pci.c
+++ b/hw/vfio/pci.c
@@ -372,6 +372,10 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool
msix)
int ret = 0, i, argsz;
int32_t *fds;
+ if (!vdev->nr_vectors) {
+ return 0;
+ }
+
argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
irq_set = g_malloc0(argsz);
@@ -495,6 +499,11 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev,
unsigned int nr,
}
}
+ if (vdev->defer_add_virq) {
+ vdev->nr_vectors = MAX(vdev->nr_vectors, nr + 1);
+ goto clear_pending;
+ }
+
/*
* We don't want to have the host allocate all possible MSI vectors
* for a device if they're not in use, so we shutdown and incrementally
@@ -524,6 +533,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev,
unsigned int nr,
}
}
+clear_pending:
/* Disable PBA emulation when nothing more is pending. */
clear_bit(nr, vdev->msix->pending);
if (find_first_bit(vdev->msix->pending,
@@ -608,6 +618,16 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
vfio_msix_vector_release, NULL)) {
error_report("vfio: msix_set_vector_notifiers failed");
+ return;
+ }
+
+ if (!pdev->msix_function_masked && vdev->defer_add_virq) {
+ int ret;
+ vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
+ ret = vfio_enable_vectors(vdev, true);
+ if (ret) {
+ error_report("vfio: failed to enable vectors, %d", ret);
+ }
}
trace_vfio_msix_enable(vdev->vbasedev.name);
@@ -2456,7 +2476,9 @@ static int vfio_pci_load_config(VFIODevice *vbasedev,
QEMUFile *f)
if (msi_enabled(pdev)) {
vfio_msi_enable(vdev);
} else if (msix_enabled(pdev)) {
+ vdev->defer_add_virq = true;
vfio_msix_enable(vdev);
+ vdev->defer_add_virq = false;
}
return ret;
diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
index 6477751..4235c83 100644
--- a/hw/vfio/pci.h
+++ b/hw/vfio/pci.h
@@ -171,6 +171,7 @@ struct VFIOPCIDevice {
bool no_kvm_ioeventfd;
bool no_vfio_ioeventfd;
bool enable_ramfb;
+ bool defer_add_virq;
VFIODisplay *dpy;
Notifier irqchip_change_notifier;
};
--
1.8.3.1
