On 8/25/21 9:56 AM, Longpeng(Mike) wrote:
> The vf's unmasked msix vectors will be enable one by one in
> migraiton resume phase, VFIO_DEVICE_SET_IRQS will be called
Typo "migration"
> for each vector, it's a bit expensive if the vf has more
> vectors.
>
> We can call VFIO_DEVICE_SET_IRQS once outside the loop of set
> vector notifiers to reduce the cost.
>
> The test VM has 128 vcpus and 8 VF (with 65 vectors enabled),
> we mesure the cost of the vfio_msix_enable for each one, and
Typo "measure"
> we can see 10% costs can be reduced.
>
> Origin Apply this patch
> 1st 8 4
> 2nd 15 11
> 3rd 22 18
> 4th 24 25
> 5th 36 33
> 6th 44 40
> 7th 51 47
> 8th 58 54
> Total 258ms 232ms
>
> Signed-off-by: Longpeng(Mike) <longpe...@huawei.com>
> ---
> hw/vfio/pci.c | 22 ++++++++++++++++++++++
> hw/vfio/pci.h | 1 +
> 2 files changed, 23 insertions(+)
>
> diff --git a/hw/vfio/pci.c b/hw/vfio/pci.c
> index 7cc43fe..ca37fb7 100644
> --- a/hw/vfio/pci.c
> +++ b/hw/vfio/pci.c
> @@ -372,6 +372,10 @@ static int vfio_enable_vectors(VFIOPCIDevice *vdev, bool
> msix)
> int ret = 0, i, argsz;
> int32_t *fds;
>
> + if (!vdev->nr_vectors) {
> + return 0;
> + }
> +
> argsz = sizeof(*irq_set) + (vdev->nr_vectors * sizeof(*fds));
>
> irq_set = g_malloc0(argsz);
> @@ -495,6 +499,11 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev,
> unsigned int nr,
> }
> }
>
> + if (vdev->defer_add_virq) {
> + vdev->nr_vectors = MAX(vdev->nr_vectors, nr + 1);
> + goto clear_pending;
> + }
> +
> /*
> * We don't want to have the host allocate all possible MSI vectors
> * for a device if they're not in use, so we shutdown and incrementally
> @@ -524,6 +533,7 @@ static int vfio_msix_vector_do_use(PCIDevice *pdev,
> unsigned int nr,
> }
> }
>
> +clear_pending:
> /* Disable PBA emulation when nothing more is pending. */
> clear_bit(nr, vdev->msix->pending);
> if (find_first_bit(vdev->msix->pending,
> @@ -608,6 +618,16 @@ static void vfio_msix_enable(VFIOPCIDevice *vdev)
> if (msix_set_vector_notifiers(pdev, vfio_msix_vector_use,
> vfio_msix_vector_release, NULL)) {
> error_report("vfio: msix_set_vector_notifiers failed");
> + return;
> + }
> +
> + if (!pdev->msix_function_masked && vdev->defer_add_virq) {
> + int ret;
> + vfio_disable_irqindex(&vdev->vbasedev, VFIO_PCI_MSIX_IRQ_INDEX);
> + ret = vfio_enable_vectors(vdev, true);
> + if (ret) {
> + error_report("vfio: failed to enable vectors, %d", ret);
> + }
> }
>
> trace_vfio_msix_enable(vdev->vbasedev.name);
> @@ -2456,7 +2476,9 @@ static int vfio_pci_load_config(VFIODevice *vbasedev,
> QEMUFile *f)
> if (msi_enabled(pdev)) {
> vfio_msi_enable(vdev);
> } else if (msix_enabled(pdev)) {
> + vdev->defer_add_virq = true;
> vfio_msix_enable(vdev);
What about passing defer_add_virq as boolean argument
to vfio_msix_enable()?
> + vdev->defer_add_virq = false;
> }
>
> return ret;
> diff --git a/hw/vfio/pci.h b/hw/vfio/pci.h
> index 6477751..4235c83 100644
> --- a/hw/vfio/pci.h
> +++ b/hw/vfio/pci.h
> @@ -171,6 +171,7 @@ struct VFIOPCIDevice {
> bool no_kvm_ioeventfd;
> bool no_vfio_ioeventfd;
> bool enable_ramfb;
> + bool defer_add_virq;
> VFIODisplay *dpy;
> Notifier irqchip_change_notifier;
> };
>
