Author: jhb Date: Tue Nov 24 23:18:52 2020 New Revision: 368003 URL: https://svnweb.freebsd.org/changeset/base/368003
Log: Honor the disabled setting for MSI-X interrupts for passthrough devices. Add a new ioctl to disable all MSI-X interrupts for a PCI passthrough device and invoke it if a write to the MSI-X capability registers disables MSI-X. This avoids leaving MSI-X interrupts enabled on the host if a guest device driver has disabled them (e.g. as part of detaching a guest device driver). This was found by Chelsio QA when testing that a Linux guest could switch from MSI-X to MSI interrupts when using the cxgb4vf driver. While here, explicitly fail requests to enable MSI on a passthrough device if MSI-X is enabled and vice versa. Reported by: Sony Arpita Das @ Chelsio Reviewed by: grehan, markj MFC after: 2 weeks Sponsored by: Chelsio Communications Differential Revision: https://reviews.freebsd.org/D27212 Modified: head/lib/libvmmapi/vmmapi.c head/lib/libvmmapi/vmmapi.h head/sys/amd64/include/vmm_dev.h head/sys/amd64/vmm/io/ppt.c head/sys/amd64/vmm/io/ppt.h head/sys/amd64/vmm/vmm_dev.c head/usr.sbin/bhyve/pci_passthru.c Modified: head/lib/libvmmapi/vmmapi.c ============================================================================== --- head/lib/libvmmapi/vmmapi.c Tue Nov 24 22:52:12 2020 (r368002) +++ head/lib/libvmmapi/vmmapi.c Tue Nov 24 23:18:52 2020 (r368003) @@ -1017,6 +1017,19 @@ vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int return ioctl(ctx->fd, VM_PPTDEV_MSIX, &pptmsix); } +int +vm_disable_pptdev_msix(struct vmctx *ctx, int bus, int slot, int func) +{ + struct vm_pptdev ppt; + + bzero(&ppt, sizeof(ppt)); + ppt.bus = bus; + ppt.slot = slot; + ppt.func = func; + + return ioctl(ctx->fd, VM_PPTDEV_DISABLE_MSIX, &ppt); +} + uint64_t * vm_get_stats(struct vmctx *ctx, int vcpu, struct timeval *ret_tv, int *ret_entries) @@ -1641,7 +1654,8 @@ vm_get_ioctls(size_t *len) VM_ISA_DEASSERT_IRQ, VM_ISA_PULSE_IRQ, VM_ISA_SET_IRQ_TRIGGER, VM_SET_CAPABILITY, VM_GET_CAPABILITY, VM_BIND_PPTDEV, VM_UNBIND_PPTDEV, VM_MAP_PPTDEV_MMIO, VM_PPTDEV_MSI, - VM_PPTDEV_MSIX, VM_INJECT_NMI, VM_STATS, VM_STAT_DESC, + VM_PPTDEV_MSIX, VM_PPTDEV_DISABLE_MSIX, + VM_INJECT_NMI, VM_STATS, VM_STAT_DESC, VM_SET_X2APIC_STATE, VM_GET_X2APIC_STATE, VM_GET_HPET_CAPABILITIES, VM_GET_GPA_PMAP, VM_GLA2GPA, VM_GLA2GPA_NOFAULT, Modified: head/lib/libvmmapi/vmmapi.h ============================================================================== --- head/lib/libvmmapi/vmmapi.h Tue Nov 24 22:52:12 2020 (r368002) +++ head/lib/libvmmapi/vmmapi.h Tue Nov 24 23:18:52 2020 (r368003) @@ -181,6 +181,7 @@ int vm_setup_pptdev_msi(struct vmctx *ctx, int vcpu, i int vm_setup_pptdev_msix(struct vmctx *ctx, int vcpu, int bus, int slot, int func, int idx, uint64_t addr, uint64_t msg, uint32_t vector_control); +int vm_disable_pptdev_msix(struct vmctx *ctx, int bus, int slot, int func); int vm_get_intinfo(struct vmctx *ctx, int vcpu, uint64_t *i1, uint64_t *i2); int vm_set_intinfo(struct vmctx *ctx, int vcpu, uint64_t exit_intinfo); Modified: head/sys/amd64/include/vmm_dev.h ============================================================================== --- head/sys/amd64/include/vmm_dev.h Tue Nov 24 22:52:12 2020 (r368002) +++ head/sys/amd64/include/vmm_dev.h Tue Nov 24 23:18:52 2020 (r368003) @@ -301,6 +301,7 @@ enum { IOCNUM_MAP_PPTDEV_MMIO = 42, IOCNUM_PPTDEV_MSI = 43, IOCNUM_PPTDEV_MSIX = 44, + IOCNUM_PPTDEV_DISABLE_MSIX = 45, /* statistics */ IOCNUM_VM_STATS = 50, @@ -413,6 +414,8 @@ enum { _IOW('v', IOCNUM_PPTDEV_MSI, struct vm_pptdev_msi) #define VM_PPTDEV_MSIX \ _IOW('v', IOCNUM_PPTDEV_MSIX, struct vm_pptdev_msix) +#define VM_PPTDEV_DISABLE_MSIX \ + _IOW('v', IOCNUM_PPTDEV_DISABLE_MSIX, struct vm_pptdev) #define VM_INJECT_NMI \ _IOW('v', IOCNUM_INJECT_NMI, struct vm_nmi) #define VM_STATS \ Modified: head/sys/amd64/vmm/io/ppt.c ============================================================================== --- head/sys/amd64/vmm/io/ppt.c Tue Nov 24 22:52:12 2020 (r368002) +++ head/sys/amd64/vmm/io/ppt.c Tue Nov 24 23:18:52 2020 (r368003) @@ -518,6 +518,10 @@ ppt_setup_msi(struct vm *vm, int vcpu, int bus, int sl if (ppt->vm != vm) /* Make sure we own this device */ return (EBUSY); + /* Reject attempts to enable MSI while MSI-X is active. */ + if (ppt->msix.num_msgs != 0 && numvec != 0) + return (EBUSY); + /* Free any allocated resources */ ppt_teardown_msi(ppt); @@ -607,6 +611,10 @@ ppt_setup_msix(struct vm *vm, int vcpu, int bus, int s if (ppt->vm != vm) /* Make sure we own this device */ return (EBUSY); + /* Reject attempts to enable MSI-X while MSI is active. */ + if (ppt->msi.num_msgs != 0) + return (EBUSY); + dinfo = device_get_ivars(ppt->dev); if (!dinfo) return (ENXIO); @@ -698,5 +706,20 @@ ppt_setup_msix(struct vm *vm, int vcpu, int bus, int s ppt_teardown_msix_intr(ppt, idx); } + return (0); +} + +int +ppt_disable_msix(struct vm *vm, int bus, int slot, int func) +{ + struct pptdev *ppt; + + ppt = ppt_find(bus, slot, func); + if (ppt == NULL) + return (ENOENT); + if (ppt->vm != vm) /* Make sure we own this device */ + return (EBUSY); + + ppt_teardown_msix(ppt); return (0); } Modified: head/sys/amd64/vmm/io/ppt.h ============================================================================== --- head/sys/amd64/vmm/io/ppt.h Tue Nov 24 22:52:12 2020 (r368002) +++ head/sys/amd64/vmm/io/ppt.h Tue Nov 24 23:18:52 2020 (r368003) @@ -38,6 +38,7 @@ int ppt_setup_msi(struct vm *vm, int vcpu, int bus, in uint64_t addr, uint64_t msg, int numvec); int ppt_setup_msix(struct vm *vm, int vcpu, int bus, int slot, int func, int idx, uint64_t addr, uint64_t msg, uint32_t vector_control); +int ppt_disable_msix(struct vm *vm, int bus, int slot, int func); int ppt_assigned_devices(struct vm *vm); bool ppt_is_mmio(struct vm *vm, vm_paddr_t gpa); Modified: head/sys/amd64/vmm/vmm_dev.c ============================================================================== --- head/sys/amd64/vmm/vmm_dev.c Tue Nov 24 22:52:12 2020 (r368002) +++ head/sys/amd64/vmm/vmm_dev.c Tue Nov 24 23:18:52 2020 (r368003) @@ -514,6 +514,11 @@ vmmdev_ioctl(struct cdev *cdev, u_long cmd, caddr_t da pptmsix->addr, pptmsix->msg, pptmsix->vector_control); break; + case VM_PPTDEV_DISABLE_MSIX: + pptdev = (struct vm_pptdev *)data; + error = ppt_disable_msix(sc->vm, pptdev->bus, pptdev->slot, + pptdev->func); + break; case VM_MAP_PPTDEV_MMIO: pptmmio = (struct vm_pptdev_mmio *)data; error = ppt_map_mmio(sc->vm, pptmmio->bus, pptmmio->slot, Modified: head/usr.sbin/bhyve/pci_passthru.c ============================================================================== --- head/usr.sbin/bhyve/pci_passthru.c Tue Nov 24 22:52:12 2020 (r368002) +++ head/usr.sbin/bhyve/pci_passthru.c Tue Nov 24 23:18:52 2020 (r368003) @@ -869,6 +869,11 @@ passthru_cfgwrite(struct vmctx *ctx, int vcpu, struct if (error) err(1, "vm_setup_pptdev_msix"); } + } else { + error = vm_disable_pptdev_msix(ctx, sc->psc_sel.pc_bus, + sc->psc_sel.pc_dev, sc->psc_sel.pc_func); + if (error) + err(1, "vm_disable_pptdev_msix"); } return (0); } _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
