Author: rstone Date: Sun Mar 1 00:39:48 2015 New Revision: 279444 URL: https://svnweb.freebsd.org/changeset/base/279444
Log: Allow passthrough devices to be hinted. Allow the ppt driver to attach to devices that were hinted to be passthrough devices by the PCI code creating them with a driver name of "ppt". Add a tunable that allows the IOMMU to be forced to be used. With SR-IOV passthrough devices the VFs may be created after vmm.ko is loaded. The current code will not initialize the IOMMU in that case, meaning that the passthrough devices can't actually be used. Differential Revision: https://reviews.freebsd.org/D73 Reviewed by: neel MFC after: 1 month Sponsored by: Sandvine Inc. Modified: head/sys/amd64/vmm/io/ppt.c head/sys/amd64/vmm/vmm.c Modified: head/sys/amd64/vmm/io/ppt.c ============================================================================== --- head/sys/amd64/vmm/io/ppt.c Sun Mar 1 00:39:40 2015 (r279443) +++ head/sys/amd64/vmm/io/ppt.c Sun Mar 1 00:39:48 2015 (r279444) @@ -56,7 +56,6 @@ __FBSDID("$FreeBSD$"); /* XXX locking */ -#define MAX_PPTDEVS (sizeof(pptdevs) / sizeof(pptdevs[0])) #define MAX_MSIMSGS 32 /* @@ -77,9 +76,10 @@ struct pptintr_arg { /* pptintr(pptin uint64_t msg_data; }; -static struct pptdev { +struct pptdev { device_t dev; struct vm *vm; /* owner of this device */ + TAILQ_ENTRY(pptdev) next; struct vm_memory_segment mmio[MAX_MMIOSEGS]; struct { int num_msgs; /* guest state */ @@ -99,7 +99,7 @@ static struct pptdev { void **cookie; struct pptintr_arg *arg; } msix; -} pptdevs[64]; +}; SYSCTL_DECL(_hw_vmm); SYSCTL_NODE(_hw_vmm, OID_AUTO, ppt, CTLFLAG_RW, 0, "bhyve passthru devices"); @@ -108,6 +108,8 @@ static int num_pptdevs; SYSCTL_INT(_hw_vmm_ppt, OID_AUTO, devices, CTLFLAG_RD, &num_pptdevs, 0, "number of pci passthru devices"); +static TAILQ_HEAD(, pptdev) pptdev_list = TAILQ_HEAD_INITIALIZER(pptdev_list); + static int ppt_probe(device_t dev) { @@ -125,26 +127,30 @@ ppt_probe(device_t dev) * - be allowed by administrator to be used in this role * - be an endpoint device */ - if (vmm_is_pptdev(bus, slot, func) && - (dinfo->cfg.hdrtype & PCIM_HDRTYPE) == PCIM_HDRTYPE_NORMAL) + if ((dinfo->cfg.hdrtype & PCIM_HDRTYPE) != PCIM_HDRTYPE_NORMAL) + return (ENXIO); + else if (vmm_is_pptdev(bus, slot, func)) return (0); else - return (ENXIO); + /* + * Returning BUS_PROBE_NOWILDCARD here matches devices that the + * SR-IOV infrastructure specified as "ppt" passthrough devices. + * All normal devices that did not have "ppt" specified as their + * driver will not be matched by this. + */ + return (BUS_PROBE_NOWILDCARD); } static int ppt_attach(device_t dev) { - int n; + struct pptdev *ppt; - if (num_pptdevs >= MAX_PPTDEVS) { - printf("ppt_attach: maximum number of pci passthrough devices " - "exceeded\n"); - return (ENXIO); - } + ppt = device_get_softc(dev); - n = num_pptdevs++; - pptdevs[n].dev = dev; + num_pptdevs++; + TAILQ_INSERT_TAIL(&pptdev_list, ppt, next); + ppt->dev = dev; if (bootverbose) device_printf(dev, "attached\n"); @@ -155,10 +161,14 @@ ppt_attach(device_t dev) static int ppt_detach(device_t dev) { - /* - * XXX check whether there are any pci passthrough devices assigned - * to guests before we allow this driver to detach. - */ + struct pptdev *ppt; + + ppt = device_get_softc(dev); + + if (ppt->vm != NULL) + return (EBUSY); + num_pptdevs--; + TAILQ_REMOVE(&pptdev_list, ppt, next); return (0); } @@ -172,22 +182,23 @@ static device_method_t ppt_methods[] = { }; static devclass_t ppt_devclass; -DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, 0); +DEFINE_CLASS_0(ppt, ppt_driver, ppt_methods, sizeof(struct pptdev)); DRIVER_MODULE(ppt, pci, ppt_driver, ppt_devclass, NULL, NULL); static struct pptdev * ppt_find(int bus, int slot, int func) { device_t dev; - int i, b, s, f; + struct pptdev *ppt; + int b, s, f; - for (i = 0; i < num_pptdevs; i++) { - dev = pptdevs[i].dev; + TAILQ_FOREACH(ppt, &pptdev_list, next) { + dev = ppt->dev; b = pci_get_bus(dev); s = pci_get_slot(dev); f = pci_get_function(dev); if (bus == b && slot == s && func == f) - return (&pptdevs[i]); + return (ppt); } return (NULL); } @@ -297,11 +308,12 @@ ppt_avail_devices(void) int ppt_assigned_devices(struct vm *vm) { - int i, num; + struct pptdev *ppt; + int num; num = 0; - for (i = 0; i < num_pptdevs; i++) { - if (pptdevs[i].vm == vm) + TAILQ_FOREACH(ppt, &pptdev_list, next) { + if (ppt->vm == vm) num++; } return (num); @@ -310,12 +322,11 @@ ppt_assigned_devices(struct vm *vm) boolean_t ppt_is_mmio(struct vm *vm, vm_paddr_t gpa) { - int i, n; + int i; struct pptdev *ppt; struct vm_memory_segment *seg; - for (n = 0; n < num_pptdevs; n++) { - ppt = &pptdevs[n]; + TAILQ_FOREACH(ppt, &pptdev_list, next) { if (ppt->vm != vm) continue; @@ -377,12 +388,13 @@ ppt_unassign_device(struct vm *vm, int b int ppt_unassign_all(struct vm *vm) { - int i, bus, slot, func; + struct pptdev *ppt; + int bus, slot, func; device_t dev; - for (i = 0; i < num_pptdevs; i++) { - if (pptdevs[i].vm == vm) { - dev = pptdevs[i].dev; + TAILQ_FOREACH(ppt, &pptdev_list, next) { + if (ppt->vm == vm) { + dev = ppt->dev; bus = pci_get_bus(dev); slot = pci_get_slot(dev); func = pci_get_function(dev); Modified: head/sys/amd64/vmm/vmm.c ============================================================================== --- head/sys/amd64/vmm/vmm.c Sun Mar 1 00:39:40 2015 (r279443) +++ head/sys/amd64/vmm/vmm.c Sun Mar 1 00:39:48 2015 (r279444) @@ -217,6 +217,11 @@ SYSCTL_INT(_hw_vmm, OID_AUTO, trace_gues &trace_guest_exceptions, 0, "Trap into hypervisor on all guest exceptions and reflect them back"); +static int vmm_force_iommu = 0; +TUNABLE_INT("hw.vmm.force_iommu", &vmm_force_iommu); +SYSCTL_INT(_hw_vmm, OID_AUTO, force_iommu, CTLFLAG_RDTUN, &vmm_force_iommu, 0, + "Force use of I/O MMU even if no passthrough devices were found."); + static void vcpu_cleanup(struct vm *vm, int i, bool destroy) { @@ -321,7 +326,7 @@ vmm_handler(module_t mod, int what, void switch (what) { case MOD_LOAD: vmmdev_init(); - if (ppt_avail_devices() > 0) + if (vmm_force_iommu || ppt_avail_devices() > 0) iommu_init(); error = vmm_init(); if (error == 0) _______________________________________________ svn-src-head@freebsd.org mailing list http://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
