Signed-off-by: Alexey Kardashevskiy <a...@ozlabs.ru> --- drivers/vfio/vfio_iommu_spapr_tce.c | 243 +++++++++++++++++++++++------------- 1 file changed, 155 insertions(+), 88 deletions(-)
diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c index d0987ae..8bcafb7 100644 --- a/drivers/vfio/vfio_iommu_spapr_tce.c +++ b/drivers/vfio/vfio_iommu_spapr_tce.c @@ -84,9 +84,15 @@ static void decrement_locked_vm(long npages) */ struct tce_container { struct mutex lock; - struct iommu_group *grp; bool enabled; struct list_head mem_list; + struct iommu_table tables[POWERPC_IOMMU_MAX_TABLES]; + struct list_head group_list; +}; + +struct tce_iommu_group { + struct list_head next; + struct iommu_group *grp; }; struct tce_memory { @@ -265,17 +271,21 @@ static bool tce_check_page_size(struct page *page, unsigned page_shift) return false; } +static inline bool tce_groups_attached(struct tce_container *container) +{ + return !list_empty(&container->group_list); +} + static struct iommu_table *spapr_tce_find_table( struct tce_container *container, phys_addr_t ioba) { long i; struct iommu_table *ret = NULL; - struct powerpc_iommu *iommu = iommu_group_get_iommudata(container->grp); mutex_lock(&container->lock); for (i = 0; i < POWERPC_IOMMU_MAX_TABLES; ++i) { - struct iommu_table *tbl = &iommu->tables[i]; + struct iommu_table *tbl = &container->tables[i]; unsigned long entry = ioba >> tbl->it_page_shift; unsigned long start = tbl->it_offset; unsigned long end = start + tbl->it_size; @@ -290,13 +300,31 @@ static struct iommu_table *spapr_tce_find_table( return ret; } +static unsigned long tce_default_winsize(struct tce_container *container) +{ + struct tce_iommu_group *tcegrp; + struct powerpc_iommu *iommu; + + if (!tce_groups_attached(container)) + return 0; + + tcegrp = list_first_entry(&container->group_list, + struct tce_iommu_group, next); + if (!tcegrp) + return 0; + + iommu = iommu_group_get_iommudata(tcegrp->grp); + if (!iommu) + return 0; + + return iommu->tce32_size; +} + static int tce_iommu_enable(struct tce_container *container) { int ret = 0; - struct powerpc_iommu *iommu; - struct iommu_table *tbl; - if (!container->grp) + if (!tce_groups_attached(container)) return -ENXIO; if (container->enabled) @@ -328,12 +356,8 @@ static int tce_iommu_enable(struct tce_container *container) * KVM agnostic. */ if (!tce_preregistered(container)) { - iommu = iommu_group_get_iommudata(container->grp); - if (!iommu) - return -EFAULT; - - tbl = &iommu->tables[0]; - ret = try_increment_locked_vm(IOMMU_TABLE_PAGES(tbl)); + ret = try_increment_locked_vm( + tce_default_winsize(container) >> PAGE_SHIFT); if (ret) return ret; } @@ -343,27 +367,23 @@ static int tce_iommu_enable(struct tce_container *container) return ret; } +static int tce_iommu_clear(struct tce_container *container, + struct iommu_table *tbl, + unsigned long entry, unsigned long pages); + static void tce_iommu_disable(struct tce_container *container) { - struct powerpc_iommu *iommu; - struct iommu_table *tbl; - if (!container->enabled) return; container->enabled = false; - if (!container->grp || !current->mm) + if (!current->mm) return; - if (!tce_preregistered(container)) { - iommu = iommu_group_get_iommudata(container->grp); - if (!iommu) - return; - - tbl = &iommu->tables[0]; - decrement_locked_vm(IOMMU_TABLE_PAGES(tbl)); - } + if (!tce_preregistered(container)) + decrement_locked_vm( + tce_default_winsize(container) >> PAGE_SHIFT); } static void *tce_iommu_open(unsigned long arg) @@ -381,20 +401,44 @@ static void *tce_iommu_open(unsigned long arg) mutex_init(&container->lock); INIT_LIST_HEAD_RCU(&container->mem_list); + INIT_LIST_HEAD_RCU(&container->group_list); return container; } static void tce_iommu_release(void *iommu_data) { + int i; + struct powerpc_iommu *iommu; + struct tce_iommu_group *tcegrp; struct tce_container *container = iommu_data; struct tce_memory *mem, *memtmp; + struct powerpc_iommu_ops *iommuops = NULL; - WARN_ON(container->grp); tce_iommu_disable(container); - if (container->grp) - tce_iommu_detach_group(iommu_data, container->grp); + while (tce_groups_attached(container)) { + tcegrp = list_first_entry(&container->group_list, + struct tce_iommu_group, next); + iommu = iommu_group_get_iommudata(tcegrp->grp); + iommuops = iommu->ops; + tce_iommu_detach_group(iommu_data, tcegrp->grp); + } + + /* Free tables */ + if (iommuops) { + for (i = 0; i < POWERPC_IOMMU_MAX_TABLES; ++i) { + struct iommu_table *tbl = &container->tables[i]; + + tce_iommu_clear(container, tbl, + tbl->it_offset, tbl->it_size); + + if (!tce_preregistered(container)) + decrement_locked_vm(IOMMU_TABLE_PAGES(tbl)); + + iommuops->free_table(tbl); + } + } list_for_each_entry_safe(mem, memtmp, &container->mem_list, next) tce_do_unregister_pages(container, mem); @@ -568,16 +612,17 @@ static long tce_iommu_ioctl(void *iommu_data, case VFIO_IOMMU_SPAPR_TCE_GET_INFO: { struct vfio_iommu_spapr_tce_info info; - struct iommu_table *tbl; + struct tce_iommu_group *tcegrp; struct powerpc_iommu *iommu; - if (WARN_ON(!container->grp)) + if (!tce_groups_attached(container)) return -ENXIO; - iommu = iommu_group_get_iommudata(container->grp); + tcegrp = list_first_entry(&container->group_list, + struct tce_iommu_group, next); + iommu = iommu_group_get_iommudata(tcegrp->grp); - tbl = &iommu->tables[0]; - if (WARN_ON_ONCE(!tbl)) + if (!iommu) return -ENXIO; minsz = offsetofend(struct vfio_iommu_spapr_tce_info, @@ -589,9 +634,8 @@ static long tce_iommu_ioctl(void *iommu_data, if (info.argsz < minsz) return -EINVAL; - info.dma32_window_start = tbl->it_offset << tbl->it_page_shift; - info.dma32_window_size = tbl->it_size << tbl->it_page_shift; - info.flags = 0; + info.dma32_window_start = iommu->tce32_start; + info.dma32_window_size = iommu->tce32_size; if (copy_to_user((void __user *)arg, &info, minsz)) return -EFAULT; @@ -603,9 +647,8 @@ static long tce_iommu_ioctl(void *iommu_data, struct iommu_table *tbl; unsigned long tce; - if (WARN_ON(!container->grp || - !iommu_group_get_iommudata(container->grp))) - return -ENXIO; + if (!container->enabled) + return -EPERM; minsz = offsetofend(struct vfio_iommu_type1_dma_map, size); @@ -655,10 +698,6 @@ static long tce_iommu_ioctl(void *iommu_data, struct vfio_iommu_type1_dma_unmap param; struct iommu_table *tbl; - if (WARN_ON(!container->grp || - !iommu_group_get_iommudata(container->grp))) - return -ENXIO; - minsz = offsetofend(struct vfio_iommu_type1_dma_unmap, size); @@ -747,12 +786,20 @@ static long tce_iommu_ioctl(void *iommu_data, tce_iommu_disable(container); mutex_unlock(&container->lock); return 0; - case VFIO_EEH_PE_OP: - if (!container->grp) - return -ENODEV; - return vfio_spapr_iommu_eeh_ioctl(container->grp, - cmd, arg); + case VFIO_EEH_PE_OP: { + struct tce_iommu_group *tcegrp; + + ret = 0; + list_for_each_entry(tcegrp, &container->group_list, next) { + ret = vfio_spapr_iommu_eeh_ioctl(tcegrp->grp, + cmd, arg); + if (ret) + return ret; + } + return ret; + } + } return -ENOTTY; @@ -761,40 +808,63 @@ static long tce_iommu_ioctl(void *iommu_data, static int tce_iommu_attach_group(void *iommu_data, struct iommu_group *iommu_group) { - int ret = 0; + int ret = 0, i; struct tce_container *container = iommu_data; - struct powerpc_iommu *iommu; + struct powerpc_iommu *iommu = iommu_group_get_iommudata(iommu_group); + struct tce_iommu_group *tcegrp; mutex_lock(&container->lock); /* pr_debug("tce_vfio: Attaching group #%u to iommu %p\n", iommu_group_id(iommu_group), iommu_group); */ - if (container->grp) { - pr_warn("tce_vfio: Only one group per IOMMU container is allowed, existing id=%d, attaching id=%d\n", - iommu_group_id(container->grp), - iommu_group_id(iommu_group)); - ret = -EBUSY; - } else if (container->enabled) { - pr_err("tce_vfio: attaching group #%u to enabled container\n", - iommu_group_id(iommu_group)); - ret = -EBUSY; + + list_for_each_entry(tcegrp, &container->group_list, next) { + struct powerpc_iommu *iommutmp; + + if (tcegrp->grp == iommu_group) { + pr_warn("tce_vfio: Group %d is already attached\n", + iommu_group_id(iommu_group)); + ret = -EBUSY; + goto unlock_exit; + } + iommutmp = iommu_group_get_iommudata(tcegrp->grp); + if (iommutmp->ops != iommu->ops) { + pr_warn("tce_vfio: Group %d is incompatible with group %d\n", + iommu_group_id(iommu_group), + iommu_group_id(tcegrp->grp)); + ret = -EBUSY; + goto unlock_exit; + } + } + + /* + * Disable iommu bypass, otherwise the user can DMA to all of + * our physical memory via the bypass window instead of just + * the pages that has been explicitly mapped into the iommu + */ + if (iommu->ops && iommu->ops->set_ownership) { + iommu->ops->set_ownership(iommu, true); } else { - iommu = iommu_group_get_iommudata(iommu_group); - if (WARN_ON_ONCE(!iommu)) - return -ENXIO; - /* - * Disable iommu bypass, otherwise the user can DMA to all of - * our physical memory via the bypass window instead of just - * the pages that has been explicitly mapped into the iommu - */ - if (iommu->ops && iommu->ops->set_ownership) { - iommu->ops->set_ownership(iommu, true); - container->grp = iommu_group; - } else { - return -ENODEV; - } + ret = -ENODEV; + goto unlock_exit; } + tcegrp = kzalloc(sizeof(*tcegrp), GFP_KERNEL); + tcegrp->grp = iommu_group; + list_add(&tcegrp->next, &container->group_list); + for (i = 0; i < POWERPC_IOMMU_MAX_TABLES; ++i) { + struct iommu_table *tbl = &container->tables[i]; + + if (!tbl->it_size) + continue; + + /* Set the default window to a new group */ + ret = iommu->ops->set_window(iommu, i, tbl); + if (ret) + goto unlock_exit; + } + +unlock_exit: mutex_unlock(&container->lock); return ret; @@ -805,33 +875,30 @@ static void tce_iommu_detach_group(void *iommu_data, { struct tce_container *container = iommu_data; struct powerpc_iommu *iommu; + struct tce_iommu_group *tcegrp, *tcegrptmp; + long i; mutex_lock(&container->lock); - if (iommu_group != container->grp) { - pr_warn("tce_vfio: detaching group #%u, expected group is #%u\n", - iommu_group_id(iommu_group), - iommu_group_id(container->grp)); - } else { - if (container->enabled) { - pr_warn("tce_vfio: detaching group #%u from enabled container, forcing disable\n", - iommu_group_id(container->grp)); - tce_iommu_disable(container); - } - /* pr_debug("tce_vfio: detaching group #%u from iommu %p\n", - iommu_group_id(iommu_group), iommu_group); */ - container->grp = NULL; + /* Detach windows from IOMMUs */ + list_for_each_entry_safe(tcegrp, tcegrptmp, &container->group_list, + next) { + if (tcegrp->grp != iommu_group) + continue; + list_del(&tcegrp->next); iommu = iommu_group_get_iommudata(iommu_group); BUG_ON(!iommu); - tce_iommu_clear(container, &iommu->tables[0], - iommu->tables[0].it_offset, - iommu->tables[0].it_size); + for (i = 0; i < POWERPC_IOMMU_MAX_TABLES; ++i) + iommu->ops->unset_window(iommu, i); /* Kernel owns the device now, we can restore bypass */ if (iommu->ops && iommu->ops->set_ownership) iommu->ops->set_ownership(iommu, false); + + kfree(tcegrp); + break; } mutex_unlock(&container->lock); } -- 2.0.0 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to majord...@vger.kernel.org More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/