From: Suravee Suthikulpanit <suravee.suthikulpa...@amd.com>
Implement the newly added IOTLB flushing interface by introducing
per-protection-domain IOTLB flush list, which maintains a list of
IOVAs to be invalidated (by INVALIDATE_IOTLB_PAGES command) during
IOTLB sync.
Cc: Joerg Roedel <jroe...@suse.de>
Signed-off-by: Suravee Suthikulpanit <suravee.suthikulpa...@amd.com>
---
drivers/iommu/amd_iommu.c | 77 ++++++++++++++++++++++++++++++++++++++++-
drivers/iommu/amd_iommu_init.c | 2 --
drivers/iommu/amd_iommu_types.h | 2 ++
3 files changed, 78 insertions(+), 3 deletions(-)
diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 8e8874d..bf92809 100644
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -130,6 +130,12 @@ struct dma_ops_domain {
static struct iova_domain reserved_iova_ranges;
static struct lock_class_key reserved_rbtree_key;
+struct iotlb_flush_entry {
+ struct list_head list;
+ unsigned long iova;
+ size_t size;
+};
+
/****************************************************************************
*
* Helper functions
@@ -2838,11 +2844,13 @@ static void protection_domain_free(struct
protection_domain *domain)
static int protection_domain_init(struct protection_domain *domain)
{
spin_lock_init(&domain->lock);
+ spin_lock_init(&domain->iotlb_flush_list_lock);
mutex_init(&domain->api_lock);
domain->id = domain_id_alloc();
if (!domain->id)
return -ENOMEM;
INIT_LIST_HEAD(&domain->dev_list);
+ INIT_LIST_HEAD(&domain->iotlb_flush_list);
return 0;
}
@@ -3047,7 +3055,6 @@ static size_t amd_iommu_unmap(struct iommu_domain *dom,
unsigned long iova,
unmap_size = iommu_unmap_page(domain, iova, page_size);
mutex_unlock(&domain->api_lock);
- domain_flush_tlb_pde(domain);
domain_flush_complete(domain);
return unmap_size;
@@ -3167,6 +3174,71 @@ static bool amd_iommu_is_attach_deferred(struct
iommu_domain *domain,
return dev_data->defer_attach;
}
+static void amd_iommu_flush_iotlb_all(struct iommu_domain *domain)
+{
+ struct protection_domain *dom = to_pdomain(domain);
+
+ domain_flush_tlb_pde(dom);
+}
+
+static void amd_iommu_iotlb_range_add(struct iommu_domain *domain,
+ unsigned long iova, size_t size)
+{
+ struct protection_domain *pdom = to_pdomain(domain);
+ struct iotlb_flush_entry *entry, *p;
+ unsigned long flags;
+ bool found = false;
+
+ spin_lock_irqsave(&pdom->iotlb_flush_list_lock, flags);
+ list_for_each_entry(p, &pdom->iotlb_flush_list, list) {
+ if (iova != p->iova)
+ continue;
+
+ if (size > p->size) {
+ p->size = size;
+ pr_debug("%s: update range: iova=%#lx, size = %#lx\n",
+ __func__, p->iova, p->size);
+ }
+ found = true;
+ break;
+ }
+
+ if (!found) {
+ entry = kzalloc(sizeof(struct iotlb_flush_entry),
+ GFP_ATOMIC);
+ if (!entry)
+ return;
+
+ pr_debug("%s: new range: iova=%lx, size=%#lx\n",
+ __func__, iova, size);
+
+ entry->iova = iova;
+ entry->size = size;
+ list_add(&entry->list, &pdom->iotlb_flush_list);
+ }
+ spin_unlock_irqrestore(&pdom->iotlb_flush_list_lock, flags);
+}
+
+static void amd_iommu_iotlb_sync(struct iommu_domain *domain)
+{
+ struct protection_domain *pdom = to_pdomain(domain);
+ struct iotlb_flush_entry *entry, *next;
+ unsigned long flags;
+
+ /* Note:
+ * Currently, IOMMU driver just flushes the whole IO/TLB for
+ * a given domain. So, just remove entries from the list here.
+ */
+ spin_lock_irqsave(&pdom->iotlb_flush_list_lock, flags);
+ list_for_each_entry_safe(entry, next, &pdom->iotlb_flush_list, list) {
+ list_del(&entry->list);
+ kfree(entry);
+ }
+ spin_unlock_irqrestore(&pdom->iotlb_flush_list_lock, flags);
+
+ domain_flush_tlb_pde(pdom);
+}
+
const struct iommu_ops amd_iommu_ops = {
.capable = amd_iommu_capable,
.domain_alloc = amd_iommu_domain_alloc,
@@ -3185,6 +3257,9 @@ static bool amd_iommu_is_attach_deferred(struct
iommu_domain *domain,
.apply_resv_region = amd_iommu_apply_resv_region,
.is_attach_deferred = amd_iommu_is_attach_deferred,
.pgsize_bitmap = AMD_IOMMU_PGSIZES,
+ .flush_iotlb_all = amd_iommu_flush_iotlb_all,
+ .iotlb_range_add = amd_iommu_iotlb_range_add,
+ .iotlb_sync = amd_iommu_iotlb_sync,
};
/*****************************************************************************
diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c
index 6fe2d03..1659377 100644
--- a/drivers/iommu/amd_iommu_init.c
+++ b/drivers/iommu/amd_iommu_init.c
@@ -2489,8 +2489,6 @@ static int __init early_amd_iommu_init(void)
*/
__set_bit(0, amd_iommu_pd_alloc_bitmap);
- spin_lock_init(&amd_iommu_pd_lock);
-
/*
* now the data structures are allocated and basically initialized
* start the real acpi table scan
diff --git a/drivers/iommu/amd_iommu_types.h b/drivers/iommu/amd_iommu_types.h
index f6b24c7..30e1c68 100644
--- a/drivers/iommu/amd_iommu_types.h
+++ b/drivers/iommu/amd_iommu_types.h
@@ -462,9 +462,11 @@ struct amd_iommu_fault {
struct protection_domain {
struct list_head list; /* for list of all protection domains */
struct list_head dev_list; /* List of all devices in this domain */
+ struct list_head iotlb_flush_list; /* store iovas for iotlb sync */
struct iommu_domain domain; /* generic domain handle used by
iommu core code */
spinlock_t lock; /* mostly used to lock the page table*/
+ spinlock_t iotlb_flush_list_lock; /* protect iova flush list */
struct mutex api_lock; /* protect page tables in the iommu-api path */
u16 id; /* the domain id written to the device table */
int mode; /* paging mode (0-6 levels) */
--
1.8.3.1
