Move the barriers and range iteration down into the _tlbie* level,
which improves readability.
Signed-off-by: Nicholas Piggin <npig...@gmail.com>
---
arch/powerpc/mm/tlb-radix.c | 71 ++++++++++++++++++++++++++-------------------
1 file changed, 41 insertions(+), 30 deletions(-)
diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c
index 49e71c68f5b1..645a35b7bc9d 100644
--- a/arch/powerpc/mm/tlb-radix.c
+++ b/arch/powerpc/mm/tlb-radix.c
@@ -85,7 +85,7 @@ static inline void _tlbie_pid(unsigned long pid, unsigned
long ric)
}
static inline void __tlbiel_va(unsigned long va, unsigned long pid,
- unsigned long ap, unsigned long ric)
+ unsigned long ap, unsigned long ric)
{
unsigned long rb,rs,prs,r;
@@ -101,13 +101,28 @@ static inline void __tlbiel_va(unsigned long va, unsigned
long pid,
}
static inline void _tlbiel_va(unsigned long va, unsigned long pid,
- unsigned long ap, unsigned long ric)
+ unsigned long psize, unsigned long ric)
{
+ unsigned long ap = mmu_get_ap(psize);
+
asm volatile("ptesync": : :"memory");
__tlbiel_va(va, pid, ap, ric);
asm volatile("ptesync": : :"memory");
}
+static inline void _tlbiel_va_range(unsigned long start, unsigned long end,
+ unsigned long pid, unsigned long page_size,
+ unsigned long psize)
+{
+ unsigned long addr;
+ unsigned long ap = mmu_get_ap(psize);
+
+ asm volatile("ptesync": : :"memory");
+ for (addr = start; addr < end; addr += page_size)
+ __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
+ asm volatile("ptesync": : :"memory");
+}
+
static inline void __tlbie_va(unsigned long va, unsigned long pid,
unsigned long ap, unsigned long ric)
{
@@ -125,13 +140,27 @@ static inline void __tlbie_va(unsigned long va, unsigned
long pid,
}
static inline void _tlbie_va(unsigned long va, unsigned long pid,
- unsigned long ap, unsigned long ric)
+ unsigned long psize, unsigned long ric)
{
+ unsigned long ap = mmu_get_ap(psize);
+
asm volatile("ptesync": : :"memory");
__tlbie_va(va, pid, ap, ric);
asm volatile("eieio; tlbsync; ptesync": : :"memory");
}
+static inline void _tlbie_va_range(unsigned long start, unsigned long end,
+ unsigned long pid, unsigned long page_size,
+ unsigned long psize)
+{
+ unsigned long addr;
+ unsigned long ap = mmu_get_ap(psize);
+
+ asm volatile("ptesync": : :"memory");
+ for (addr = start; addr < end; addr += page_size)
+ __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
+ asm volatile("eieio; tlbsync; ptesync": : :"memory");
+}
/*
* Base TLB flushing operations:
@@ -174,12 +203,11 @@ void radix__local_flush_tlb_page_psize(struct mm_struct
*mm, unsigned long vmadd
int psize)
{
unsigned long pid;
- unsigned long ap = mmu_get_ap(psize);
preempt_disable();
pid = mm->context.id;
if (pid != MMU_NO_CONTEXT)
- _tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
+ _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
preempt_enable();
}
@@ -239,16 +267,15 @@ void radix__flush_tlb_page_psize(struct mm_struct *mm,
unsigned long vmaddr,
int psize)
{
unsigned long pid;
- unsigned long ap = mmu_get_ap(psize);
pid = mm->context.id;
if (unlikely(pid == MMU_NO_CONTEXT))
return;
preempt_disable();
if (!mm_is_thread_local(mm))
- _tlbie_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
+ _tlbie_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
else
- _tlbiel_va(vmaddr, pid, ap, RIC_FLUSH_TLB);
+ _tlbiel_va(vmaddr, pid, psize, RIC_FLUSH_TLB);
preempt_enable();
}
@@ -335,9 +362,7 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm,
unsigned long start,
unsigned long end, int psize)
{
unsigned long pid;
- unsigned long addr;
bool local;
- unsigned long ap = mmu_get_ap(psize);
unsigned long page_size = 1UL << mmu_psize_defs[psize].shift;
@@ -356,17 +381,10 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm,
unsigned long start,
_tlbie_pid(pid, RIC_FLUSH_TLB);
} else {
- asm volatile("ptesync": : :"memory");
- for (addr = start; addr < end; addr += page_size) {
- if (local)
- __tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
- else
- __tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
- }
if (local)
- asm volatile("ptesync": : :"memory");
+ _tlbiel_va_range(start, end, pid, page_size, psize);
else
- asm volatile("eieio; tlbsync; ptesync": : :"memory");
+ _tlbie_va_range(start, end, pid, page_size, psize);
}
preempt_enable();
@@ -375,9 +393,8 @@ void radix__flush_tlb_range_psize(struct mm_struct *mm,
unsigned long start,
#ifdef CONFIG_TRANSPARENT_HUGEPAGE
void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr)
{
- bool local;
- unsigned long ap = mmu_get_ap(mmu_virtual_psize);
unsigned long pid, end;
+ bool local;
pid = mm->context.id;
if (unlikely(pid == MMU_NO_CONTEXT))
@@ -399,18 +416,12 @@ void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm,
unsigned long addr)
_tlbie_pid(pid, RIC_FLUSH_PWC);
/* Then iterate the pages */
- asm volatile("ptesync": : :"memory");
end = addr + HPAGE_PMD_SIZE;
- for (; addr < end; addr += PAGE_SIZE) {
- if (local)
- _tlbiel_va(addr, pid, ap, RIC_FLUSH_TLB);
- else
- _tlbie_va(addr, pid, ap, RIC_FLUSH_TLB);
- }
+
if (local)
- asm volatile("ptesync": : :"memory");
+ _tlbiel_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize);
else
- asm volatile("eieio; tlbsync; ptesync": : :"memory");
+ _tlbie_va_range(addr, end, pid, PAGE_SIZE, mmu_virtual_psize);
preempt_enable();
}
#endif /* CONFIG_TRANSPARENT_HUGEPAGE */
--
2.15.0
