Nicholas Piggin <npig...@gmail.com> writes: > When permissiveness is relaxed, or found to have been relaxed by > another thread, we flush that address out of the TLB to avoid a > future fault or micro-fault due to a stale TLB entry. > > Currently for processes with TLBs on other CPUs, this flush is always > done with a global tlbie. Although that could reduce faults on remote > CPUs, a broadcast operation seems to be wasteful for something that > can be handled in-core by the remote CPU if it comes to it. > > This is not benchmarked yet. It does seem cut some tlbie operations > from the bus. > > Signed-off-by: Nicholas Piggin <npig...@gmail.com> > --- > .../powerpc/include/asm/book3s/64/tlbflush-radix.h | 5 ++++ > arch/powerpc/include/asm/book3s/64/tlbflush.h | 11 +++++++++ > arch/powerpc/mm/pgtable-book3s64.c | 5 +++- > arch/powerpc/mm/pgtable.c | 2 +- > arch/powerpc/mm/tlb-radix.c | 27 > ++++++++++++++++++++++ > 5 files changed, 48 insertions(+), 2 deletions(-) > > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > index b12460b306a7..34cd864b8fc1 100644 > --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h > @@ -16,6 +16,8 @@ extern bool radix__flush_tlb_range_psize(struct mm_struct > *mm, unsigned long sta > unsigned long end, int psize); > extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma, > unsigned long start, unsigned long end); > +extern void radix__local_flush_pmd_tlb_range(struct vm_area_struct *vma, > + unsigned long start, unsigned long end); > extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long > start, > unsigned long end); > extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long > end); > @@ -24,6 +26,9 @@ extern void radix__local_flush_tlb_mm(struct mm_struct *mm); > extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned > long vmaddr); > extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned > long vmaddr, > int psize); > +extern void radix__local_flush_tlb_range_psize(struct mm_struct *mm, > + unsigned long start, unsigned long end, > + int psize); > extern void radix__tlb_flush(struct mmu_gather *tlb); > #ifdef CONFIG_SMP > extern void radix__flush_tlb_mm(struct mm_struct *mm); > diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h > b/arch/powerpc/include/asm/book3s/64/tlbflush.h > index 72b925f97bab..8a8b3e11a28e 100644 > --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h > +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h > @@ -83,6 +83,17 @@ static inline void flush_tlb_page(struct vm_area_struct > *vma, > #define flush_tlb_mm(mm) local_flush_tlb_mm(mm) > #define flush_tlb_page(vma, addr) local_flush_tlb_page(vma, addr) > #endif /* CONFIG_SMP */ > + > +#define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault > +static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma, > + unsigned long address) > +{ > + if (radix_enabled()) > + radix__local_flush_tlb_page(vma, address); > + else > + flush_tlb_page(vma, address); > +} > + > /* > * flush the page walk cache for the address > */ > diff --git a/arch/powerpc/mm/pgtable-book3s64.c > b/arch/powerpc/mm/pgtable-book3s64.c > index 3b65917785a5..e46f346388d6 100644 > --- a/arch/powerpc/mm/pgtable-book3s64.c > +++ b/arch/powerpc/mm/pgtable-book3s64.c > @@ -40,7 +40,10 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, > unsigned long address, > if (changed) { > __ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp), > pmd_pte(entry), address); > - flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE); > + if (radix_enabled()) > + radix__local_flush_pmd_tlb_range(vma, address, address > + HPAGE_PMD_SIZE); > + else > + flush_pmd_tlb_range(vma, address, address + > HPAGE_PMD_SIZE); ^^^^ this is no-op for hash.
> } > return changed; > } > diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c > index a03ff3d99e0c..acd6ae8062ce 100644 -aneesh