Nicholas Piggin <npig...@gmail.com> writes:

> When permissiveness is relaxed, or found to have been relaxed by
> another thread, we flush that address out of the TLB to avoid a
> future fault or micro-fault due to a stale TLB entry.
>
> Currently for processes with TLBs on other CPUs, this flush is always
> done with a global tlbie. Although that could reduce faults on remote
> CPUs, a broadcast operation seems to be wasteful for something that
> can be handled in-core by the remote CPU if it comes to it.
>
> This is not benchmarked yet. It does seem cut some tlbie operations
> from the bus.
>
> Signed-off-by: Nicholas Piggin <npig...@gmail.com>
> ---
>  .../powerpc/include/asm/book3s/64/tlbflush-radix.h |  5 ++++
>  arch/powerpc/include/asm/book3s/64/tlbflush.h      | 11 +++++++++
>  arch/powerpc/mm/pgtable-book3s64.c                 |  5 +++-
>  arch/powerpc/mm/pgtable.c                          |  2 +-
>  arch/powerpc/mm/tlb-radix.c                        | 27 
> ++++++++++++++++++++++
>  5 files changed, 48 insertions(+), 2 deletions(-)
>
> diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h 
> b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> index b12460b306a7..34cd864b8fc1 100644
> --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h
> @@ -16,6 +16,8 @@ extern bool radix__flush_tlb_range_psize(struct mm_struct 
> *mm, unsigned long sta
>                                        unsigned long end, int psize);
>  extern void radix__flush_pmd_tlb_range(struct vm_area_struct *vma,
>                                      unsigned long start, unsigned long end);
> +extern void radix__local_flush_pmd_tlb_range(struct vm_area_struct *vma,
> +                             unsigned long start, unsigned long end);
>  extern void radix__flush_tlb_range(struct vm_area_struct *vma, unsigned long 
> start,
>                           unsigned long end);
>  extern void radix__flush_tlb_kernel_range(unsigned long start, unsigned long 
> end);
> @@ -24,6 +26,9 @@ extern void radix__local_flush_tlb_mm(struct mm_struct *mm);
>  extern void radix__local_flush_tlb_page(struct vm_area_struct *vma, unsigned 
> long vmaddr);
>  extern void radix__local_flush_tlb_page_psize(struct mm_struct *mm, unsigned 
> long vmaddr,
>                                             int psize);
> +extern void radix__local_flush_tlb_range_psize(struct mm_struct *mm,
> +                             unsigned long start, unsigned long end,
> +                             int psize);
>  extern void radix__tlb_flush(struct mmu_gather *tlb);
>  #ifdef CONFIG_SMP
>  extern void radix__flush_tlb_mm(struct mm_struct *mm);
> diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush.h 
> b/arch/powerpc/include/asm/book3s/64/tlbflush.h
> index 72b925f97bab..8a8b3e11a28e 100644
> --- a/arch/powerpc/include/asm/book3s/64/tlbflush.h
> +++ b/arch/powerpc/include/asm/book3s/64/tlbflush.h
> @@ -83,6 +83,17 @@ static inline void flush_tlb_page(struct vm_area_struct 
> *vma,
>  #define flush_tlb_mm(mm)             local_flush_tlb_mm(mm)
>  #define flush_tlb_page(vma, addr)    local_flush_tlb_page(vma, addr)
>  #endif /* CONFIG_SMP */
> +
> +#define flush_tlb_fix_spurious_fault flush_tlb_fix_spurious_fault
> +static inline void flush_tlb_fix_spurious_fault(struct vm_area_struct *vma,
> +                                             unsigned long address)
> +{
> +     if (radix_enabled())
> +             radix__local_flush_tlb_page(vma, address);
> +     else
> +             flush_tlb_page(vma, address);
> +}
> +
>  /*
>   * flush the page walk cache for the address
>   */
> diff --git a/arch/powerpc/mm/pgtable-book3s64.c 
> b/arch/powerpc/mm/pgtable-book3s64.c
> index 3b65917785a5..e46f346388d6 100644
> --- a/arch/powerpc/mm/pgtable-book3s64.c
> +++ b/arch/powerpc/mm/pgtable-book3s64.c
> @@ -40,7 +40,10 @@ int pmdp_set_access_flags(struct vm_area_struct *vma, 
> unsigned long address,
>       if (changed) {
>               __ptep_set_access_flags(vma->vm_mm, pmdp_ptep(pmdp),
>                                       pmd_pte(entry), address);
> -             flush_pmd_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
> +             if (radix_enabled())
> +                     radix__local_flush_pmd_tlb_range(vma, address, address 
> + HPAGE_PMD_SIZE);
> +             else
> +                     flush_pmd_tlb_range(vma, address, address + 
> HPAGE_PMD_SIZE);
                        ^^^^  this is no-op for hash.


>       }
>       return changed;
>  }
> diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c
> index a03ff3d99e0c..acd6ae8062ce 100644


-aneesh

Reply via email to