Loads with non-linear access patterns were producing a very high
ratio of recursive pt faults to regular tlb misses.  Rather than
choose between a 4-level table walk or a 1-level virtual page table
lookup, use a hybrid scheme with a virtual linear pmd, followed by a
2-level lookup in the normal handler.

This adds about 5 cycles (assuming no cache misses, and e5500 timing)
to a normal TLB miss, but greatly reduces the recursive fault rate
for loads which don't have locality within 2 MiB regions but do have
significant locality within 1 GiB regions.  Improvements of close to 50%
were seen on such benchmarks.

Signed-off-by: Scott Wood <scottw...@freescale.com>
---
 arch/powerpc/mm/tlb_low_64e.S |   23 +++++++++++++++--------
 1 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S
index af08922..17726d3 100644
--- a/arch/powerpc/mm/tlb_low_64e.S
+++ b/arch/powerpc/mm/tlb_low_64e.S
@@ -24,7 +24,7 @@
 #ifdef CONFIG_PPC_64K_PAGES
 #define VPTE_PMD_SHIFT (PTE_INDEX_SIZE+1)
 #else
-#define VPTE_PMD_SHIFT (PTE_INDEX_SIZE)
+#define VPTE_PMD_SHIFT 0
 #endif
 #define VPTE_PUD_SHIFT (VPTE_PMD_SHIFT + PMD_INDEX_SIZE)
 #define VPTE_PGD_SHIFT (VPTE_PUD_SHIFT + PUD_INDEX_SIZE)
@@ -185,7 +185,7 @@ normal_tlb_miss:
        /* Insert the bottom bits in */
        rlwimi  r14,r15,0,16,31
 #else
-       rldicl  r14,r16,64-(PAGE_SHIFT-3),PAGE_SHIFT-3+4
+       rldicl  r14,r16,64-(PMD_SHIFT-3),PMD_SHIFT-3+4
 #endif
        sldi    r15,r10,60
        clrrdi  r14,r14,3
@@ -202,6 +202,16 @@ MMU_FTR_SECTION_ELSE
        ld      r14,0(r10)
 ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBRSRV)
 
+#ifndef CONFIG_PPC_64K_PAGES
+       rldicl  r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3
+       clrrdi  r15,r15,3
+
+       cmpldi  cr0,r14,0
+       beq     normal_tlb_miss_access_fault
+
+       ldx     r14,r14,r15
+#endif
+
 finish_normal_tlb_miss:
        /* Check if required permissions are met */
        andc.   r15,r11,r14
@@ -353,14 +363,11 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
 #ifndef CONFIG_PPC_64K_PAGES
        /* Get to PUD entry */
        rldicl  r11,r16,64-VPTE_PUD_SHIFT,64-PUD_INDEX_SIZE-3
-       clrrdi  r10,r11,3
-       ldx     r15,r10,r15
-       cmpldi  cr0,r15,0
-       beq     virt_page_table_tlb_miss_fault
-#endif /* CONFIG_PPC_64K_PAGES */
-
+#else
        /* Get to PMD entry */
        rldicl  r11,r16,64-VPTE_PMD_SHIFT,64-PMD_INDEX_SIZE-3
+#endif
+
        clrrdi  r10,r11,3
        ldx     r15,r10,r15
        cmpldi  cr0,r15,0
-- 
1.7.4.1


_______________________________________________
Linuxppc-dev mailing list
Linuxppc-dev@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/linuxppc-dev

Reply via email to