Loads with non-linear access patterns were producing a very high
ratio of recursive pt faults to regular tlb misses. Rather than
choose between a 4-level table walk or a 1-level virtual page table
lookup, use a hybrid scheme with a virtual linear pmd, followed by a
2-level lookup in the normal handler.
This adds about 5 cycles (assuming no cache misses, and e5500 timing)
to a normal TLB miss, but greatly reduces the recursive fault rate
for loads which don't have locality within 2 MiB regions but do have
significant locality within 1 GiB regions. Improvements of close to 50%
were seen on such benchmarks.
Signed-off-by: Scott Wood <scottw...@freescale.com>
---
arch/powerpc/mm/tlb_low_64e.S | 23 +++++++++++++++--------
1 files changed, 15 insertions(+), 8 deletions(-)
diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S
index af08922..17726d3 100644
--- a/arch/powerpc/mm/tlb_low_64e.S
+++ b/arch/powerpc/mm/tlb_low_64e.S
@@ -24,7 +24,7 @@
#ifdef CONFIG_PPC_64K_PAGES
#define VPTE_PMD_SHIFT (PTE_INDEX_SIZE+1)
#else
-#define VPTE_PMD_SHIFT (PTE_INDEX_SIZE)
+#define VPTE_PMD_SHIFT 0
#endif
#define VPTE_PUD_SHIFT (VPTE_PMD_SHIFT + PMD_INDEX_SIZE)
#define VPTE_PGD_SHIFT (VPTE_PUD_SHIFT + PUD_INDEX_SIZE)
@@ -185,7 +185,7 @@ normal_tlb_miss:
/* Insert the bottom bits in */
rlwimi r14,r15,0,16,31
#else
- rldicl r14,r16,64-(PAGE_SHIFT-3),PAGE_SHIFT-3+4
+ rldicl r14,r16,64-(PMD_SHIFT-3),PMD_SHIFT-3+4
#endif
sldi r15,r10,60
clrrdi r14,r14,3
@@ -202,6 +202,16 @@ MMU_FTR_SECTION_ELSE
ld r14,0(r10)
ALT_MMU_FTR_SECTION_END_IFSET(MMU_FTR_USE_TLBRSRV)
+#ifndef CONFIG_PPC_64K_PAGES
+ rldicl r15,r16,64-PAGE_SHIFT+3,64-PTE_INDEX_SIZE-3
+ clrrdi r15,r15,3
+
+ cmpldi cr0,r14,0
+ beq normal_tlb_miss_access_fault
+
+ ldx r14,r14,r15
+#endif
+
finish_normal_tlb_miss:
/* Check if required permissions are met */
andc. r15,r11,r14
@@ -353,14 +363,11 @@ END_MMU_FTR_SECTION_IFSET(MMU_FTR_USE_TLBRSRV)
#ifndef CONFIG_PPC_64K_PAGES
/* Get to PUD entry */
rldicl r11,r16,64-VPTE_PUD_SHIFT,64-PUD_INDEX_SIZE-3
- clrrdi r10,r11,3
- ldx r15,r10,r15
- cmpldi cr0,r15,0
- beq virt_page_table_tlb_miss_fault
-#endif /* CONFIG_PPC_64K_PAGES */
-
+#else
/* Get to PMD entry */
rldicl r11,r16,64-VPTE_PMD_SHIFT,64-PMD_INDEX_SIZE-3
+#endif
+
clrrdi r10,r11,3
ldx r15,r10,r15
cmpldi cr0,r15,0
--
1.7.4.1
_______________________________________________
Linuxppc-dev mailing list
Linuxppc-dev@lists.ozlabs.org
https://lists.ozlabs.org/listinfo/linuxppc-dev
