Author: jhibbits
Date: Wed May 8 16:05:18 2019
New Revision: 347350
URL: https://svnweb.freebsd.org/changeset/base/347350
Log:
powerpc/booke: Do as much work outside of TLB locks as possible
Reduce the surface area of the TLB locks. Unfortunately the same trick for
serializing the tlbie instruction on OEA64 cannot be used here to reduce the
scope of the tlbivax mutex to the tlbsync only, as the mutex also serializes
the TLB miss lock as a side effect, so contention on this lock may not be
reducible any further.
Modified:
head/sys/powerpc/booke/pmap.c
Modified: head/sys/powerpc/booke/pmap.c
==============================================================================
--- head/sys/powerpc/booke/pmap.c Wed May 8 16:04:33 2019
(r347349)
+++ head/sys/powerpc/booke/pmap.c Wed May 8 16:05:18 2019
(r347350)
@@ -1382,7 +1382,7 @@ pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offs
unsigned int pp2d_idx = PP2D_IDX(va);
unsigned int pdir_idx = PDIR_IDX(va);
unsigned int ptbl_idx = PTBL_IDX(va);
- pte_t *ptbl, *pte;
+ pte_t *ptbl, *pte, pte_tmp;
pte_t **pdir;
/* Get the page directory pointer. */
@@ -1400,12 +1400,13 @@ pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offs
KASSERT(nosleep, ("nosleep and NULL ptbl"));
return (ENOMEM);
}
+ pte = &ptbl[ptbl_idx];
} else {
/*
* Check if there is valid mapping for requested va, if there
* is, remove it.
*/
- pte = &pdir[pdir_idx][ptbl_idx];
+ pte = &ptbl[ptbl_idx];
if (PTE_ISVALID(pte)) {
pte_remove(mmu, pmap, va, PTBL_HOLD);
} else {
@@ -1437,14 +1438,16 @@ pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offs
pv_insert(pmap, va, m);
}
+ pmap->pm_stats.resident_count++;
+
+ pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
+ pte_tmp |= (PTE_VALID | flags);
+
mtx_lock_spin(&tlbivax_mutex);
tlb_miss_lock();
tlb0_flush_entry(va);
- pmap->pm_stats.resident_count++;
- pte = &pdir[pdir_idx][ptbl_idx];
- *pte = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
- *pte |= (PTE_VALID | flags);
+ *pte = pte_tmp;
tlb_miss_unlock();
mtx_unlock_spin(&tlbivax_mutex);
@@ -1583,7 +1586,7 @@ pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offs
{
unsigned int pdir_idx = PDIR_IDX(va);
unsigned int ptbl_idx = PTBL_IDX(va);
- pte_t *ptbl, *pte;
+ pte_t *ptbl, *pte, pte_tmp;
CTR4(KTR_PMAP, "%s: su = %d pmap = %p va = %p", __func__,
pmap == kernel_pmap, pmap, va);
@@ -1598,6 +1601,8 @@ pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offs
KASSERT(nosleep, ("nosleep and NULL ptbl"));
return (ENOMEM);
}
+ pmap->pm_pdir[pdir_idx] = ptbl;
+ pte = &ptbl[ptbl_idx];
} else {
/*
* Check if there is valid mapping for requested
@@ -1629,20 +1634,14 @@ pte_enter(mmu_t mmu, pmap_t pmap, vm_page_t m, vm_offs
pmap->pm_stats.resident_count++;
+ pte_tmp = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
+ pte_tmp |= (PTE_VALID | flags | PTE_PS_4KB); /* 4KB pages only */
+
mtx_lock_spin(&tlbivax_mutex);
tlb_miss_lock();
tlb0_flush_entry(va);
- if (pmap->pm_pdir[pdir_idx] == NULL) {
- /*
- * If we just allocated a new page table, hook it in
- * the pdir.
- */
- pmap->pm_pdir[pdir_idx] = ptbl;
- }
- pte = &(pmap->pm_pdir[pdir_idx][ptbl_idx]);
- *pte = PTE_RPN_FROM_PA(VM_PAGE_TO_PHYS(m));
- *pte |= (PTE_VALID | flags | PTE_PS_4KB); /* 4KB pages only */
+ *pte = pte_tmp;
tlb_miss_unlock();
mtx_unlock_spin(&tlbivax_mutex);
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
