Author: alc
Date: Tue Jun 26 06:02:43 2012
New Revision: 237592
URL: http://svn.freebsd.org/changeset/base/237592
Log:
Add PV list locking to pmap_enter(). Its execution is no longer serialized
by the pvh global lock.
Add a needed atomic operation to pmap_object_init_pt().
Modified:
head/sys/amd64/amd64/pmap.c
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Tue Jun 26 05:34:31 2012 (r237591)
+++ head/sys/amd64/amd64/pmap.c Tue Jun 26 06:02:43 2012 (r237592)
@@ -264,7 +264,8 @@ static void pmap_pv_demote_pde(pmap_t pm
struct rwlock **lockp);
static boolean_t pmap_pv_insert_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
struct rwlock **lockp);
-static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa);
+static void pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
+ struct rwlock **lockp);
static void pmap_pvh_free(struct md_page *pvh, pmap_t pmap, vm_offset_t va);
static pv_entry_t pmap_pvh_remove(struct md_page *pvh, pmap_t pmap,
vm_offset_t va);
@@ -287,7 +288,8 @@ static boolean_t pmap_is_referenced_pvh(
static void pmap_kenter_attr(vm_offset_t va, vm_paddr_t pa, int mode);
static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va);
static void pmap_pde_attr(pd_entry_t *pde, int cache_bits);
-static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va);
+static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
+ struct rwlock **lockp);
static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t
sva,
vm_prot_t prot);
static void pmap_pte_attr(pt_entry_t *pte, int cache_bits);
@@ -305,10 +307,13 @@ static void pmap_update_pde(pmap_t pmap,
pd_entry_t newpde);
static void pmap_update_pde_invalidate(vm_offset_t va, pd_entry_t newpde);
-static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags);
-static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
+static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex,
+ struct rwlock **lockp);
+static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va,
+ struct rwlock **lockp);
+static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va,
+ struct rwlock **lockp);
-static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags);
static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_page_t* free);
static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *);
@@ -1686,8 +1691,10 @@ pmap_pinit(pmap_t pmap)
}
/*
- * this routine is called if the page table page is not
- * mapped correctly.
+ * This routine is called if the desired page table page does not exist.
+ *
+ * If page table page allocation fails, this routine may sleep before
+ * returning NULL. It sleeps only if a lock pointer was given.
*
* Note: If a page allocation fails at page table level two or three,
* one or two pages may be held during the wait, only to be released
@@ -1695,25 +1702,26 @@ pmap_pinit(pmap_t pmap)
* race conditions.
*/
static vm_page_t
-_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags)
+_pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, struct rwlock **lockp)
{
vm_page_t m, pdppg, pdpg;
- KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
- (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
- ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
-
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+
/*
* Allocate a page table page.
*/
if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
- if (flags & M_WAITOK) {
+ if (lockp != NULL) {
+ if (*lockp != NULL) {
+ rw_wunlock(*lockp);
+ *lockp = NULL;
+ }
PMAP_UNLOCK(pmap);
- rw_wunlock(&pvh_global_lock);
+ rw_runlock(&pvh_global_lock);
VM_WAIT;
- rw_wlock(&pvh_global_lock);
+ rw_rlock(&pvh_global_lock);
PMAP_LOCK(pmap);
}
@@ -1754,7 +1762,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t
if ((*pml4 & PG_V) == 0) {
/* Have to allocate a new pdp, recurse */
if (_pmap_allocpte(pmap, NUPDE + NUPDPE + pml4index,
- flags) == NULL) {
+ lockp) == NULL) {
--m->wire_count;
atomic_subtract_int(&cnt.v_wire_count, 1);
vm_page_free_zero(m);
@@ -1787,7 +1795,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t
if ((*pml4 & PG_V) == 0) {
/* Have to allocate a new pd, recurse */
if (_pmap_allocpte(pmap, NUPDE + pdpindex,
- flags) == NULL) {
+ lockp) == NULL) {
--m->wire_count;
atomic_subtract_int(&cnt.v_wire_count, 1);
vm_page_free_zero(m);
@@ -1801,7 +1809,7 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t
if ((*pdp & PG_V) == 0) {
/* Have to allocate a new pd, recurse */
if (_pmap_allocpte(pmap, NUPDE + pdpindex,
- flags) == NULL) {
+ lockp) == NULL) {
--m->wire_count;
atomic_subtract_int(&cnt.v_wire_count,
1);
@@ -1827,15 +1835,12 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t
}
static vm_page_t
-pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags)
+pmap_allocpde(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
{
vm_pindex_t pdpindex, ptepindex;
pdp_entry_t *pdpe;
vm_page_t pdpg;
- KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
- (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
- ("pmap_allocpde: flags is neither M_NOWAIT nor M_WAITOK"));
retry:
pdpe = pmap_pdpe(pmap, va);
if (pdpe != NULL && (*pdpe & PG_V) != 0) {
@@ -1846,24 +1851,20 @@ retry:
/* Allocate a pd page. */
ptepindex = pmap_pde_pindex(va);
pdpindex = ptepindex >> NPDPEPGSHIFT;
- pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, flags);
- if (pdpg == NULL && (flags & M_WAITOK))
+ pdpg = _pmap_allocpte(pmap, NUPDE + pdpindex, lockp);
+ if (pdpg == NULL && lockp != NULL)
goto retry;
}
return (pdpg);
}
static vm_page_t
-pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
+pmap_allocpte(pmap_t pmap, vm_offset_t va, struct rwlock **lockp)
{
vm_pindex_t ptepindex;
pd_entry_t *pd;
vm_page_t m;
- KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
- (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
- ("pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
-
/*
* Calculate pagetable page index
*/
@@ -1879,7 +1880,7 @@ retry:
* normal 4K page.
*/
if (pd != NULL && (*pd & (PG_PS | PG_V)) == (PG_PS | PG_V)) {
- if (!pmap_demote_pde(pmap, pd, va)) {
+ if (!pmap_demote_pde_locked(pmap, pd, va, lockp)) {
/*
* Invalidation of the 2MB page mapping may have caused
* the deallocation of the underlying PD page.
@@ -1900,8 +1901,8 @@ retry:
* Here if the pte page isn't mapped, or if it has been
* deallocated.
*/
- m = _pmap_allocpte(pmap, ptepindex, flags);
- if (m == NULL && (flags & M_WAITOK))
+ m = _pmap_allocpte(pmap, ptepindex, lockp);
+ if (m == NULL && lockp != NULL)
goto retry;
}
return (m);
@@ -2504,16 +2505,18 @@ pmap_pv_demote_pde(pmap_t pmap, vm_offse
* for the 2MB page mapping.
*/
static void
-pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa)
+pmap_pv_promote_pde(pmap_t pmap, vm_offset_t va, vm_paddr_t pa,
+ struct rwlock **lockp)
{
struct md_page *pvh;
pv_entry_t pv;
vm_offset_t va_last;
vm_page_t m;
- rw_assert(&pvh_global_lock, RA_WLOCKED);
+ rw_assert(&pvh_global_lock, RA_LOCKED);
KASSERT((pa & PDRMASK) == 0,
("pmap_pv_promote_pde: pa is not 2mpage aligned"));
+ CHANGE_PV_LIST_LOCK_TO_PHYS(lockp, pa);
/*
* Transfer the first page's pv entry for this mapping to the 2mpage's
@@ -3255,7 +3258,8 @@ retry:
* identical characteristics.
*/
static void
-pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va)
+pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va,
+ struct rwlock **lockp)
{
pd_entry_t newpde;
pt_entry_t *firstpte, oldpte, pa, *pte;
@@ -3273,7 +3277,7 @@ pmap_promote_pde(pmap_t pmap, pd_entry_t
setpde:
newpde = *firstpte;
if ((newpde & ((PG_FRAME & PDRMASK) | PG_A | PG_V)) != (PG_A | PG_V)) {
- pmap_pde_p_failures++;
+ atomic_add_long(&pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@@ -3298,7 +3302,7 @@ setpde:
setpte:
oldpte = *pte;
if ((oldpte & (PG_FRAME | PG_A | PG_V)) != pa) {
- pmap_pde_p_failures++;
+ atomic_add_long(&pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@@ -3317,7 +3321,7 @@ setpte:
" in pmap %p", oldpteva, pmap);
}
if ((oldpte & PG_PTE_PROMOTE) != (newpde & PG_PTE_PROMOTE)) {
- pmap_pde_p_failures++;
+ atomic_add_long(&pmap_pde_p_failures, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return;
@@ -3342,7 +3346,7 @@ setpte:
* Promote the pv entries.
*/
if ((newpde & PG_MANAGED) != 0)
- pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME);
+ pmap_pv_promote_pde(pmap, va, newpde & PG_PS_FRAME, lockp);
/*
* Propagate the PAT index to its proper position.
@@ -3358,7 +3362,7 @@ setpte:
else
pde_store(pde, PG_PS | newpde);
- pmap_pde_promotions++;
+ atomic_add_long(&pmap_pde_promotions, 1);
CTR2(KTR_PMAP, "pmap_promote_pde: success for va %#lx"
" in pmap %p", va, pmap);
}
@@ -3379,6 +3383,7 @@ void
pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m,
vm_prot_t prot, boolean_t wired)
{
+ struct rwlock *lock;
pd_entry_t *pde;
pt_entry_t *pte;
pt_entry_t newpte, origpte;
@@ -3398,7 +3403,8 @@ pmap_enter(pmap_t pmap, vm_offset_t va,
mpte = NULL;
- rw_wlock(&pvh_global_lock);
+ lock = NULL;
+ rw_rlock(&pvh_global_lock);
PMAP_LOCK(pmap);
/*
@@ -3406,7 +3412,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va,
* resident, we are creating it here.
*/
if (va < VM_MAXUSER_ADDRESS)
- mpte = pmap_allocpte(pmap, va, M_WAITOK);
+ mpte = pmap_allocpte(pmap, va, &lock);
pde = pmap_pde(pmap, va);
if (pde != NULL && (*pde & PG_V) != 0) {
@@ -3460,6 +3466,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va,
pmap->pm_stats.wired_count--;
if (origpte & PG_MANAGED) {
om = PHYS_TO_VM_PAGE(opa);
+ CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, om);
pv = pmap_pvh_remove(&om->md, pmap, va);
}
if (mpte != NULL) {
@@ -3479,6 +3486,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va,
("pmap_enter: managed mapping within the clean submap"));
if (pv == NULL)
pv = get_pv_entry(pmap, FALSE);
+ CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, m);
pv->pv_va = va;
TAILQ_INSERT_TAIL(&m->md.pv_list, pv, pv_list);
pa |= PG_MANAGED;
@@ -3534,11 +3542,13 @@ validate:
if ((newpte & PG_RW) == 0)
invlva = TRUE;
}
- if ((origpte & PG_MANAGED) != 0 &&
- TAILQ_EMPTY(&om->md.pv_list) &&
- ((om->flags & PG_FICTITIOUS) != 0 ||
- TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
- vm_page_aflag_clear(om, PGA_WRITEABLE);
+ if ((origpte & PG_MANAGED) != 0) {
+ CHANGE_PV_LIST_LOCK_TO_VM_PAGE(&lock, om);
+ if (TAILQ_EMPTY(&om->md.pv_list) &&
+ ((om->flags & PG_FICTITIOUS) != 0 ||
+ TAILQ_EMPTY(&pa_to_pvh(opa)->pv_list)))
+ vm_page_aflag_clear(om, PGA_WRITEABLE);
+ }
if (invlva)
pmap_invalidate_page(pmap, va);
} else
@@ -3552,9 +3562,11 @@ validate:
if ((mpte == NULL || mpte->wire_count == NPTEPG) &&
pg_ps_enabled && (m->flags & PG_FICTITIOUS) == 0 &&
vm_reserv_level_iffullpop(m) == 0)
- pmap_promote_pde(pmap, pde, va);
+ pmap_promote_pde(pmap, pde, va, &lock);
- rw_wunlock(&pvh_global_lock);
+ if (lock != NULL)
+ rw_wunlock(lock);
+ rw_runlock(&pvh_global_lock);
PMAP_UNLOCK(pmap);
}
@@ -3573,7 +3585,7 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t
rw_assert(&pvh_global_lock, RA_LOCKED);
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
- if ((mpde = pmap_allocpde(pmap, va, M_NOWAIT)) == NULL) {
+ if ((mpde = pmap_allocpde(pmap, va, NULL)) == NULL) {
CTR2(KTR_PMAP, "pmap_enter_pde: failure for va %#lx"
" in pmap %p", va, pmap);
return (FALSE);
@@ -3735,7 +3747,9 @@ pmap_enter_quick_locked(pmap_t pmap, vm_
/*
* If the page table page is mapped, we just increment
- * the hold count, and activate it.
+ * the hold count, and activate it. Otherwise, we
+ * attempt to allocate a page table page. If this
+ * attempt fails, we don't retry. Instead, we give up.
*/
if (ptepa && (*ptepa & PG_V) != 0) {
if (*ptepa & PG_PS)
@@ -3743,8 +3757,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_
mpte = PHYS_TO_VM_PAGE(*ptepa & PG_FRAME);
mpte->wire_count++;
} else {
- mpte = _pmap_allocpte(pmap, ptepindex,
- M_NOWAIT);
+ mpte = _pmap_allocpte(pmap, ptepindex, NULL);
if (mpte == NULL)
return (mpte);
}
@@ -3870,7 +3883,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offs
PMAP_LOCK(pmap);
for (pa = ptepa | pmap_cache_bits(pat_mode, 1); pa < ptepa +
size; pa += NBPDR) {
- pdpg = pmap_allocpde(pmap, addr, M_NOWAIT);
+ pdpg = pmap_allocpde(pmap, addr, NULL);
if (pdpg == NULL) {
/*
* The creation of mappings below is only an
@@ -3888,7 +3901,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offs
pde_store(pde, pa | PG_PS | PG_M | PG_A |
PG_U | PG_RW | PG_V);
pmap_resident_count_inc(pmap, NBPDR /
PAGE_SIZE);
- pmap_pde_mappings++;
+ atomic_add_long(&pmap_pde_mappings, 1);
} else {
/* Continue on if the PDE is already valid. */
pdpg->wire_count--;
@@ -4020,7 +4033,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm
continue;
if (srcptepaddr & PG_PS) {
- dstmpde = pmap_allocpde(dst_pmap, addr, M_NOWAIT);
+ dstmpde = pmap_allocpde(dst_pmap, addr, NULL);
if (dstmpde == NULL)
break;
pde = (pd_entry_t *)
@@ -4058,7 +4071,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pm
dstmpte->pindex == pmap_pde_pindex(addr))
dstmpte->wire_count++;
else if ((dstmpte = pmap_allocpte(dst_pmap,
- addr, M_NOWAIT)) == NULL)
+ addr, NULL)) == NULL)
goto out;
dst_pte = (pt_entry_t *)
PHYS_TO_DMAP(VM_PAGE_TO_PHYS(dstmpte));
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
