Author: attilio
Date: Tue May 21 20:38:19 2013
New Revision: 250884
URL: http://svnweb.freebsd.org/changeset/base/250884
Log:
o Relax locking assertions for vm_page_find_least()
o Relax locking assertions for pmap_enter_object() and add them also
to architectures that currently don't have any
o Introduce VM_OBJECT_LOCK_DOWNGRADE() which is basically a downgrade
operation on the per-object rwlock
o Use all the mechanisms above to make vm_map_pmap_enter() to work
mostl of the times only with readlocks.
Sponsored by: EMC / Isilon storage division
Reviewed by: alc
Modified:
head/sys/amd64/amd64/pmap.c
head/sys/arm/arm/pmap-v6.c
head/sys/arm/arm/pmap.c
head/sys/i386/i386/pmap.c
head/sys/i386/xen/pmap.c
head/sys/ia64/ia64/pmap.c
head/sys/mips/mips/pmap.c
head/sys/powerpc/aim/mmu_oea.c
head/sys/powerpc/aim/mmu_oea64.c
head/sys/powerpc/booke/pmap.c
head/sys/sparc64/sparc64/pmap.c
head/sys/vm/vm_map.c
head/sys/vm/vm_object.h
head/sys/vm/vm_page.c
Modified: head/sys/amd64/amd64/pmap.c
==============================================================================
--- head/sys/amd64/amd64/pmap.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/amd64/amd64/pmap.c Tue May 21 20:38:19 2013 (r250884)
@@ -3722,7 +3722,8 @@ pmap_enter_object(pmap_t pmap, vm_offset
vm_page_t m, mpte;
vm_pindex_t diff, psize;
- VM_OBJECT_ASSERT_WLOCKED(m_start->object);
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
mpte = NULL;
m = m_start;
Modified: head/sys/arm/arm/pmap-v6.c
==============================================================================
--- head/sys/arm/arm/pmap-v6.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/arm/arm/pmap-v6.c Tue May 21 20:38:19 2013 (r250884)
@@ -2931,6 +2931,8 @@ pmap_enter_object(pmap_t pmap, vm_offset
vm_page_t m;
vm_pindex_t diff, psize;
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
m = m_start;
rw_wlock(&pvh_global_lock);
Modified: head/sys/arm/arm/pmap.c
==============================================================================
--- head/sys/arm/arm/pmap.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/arm/arm/pmap.c Tue May 21 20:38:19 2013 (r250884)
@@ -3587,6 +3587,8 @@ pmap_enter_object(pmap_t pmap, vm_offset
vm_page_t m;
vm_pindex_t diff, psize;
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
m = m_start;
rw_wlock(&pvh_global_lock);
Modified: head/sys/i386/i386/pmap.c
==============================================================================
--- head/sys/i386/i386/pmap.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/i386/i386/pmap.c Tue May 21 20:38:19 2013 (r250884)
@@ -3677,7 +3677,8 @@ pmap_enter_object(pmap_t pmap, vm_offset
vm_page_t m, mpte;
vm_pindex_t diff, psize;
- VM_OBJECT_ASSERT_WLOCKED(m_start->object);
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
mpte = NULL;
m = m_start;
Modified: head/sys/i386/xen/pmap.c
==============================================================================
--- head/sys/i386/xen/pmap.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/i386/xen/pmap.c Tue May 21 20:38:19 2013 (r250884)
@@ -2871,7 +2871,8 @@ pmap_enter_object(pmap_t pmap, vm_offset
multicall_entry_t *mclp = mcl;
int error, count = 0;
- VM_OBJECT_ASSERT_WLOCKED(m_start->object);
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
mpte = NULL;
m = m_start;
Modified: head/sys/ia64/ia64/pmap.c
==============================================================================
--- head/sys/ia64/ia64/pmap.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/ia64/ia64/pmap.c Tue May 21 20:38:19 2013 (r250884)
@@ -1802,7 +1802,8 @@ pmap_enter_object(pmap_t pmap, vm_offset
vm_page_t m;
vm_pindex_t diff, psize;
- VM_OBJECT_ASSERT_WLOCKED(m_start->object);
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
m = m_start;
rw_wlock(&pvh_global_lock);
Modified: head/sys/mips/mips/pmap.c
==============================================================================
--- head/sys/mips/mips/pmap.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/mips/mips/pmap.c Tue May 21 20:38:19 2013 (r250884)
@@ -2399,7 +2399,8 @@ pmap_enter_object(pmap_t pmap, vm_offset
vm_page_t m, mpte;
vm_pindex_t diff, psize;
- VM_OBJECT_ASSERT_WLOCKED(m_start->object);
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
mpte = NULL;
m = m_start;
Modified: head/sys/powerpc/aim/mmu_oea.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea.c Tue May 21 19:59:37 2013
(r250883)
+++ head/sys/powerpc/aim/mmu_oea.c Tue May 21 20:38:19 2013
(r250884)
@@ -1217,6 +1217,8 @@ moea_enter_object(mmu_t mmu, pmap_t pm,
vm_page_t m;
vm_pindex_t diff, psize;
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
m = m_start;
rw_wlock(&pvh_global_lock);
Modified: head/sys/powerpc/aim/mmu_oea64.c
==============================================================================
--- head/sys/powerpc/aim/mmu_oea64.c Tue May 21 19:59:37 2013
(r250883)
+++ head/sys/powerpc/aim/mmu_oea64.c Tue May 21 20:38:19 2013
(r250884)
@@ -1360,6 +1360,8 @@ moea64_enter_object(mmu_t mmu, pmap_t pm
vm_page_t m;
vm_pindex_t diff, psize;
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
m = m_start;
while (m != NULL && (diff = m->pindex - m_start->pindex) < psize) {
Modified: head/sys/powerpc/booke/pmap.c
==============================================================================
--- head/sys/powerpc/booke/pmap.c Tue May 21 19:59:37 2013
(r250883)
+++ head/sys/powerpc/booke/pmap.c Tue May 21 20:38:19 2013
(r250884)
@@ -1716,6 +1716,8 @@ mmu_booke_enter_object(mmu_t mmu, pmap_t
vm_page_t m;
vm_pindex_t diff, psize;
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
m = m_start;
rw_wlock(&pvh_global_lock);
Modified: head/sys/sparc64/sparc64/pmap.c
==============================================================================
--- head/sys/sparc64/sparc64/pmap.c Tue May 21 19:59:37 2013
(r250883)
+++ head/sys/sparc64/sparc64/pmap.c Tue May 21 20:38:19 2013
(r250884)
@@ -1632,6 +1632,8 @@ pmap_enter_object(pmap_t pm, vm_offset_t
vm_page_t m;
vm_pindex_t diff, psize;
+ VM_OBJECT_ASSERT_LOCKED(m_start->object);
+
psize = atop(end - start);
m = m_start;
rw_wlock(&tte_list_global_lock);
Modified: head/sys/vm/vm_map.c
==============================================================================
--- head/sys/vm/vm_map.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/vm/vm_map.c Tue May 21 20:38:19 2013 (r250884)
@@ -1806,18 +1806,27 @@ vm_map_pmap_enter(vm_map_t map, vm_offse
if ((prot & (VM_PROT_READ | VM_PROT_EXECUTE)) == 0 || object == NULL)
return;
- VM_OBJECT_WLOCK(object);
+ VM_OBJECT_RLOCK(object);
if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
- pmap_object_init_pt(map->pmap, addr, object, pindex, size);
- goto unlock_return;
+ VM_OBJECT_RUNLOCK(object);
+ VM_OBJECT_WLOCK(object);
+ if (object->type == OBJT_DEVICE || object->type == OBJT_SG) {
+ pmap_object_init_pt(map->pmap, addr, object, pindex,
+ size);
+ VM_OBJECT_WUNLOCK(object);
+ return;
+ }
+ VM_OBJECT_LOCK_DOWNGRADE(object);
}
psize = atop(size);
if (psize > MAX_INIT_PT && (flags & MAP_PREFAULT_PARTIAL) != 0)
psize = MAX_INIT_PT;
if (psize + pindex > object->size) {
- if (object->size < pindex)
- goto unlock_return;
+ if (object->size < pindex) {
+ VM_OBJECT_RUNLOCK(object);
+ return;
+ }
psize = object->size - pindex;
}
@@ -1856,8 +1865,7 @@ vm_map_pmap_enter(vm_map_t map, vm_offse
if (p_start != NULL)
pmap_enter_object(map->pmap, start, addr + ptoa(psize),
p_start, prot);
-unlock_return:
- VM_OBJECT_WUNLOCK(object);
+ VM_OBJECT_RUNLOCK(object);
}
/*
Modified: head/sys/vm/vm_object.h
==============================================================================
--- head/sys/vm/vm_object.h Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/vm/vm_object.h Tue May 21 20:38:19 2013 (r250884)
@@ -223,6 +223,8 @@ extern struct vm_object kmem_object_stor
rw_assert(&(object)->lock, RA_RLOCKED)
#define VM_OBJECT_ASSERT_WLOCKED(object)
\
rw_assert(&(object)->lock, RA_WLOCKED)
+#define VM_OBJECT_LOCK_DOWNGRADE(object)
\
+ rw_downgrade(&(object)->lock)
#define VM_OBJECT_RLOCK(object)
\
rw_rlock(&(object)->lock)
#define VM_OBJECT_RUNLOCK(object)
\
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Tue May 21 19:59:37 2013 (r250883)
+++ head/sys/vm/vm_page.c Tue May 21 20:38:19 2013 (r250884)
@@ -959,7 +959,7 @@ vm_page_find_least(vm_object_t object, v
{
vm_page_t m;
- VM_OBJECT_ASSERT_WLOCKED(object);
+ VM_OBJECT_ASSERT_LOCKED(object);
if ((m = TAILQ_FIRST(&object->memq)) != NULL && m->pindex < pindex)
m = vm_radix_lookup_ge(&object->rtree, pindex);
return (m);
_______________________________________________
svn-src-all@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
