Author: alc
Date: Sun May 9 16:55:42 2010
New Revision: 207823
URL: http://svn.freebsd.org/changeset/base/207823
Log:
Push down the acquisition of the page queues lock into vm_pageq_remove().
(This eliminates a surprising number of page queues lock acquisitions by
vm_fault() because the page's queue is PQ_NONE and thus the page queues
lock is not needed to remove the page from a queue.)
Modified:
head/sys/vm/vm_fault.c
head/sys/vm/vm_page.c
Modified: head/sys/vm/vm_fault.c
==============================================================================
--- head/sys/vm/vm_fault.c Sun May 9 16:27:42 2010 (r207822)
+++ head/sys/vm/vm_fault.c Sun May 9 16:55:42 2010 (r207823)
@@ -363,9 +363,7 @@ RetryFault:;
vm_object_deallocate(fs.first_object);
goto RetryFault;
}
- vm_page_lock_queues();
vm_pageq_remove(fs.m);
- vm_page_unlock_queues();
vm_page_unlock(fs.m);
/*
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Sun May 9 16:27:42 2010 (r207822)
+++ head/sys/vm/vm_page.c Sun May 9 16:55:42 2010 (r207823)
@@ -169,6 +169,7 @@ TUNABLE_INT("vm.boot_pages", &boot_pages
SYSCTL_INT(_vm, OID_AUTO, boot_pages, CTLFLAG_RD, &boot_pages, 0,
"number of pages allocated for bootstrapping the VM system");
+static void vm_page_queue_remove(int queue, vm_page_t m);
static void vm_page_enqueue(int queue, vm_page_t m);
/* Make sure that u_long is at least 64 bits when PAGE_SIZE is 32K. */
@@ -1328,24 +1329,43 @@ vm_page_requeue(vm_page_t m)
}
/*
+ * vm_page_queue_remove:
+ *
+ * Remove the given page from the specified queue.
+ *
+ * The page and page queues must be locked.
+ */
+static __inline void
+vm_page_queue_remove(int queue, vm_page_t m)
+{
+ struct vpgqueues *pq;
+
+ mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+ vm_page_lock_assert(m, MA_OWNED);
+ pq = &vm_page_queues[queue];
+ TAILQ_REMOVE(&pq->pl, m, pageq);
+ (*pq->cnt)--;
+}
+
+/*
* vm_pageq_remove:
*
* Remove a page from its queue.
*
- * The queue containing the given page must be locked.
+ * The given page must be locked.
* This routine may not block.
*/
void
vm_pageq_remove(vm_page_t m)
{
int queue = VM_PAGE_GETQUEUE(m);
- struct vpgqueues *pq;
+ vm_page_lock_assert(m, MA_OWNED);
if (queue != PQ_NONE) {
+ vm_page_lock_queues();
VM_PAGE_SETQUEUE2(m, PQ_NONE);
- pq = &vm_page_queues[queue];
- TAILQ_REMOVE(&pq->pl, m, pageq);
- (*pq->cnt)--;
+ vm_page_queue_remove(queue, m);
+ vm_page_unlock_queues();
}
}
@@ -1380,18 +1400,20 @@ vm_page_enqueue(int queue, vm_page_t m)
void
vm_page_activate(vm_page_t m)
{
+ int queue;
vm_page_lock_assert(m, MA_OWNED);
- if (VM_PAGE_GETKNOWNQUEUE2(m) != PQ_ACTIVE) {
+ if ((queue = VM_PAGE_GETKNOWNQUEUE2(m)) != PQ_ACTIVE) {
if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
if (m->act_count < ACT_INIT)
m->act_count = ACT_INIT;
vm_page_lock_queues();
- vm_pageq_remove(m);
+ if (queue != PQ_NONE)
+ vm_page_queue_remove(queue, m);
vm_page_enqueue(PQ_ACTIVE, m);
vm_page_unlock_queues();
} else
- KASSERT(m->queue == PQ_NONE,
+ KASSERT(queue == PQ_NONE,
("vm_page_activate: wired page %p is queued", m));
} else {
if (m->act_count < ACT_INIT)
@@ -1472,11 +1494,8 @@ vm_page_free_toq(vm_page_t m)
* callback routine until after we've put the page on the
* appropriate free queue.
*/
- if (VM_PAGE_GETQUEUE(m) != PQ_NONE) {
- vm_page_lock_queues();
+ if ((m->flags & PG_UNMANAGED) == 0)
vm_pageq_remove(m);
- vm_page_unlock_queues();
- }
vm_page_remove(m);
/*
@@ -1554,11 +1573,8 @@ vm_page_wire(vm_page_t m)
if (m->flags & PG_FICTITIOUS)
return;
if (m->wire_count == 0) {
- if ((m->flags & PG_UNMANAGED) == 0) {
- vm_page_lock_queues();
+ if ((m->flags & PG_UNMANAGED) == 0)
vm_pageq_remove(m);
- vm_page_unlock_queues();
- }
atomic_add_int(&cnt.v_wire_count, 1);
}
m->wire_count++;
@@ -1633,22 +1649,26 @@ vm_page_unwire(vm_page_t m, int activate
static inline void
_vm_page_deactivate(vm_page_t m, int athead)
{
+ int queue;
vm_page_lock_assert(m, MA_OWNED);
/*
* Ignore if already inactive.
*/
- if (VM_PAGE_INQUEUE2(m, PQ_INACTIVE))
+ if ((queue = VM_PAGE_GETKNOWNQUEUE2(m)) == PQ_INACTIVE)
return;
if (m->wire_count == 0 && (m->flags & PG_UNMANAGED) == 0) {
vm_page_lock_queues();
vm_page_flag_clear(m, PG_WINATCFLS);
- vm_pageq_remove(m);
+ if (queue != PQ_NONE)
+ vm_page_queue_remove(queue, m);
if (athead)
- TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m,
pageq);
+ TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m,
+ pageq);
else
- TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m,
pageq);
+ TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m,
+ pageq);
VM_PAGE_SETQUEUE2(m, PQ_INACTIVE);
cnt.v_inactive_count++;
vm_page_unlock_queues();
@@ -1751,11 +1771,7 @@ vm_page_cache(vm_page_t m)
/*
* Remove the page from the paging queues.
*/
- if (VM_PAGE_GETQUEUE(m) != PQ_NONE) {
- vm_page_lock_queues();
- vm_pageq_remove(m);
- vm_page_unlock_queues();
- }
+ vm_pageq_remove(m);
/*
* Remove the page from the object's collection of resident
_______________________________________________
svn-src-head@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
