Author: markj
Date: Sun Dec 24 19:45:16 2017
New Revision: 327168
URL: https://svnweb.freebsd.org/changeset/base/327168
Log:
Fix two problems with the page daemon control loop.
Both issues caused the page daemon to erroneously go to sleep when
applications are consuming free pages at a high rate, leaving the
application threads blocked in VM_WAIT.
1) After completing an inactive queue scan, concurrent allocations may
have prevented the page daemon from meeting the v_free_min threshold.
In this case, the page daemon was going to sleep even when the
inactive queue contained plenty of clean pages.
2) pagedaemon_wakeup() may be called without the free queues lock held.
This can lead to a lost wakeup if a call occurs after the page daemon
clears vm_pageout_wanted but before going to sleep.
Fix 1) by ensuring that we start a new inactive queue scan immediately
if v_free_count < v_free_min after a prior scan.
Fix 2) by adding a new subroutine, pagedaemon_wait(), called from
vm_wait() and vm_waitpfault(). It wakes up the page daemon if either
vm_pages_needed or vm_pageout_wanted is false, and atomically sleeps
on v_free_count.
Reported by: jeff
Reviewed by: alc
MFC after: 2 weeks
Differential Revision: https://reviews.freebsd.org/D13424
Modified:
head/sys/vm/vm_page.c
head/sys/vm/vm_pageout.c
head/sys/vm/vm_pageout.h
Modified: head/sys/vm/vm_page.c
==============================================================================
--- head/sys/vm/vm_page.c Sun Dec 24 19:17:15 2017 (r327167)
+++ head/sys/vm/vm_page.c Sun Dec 24 19:45:16 2017 (r327168)
@@ -2661,15 +2661,9 @@ _vm_wait(void)
msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
PDROP | PSWP, "VMWait", 0);
} else {
- if (__predict_false(pageproc == NULL))
+ if (pageproc == NULL)
panic("vm_wait in early boot");
- if (!vm_pageout_wanted) {
- vm_pageout_wanted = true;
- wakeup(&vm_pageout_wanted);
- }
- vm_pages_needed = true;
- msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP |
PVM,
- "vmwait", 0);
+ pagedaemon_wait(PVM, "vmwait");
}
}
@@ -2699,7 +2693,6 @@ vm_page_alloc_fail(vm_object_t object, int req)
atomic_add_int(&vm_pageout_deficit,
max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
- pagedaemon_wakeup();
if (req & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) {
if (object != NULL)
VM_OBJECT_WUNLOCK(object);
@@ -2708,8 +2701,10 @@ vm_page_alloc_fail(vm_object_t object, int req)
VM_OBJECT_WLOCK(object);
if (req & VM_ALLOC_WAITOK)
return (EAGAIN);
- } else
+ } else {
mtx_unlock(&vm_page_queue_free_mtx);
+ pagedaemon_wakeup();
+ }
return (0);
}
@@ -2728,13 +2723,7 @@ vm_waitpfault(void)
{
mtx_lock(&vm_page_queue_free_mtx);
- if (!vm_pageout_wanted) {
- vm_pageout_wanted = true;
- wakeup(&vm_pageout_wanted);
- }
- vm_pages_needed = true;
- msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | PUSER,
- "pfault", 0);
+ pagedaemon_wait(PUSER, "pfault");
}
struct vm_pagequeue *
Modified: head/sys/vm/vm_pageout.c
==============================================================================
--- head/sys/vm/vm_pageout.c Sun Dec 24 19:17:15 2017 (r327167)
+++ head/sys/vm/vm_pageout.c Sun Dec 24 19:45:16 2017 (r327168)
@@ -1829,10 +1829,14 @@ vm_pageout_worker(void *arg)
pass++;
} else {
/*
- * Yes. Sleep until pages need to be reclaimed or
+ * Yes. If threads are still sleeping in VM_WAIT
+ * then we immediately start a new scan. Otherwise,
+ * sleep until the next wakeup or until pages need to
* have their reference stats updated.
*/
- if (mtx_sleep(&vm_pageout_wanted,
+ if (vm_pages_needed) {
+ mtx_unlock(&vm_page_queue_free_mtx);
+ } else if (mtx_sleep(&vm_pageout_wanted,
&vm_page_queue_free_mtx, PDROP | PVM, "psleep",
hz) == 0) {
VM_CNT_INC(v_pdwakeups);
@@ -1940,17 +1944,42 @@ vm_pageout(void)
}
/*
- * Unless the free page queue lock is held by the caller, this function
- * should be regarded as advisory. Specifically, the caller should
- * not msleep() on &vm_cnt.v_free_count following this function unless
- * the free page queue lock is held until the msleep() is performed.
+ * Perform an advisory wakeup of the page daemon.
*/
void
pagedaemon_wakeup(void)
{
+ mtx_assert(&vm_page_queue_free_mtx, MA_NOTOWNED);
+
if (!vm_pageout_wanted && curthread->td_proc != pageproc) {
vm_pageout_wanted = true;
wakeup(&vm_pageout_wanted);
}
+}
+
+/*
+ * Wake up the page daemon and wait for it to reclaim free pages.
+ *
+ * This function returns with the free queues mutex unlocked.
+ */
+void
+pagedaemon_wait(int pri, const char *wmesg)
+{
+
+ mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
+
+ /*
+ * vm_pageout_wanted may have been set by an advisory wakeup, but if the
+ * page daemon is running on a CPU, the wakeup will have been lost.
+ * Thus, deliver a potentially spurious wakeup to ensure that the page
+ * daemon has been notified of the shortage.
+ */
+ if (!vm_pageout_wanted || !vm_pages_needed) {
+ vm_pageout_wanted = true;
+ wakeup(&vm_pageout_wanted);
+ }
+ vm_pages_needed = true;
+ msleep(&vm_cnt.v_free_count, &vm_page_queue_free_mtx, PDROP | pri,
+ wmesg, 0);
}
Modified: head/sys/vm/vm_pageout.h
==============================================================================
--- head/sys/vm/vm_pageout.h Sun Dec 24 19:17:15 2017 (r327167)
+++ head/sys/vm/vm_pageout.h Sun Dec 24 19:45:16 2017 (r327168)
@@ -96,11 +96,12 @@ extern bool vm_pages_needed;
* Signal pageout-daemon and wait for it.
*/
-extern void pagedaemon_wakeup(void);
+void pagedaemon_wait(int pri, const char *wmesg);
+void pagedaemon_wakeup(void);
#define VM_WAIT vm_wait()
#define VM_WAITPFAULT vm_waitpfault()
-extern void vm_wait(void);
-extern void vm_waitpfault(void);
+void vm_wait(void);
+void vm_waitpfault(void);
#ifdef _KERNEL
int vm_pageout_flush(vm_page_t *, int, int, int, int *, boolean_t *);
_______________________________________________
svn-src-all@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-all
To unsubscribe, send any mail to "svn-src-all-unsubscr...@freebsd.org"
