The branch main has been updated by alc:
URL:
https://cgit.FreeBSD.org/src/commit/?id=c296ac7e0f1c2fc9bc8bcab0177afb123ce6993a
commit c296ac7e0f1c2fc9bc8bcab0177afb123ce6993a
Author: Alan Cox <a...@freebsd.org>
AuthorDate: 2024-11-27 08:32:07 +0000
Commit: Alan Cox <a...@freebsd.org>
CommitDate: 2024-11-30 08:59:15 +0000
vm: Optimize page rename
Rename vm_page_rename() to vm_page_iter_rename() to reflect its
reimplementation using iterators, and pass the page to this function
rather than spending clock cycles looking it up. Change its return
value from 0/1 to a bool.
Reviewed by: dougm, markj
Differential Revision: https://reviews.freebsd.org/D47829
---
sys/vm/vm_object.c | 10 ++++++----
sys/vm/vm_page.c | 28 ++++++++++++++++------------
sys/vm/vm_page.h | 3 ++-
sys/vm/vm_reserv.c | 4 ++--
4 files changed, 26 insertions(+), 19 deletions(-)
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 84981d7cc7cd..ff95469749b7 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -1604,8 +1604,9 @@ retry:
continue;
}
- /* vm_page_rename() will dirty the page. */
- if (vm_page_rename(&pages, new_object, m->pindex -
offidxstart)) {
+ /* vm_page_iter_rename() will dirty the page. */
+ if (!vm_page_iter_rename(&pages, m, new_object, m->pindex -
+ offidxstart)) {
vm_page_xunbusy(m);
VM_OBJECT_WUNLOCK(new_object);
VM_OBJECT_WUNLOCK(orig_object);
@@ -1789,9 +1790,10 @@ vm_object_collapse_scan(vm_object_t object)
* backing object to the main object.
*
* If the page was mapped to a process, it can remain mapped
- * through the rename. vm_page_rename() will dirty the page.
+ * through the rename. vm_page_iter_rename() will dirty the
+ * page.
*/
- if (vm_page_rename(&pages, object, new_pindex)) {
+ if (!vm_page_iter_rename(&pages, p, object, new_pindex)) {
vm_page_xunbusy(p);
next = vm_object_collapse_scan_wait(&pages, object,
NULL);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index a37619c7743e..8a23f900e987 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -2065,10 +2065,14 @@ vm_page_replace(vm_page_t mnew, vm_object_t object,
vm_pindex_t pindex,
}
/*
- * vm_page_rename:
+ * vm_page_iter_rename:
*
- * Move the current page, as identified by iterator, from its current
- * object to the specified target object/offset.
+ * Tries to move the specified page from its current object to a new object
+ * and pindex, using the given iterator to remove the page from its current
+ * object. Returns true if the move was successful, and false if the move
+ * was aborted due to a failed memory allocation.
+ *
+ * Panics if a page already resides in the new object at the new pindex.
*
* Note: swap associated with the page must be invalidated by the move. We
* have to do this for several reasons: (1) we aren't freeing the
@@ -2082,18 +2086,18 @@ vm_page_replace(vm_page_t mnew, vm_object_t object,
vm_pindex_t pindex,
*
* The objects must be locked.
*/
-int
-vm_page_rename(struct pctrie_iter *pages,
+bool
+vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m,
vm_object_t new_object, vm_pindex_t new_pindex)
{
- vm_page_t m, mpred;
+ vm_page_t mpred;
vm_pindex_t opidx;
+ KASSERT((m->ref_count & VPRC_OBJREF) != 0,
+ ("%s: page %p is missing object ref", __func__, m));
+ VM_OBJECT_ASSERT_WLOCKED(m->object);
VM_OBJECT_ASSERT_WLOCKED(new_object);
- m = vm_radix_iter_page(pages);
- KASSERT(m->ref_count != 0, ("vm_page_rename: page %p has no refs", m));
-
/*
* Create a custom version of vm_page_insert() which does not depend
* by m_prev and can cheat on the implementation aspects of the
@@ -2103,7 +2107,7 @@ vm_page_rename(struct pctrie_iter *pages,
m->pindex = new_pindex;
if (vm_radix_insert_lookup_lt(&new_object->rtree, m, &mpred) != 0) {
m->pindex = opidx;
- return (1);
+ return (false);
}
/*
@@ -2111,7 +2115,7 @@ vm_page_rename(struct pctrie_iter *pages,
* the listq iterator is tainted.
*/
m->pindex = opidx;
- vm_radix_iter_remove(pages);
+ vm_radix_iter_remove(old_pages);
vm_page_remove_radixdone(m);
/* Return back to the new pindex to complete vm_page_insert(). */
@@ -2121,7 +2125,7 @@ vm_page_rename(struct pctrie_iter *pages,
vm_page_insert_radixdone(m, new_object, mpred);
vm_page_dirty(m);
vm_pager_page_inserted(new_object, m);
- return (0);
+ return (true);
}
/*
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index f641942efb87..29344eadbf05 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -655,6 +655,8 @@ void vm_page_iter_init(struct pctrie_iter *, vm_object_t);
void vm_page_iter_limit_init(struct pctrie_iter *, vm_object_t, vm_pindex_t);
vm_page_t vm_page_iter_lookup(struct pctrie_iter *, vm_pindex_t);
bool vm_page_iter_remove(struct pctrie_iter *pages);
+bool vm_page_iter_rename(struct pctrie_iter *old_pages, vm_page_t m,
+ vm_object_t new_object, vm_pindex_t new_pindex);
void vm_page_launder(vm_page_t m);
vm_page_t vm_page_lookup(vm_object_t, vm_pindex_t);
vm_page_t vm_page_lookup_unlocked(vm_object_t, vm_pindex_t);
@@ -682,7 +684,6 @@ void vm_page_release_locked(vm_page_t m, int flags);
vm_page_t vm_page_relookup(vm_object_t, vm_pindex_t);
bool vm_page_remove(vm_page_t);
bool vm_page_remove_xbusy(vm_page_t);
-int vm_page_rename(struct pctrie_iter *, vm_object_t, vm_pindex_t);
void vm_page_replace(vm_page_t mnew, vm_object_t object,
vm_pindex_t pindex, vm_page_t mold);
int vm_page_sbusied(vm_page_t m);
diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c
index aa20eca129fb..ad513962d50d 100644
--- a/sys/vm/vm_reserv.c
+++ b/sys/vm/vm_reserv.c
@@ -678,7 +678,7 @@ vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t
pindex, int domain,
if (pa < low || pa + size > high ||
!vm_addr_ok(pa, size, alignment, boundary))
goto out;
- /* Handle vm_page_rename(m, new_object, ...). */
+ /* Handle vm_page_iter_rename(..., m, new_object, ...). */
if (!bit_ntest(rv->popmap, index, index + npages - 1, 0))
goto out;
if (!vm_domain_allocate(vmd, req, npages))
@@ -847,7 +847,7 @@ vm_reserv_alloc_page(vm_object_t object, vm_pindex_t
pindex, int domain,
vm_reserv_lock(rv);
/* Handle reclaim race. */
if (rv->object != object ||
- /* Handle vm_page_rename(m, new_object, ...). */
+ /* Handle vm_page_iter_rename(..., m, new_object, ...). */
bit_test(rv->popmap, index)) {
m = NULL;
goto out;
