Author: dougm Date: Mon Nov 11 16:59:49 2019 New Revision: 354618 URL: https://svnweb.freebsd.org/changeset/base/354618
Log: swap_pager_meta_free() frees allocated blocks in a way that exploits the sparsity of allocated blocks in a range, without issuing an "are you there?" query for every block in the range. swap_pager_copy() is not so smart. Modify the implementation of swap_pager_meta_free() slightly so that swap_pager_copy() can use that smarter implementation too. Based on an observation of: Yoshihiro Ota (ota_j.email.ne.jp) Reviewed by: kib,alc Tested by: pho Differential Revision: https://reviews.freebsd.org/D22280 Modified: head/sys/vm/swap_pager.c Modified: head/sys/vm/swap_pager.c ============================================================================== --- head/sys/vm/swap_pager.c Mon Nov 11 16:11:15 2019 (r354617) +++ head/sys/vm/swap_pager.c Mon Nov 11 16:59:49 2019 (r354618) @@ -422,6 +422,8 @@ static daddr_t swp_pager_getswapspace(int *npages, int */ static daddr_t swp_pager_meta_build(vm_object_t, vm_pindex_t, daddr_t); static void swp_pager_meta_free(vm_object_t, vm_pindex_t, vm_pindex_t); +static void swp_pager_meta_transfer(vm_object_t src, vm_object_t dst, + vm_pindex_t pindex, vm_pindex_t count); static void swp_pager_meta_free_all(vm_object_t); static daddr_t swp_pager_meta_ctl(vm_object_t, vm_pindex_t, int); @@ -933,6 +935,33 @@ swap_pager_reserve(vm_object_t object, vm_pindex_t sta return (0); } +static bool +swp_pager_xfer_source(vm_object_t srcobject, vm_object_t dstobject, + vm_pindex_t pindex, daddr_t addr) +{ + daddr_t dstaddr; + + if (swp_pager_meta_ctl(dstobject, pindex, 0) != SWAPBLK_NONE) { + /* Caller should destroy the source block. */ + return (false); + } + + /* + * Destination has no swapblk and is not resident, transfer source. + * swp_pager_meta_build() can sleep. + */ + vm_object_pip_add(srcobject, 1); + VM_OBJECT_WUNLOCK(srcobject); + vm_object_pip_add(dstobject, 1); + dstaddr = swp_pager_meta_build(dstobject, pindex, addr); + KASSERT(dstaddr == SWAPBLK_NONE, + ("Unexpected destination swapblk")); + vm_object_pip_wakeup(dstobject); + VM_OBJECT_WLOCK(srcobject); + vm_object_pip_wakeup(srcobject); + return (true); +} + /* * SWAP_PAGER_COPY() - copy blocks from source pager to destination pager * and destroy the source. @@ -956,8 +985,6 @@ void swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, vm_pindex_t offset, int destroysource) { - vm_pindex_t i; - daddr_t dstaddr, n_free, s_free, srcaddr; VM_OBJECT_ASSERT_WLOCKED(srcobject); VM_OBJECT_ASSERT_WLOCKED(dstobject); @@ -984,39 +1011,8 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dst /* * Transfer source to destination. */ - swp_pager_init_freerange(&s_free, &n_free); - for (i = 0; i < dstobject->size; ++i) { - srcaddr = swp_pager_meta_ctl(srcobject, i + offset, SWM_POP); - if (srcaddr == SWAPBLK_NONE) - continue; - dstaddr = swp_pager_meta_ctl(dstobject, i, 0); - if (dstaddr != SWAPBLK_NONE) { - /* - * Destination has valid swapblk or it is represented - * by a resident page. We destroy the source block. - */ - swp_pager_update_freerange(&s_free, &n_free, srcaddr); - continue; - } + swp_pager_meta_transfer(srcobject, dstobject, offset, dstobject->size); - /* - * Destination has no swapblk and is not resident, - * copy source. - * - * swp_pager_meta_build() can sleep. - */ - vm_object_pip_add(srcobject, 1); - VM_OBJECT_WUNLOCK(srcobject); - vm_object_pip_add(dstobject, 1); - dstaddr = swp_pager_meta_build(dstobject, i, srcaddr); - KASSERT(dstaddr == SWAPBLK_NONE, - ("Unexpected destination swapblk")); - vm_object_pip_wakeup(dstobject); - VM_OBJECT_WLOCK(srcobject); - vm_object_pip_wakeup(srcobject); - } - swp_pager_freeswapspace(s_free, n_free); - /* * Free left over swap blocks in source. * @@ -2003,31 +1999,30 @@ allocated: } /* - * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata + * SWP_PAGER_META_TRANSFER() - free a range of blocks in the srcobject's swap + * metadata, or transfer it into dstobject. * - * The requested range of blocks is freed, with any associated swap - * returned to the swap bitmap. - * * This routine will free swap metadata structures as they are cleaned - * out. This routine does *NOT* operate on swap metadata associated - * with resident pages. + * out. */ static void -swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) +swp_pager_meta_transfer(vm_object_t srcobject, vm_object_t dstobject, + vm_pindex_t pindex, vm_pindex_t count) { struct swblk *sb; daddr_t n_free, s_free; - vm_pindex_t last; + vm_pindex_t offset, last; int i, limit, start; - VM_OBJECT_ASSERT_WLOCKED(object); - if (object->type != OBJT_SWAP || count == 0) + VM_OBJECT_ASSERT_WLOCKED(srcobject); + if (srcobject->type != OBJT_SWAP || count == 0) return; swp_pager_init_freerange(&s_free, &n_free); + offset = pindex; last = pindex + count; for (;;) { - sb = SWAP_PCTRIE_LOOKUP_GE(&object->un_pager.swp.swp_blks, + sb = SWAP_PCTRIE_LOOKUP_GE(&srcobject->un_pager.swp.swp_blks, rounddown(pindex, SWAP_META_PAGES)); if (sb == NULL || sb->p >= last) break; @@ -2037,18 +2032,39 @@ swp_pager_meta_free(vm_object_t object, vm_pindex_t pi for (i = start; i < limit; i++) { if (sb->d[i] == SWAPBLK_NONE) continue; - swp_pager_update_freerange(&s_free, &n_free, sb->d[i]); + if (dstobject == NULL || + !swp_pager_xfer_source(srcobject, dstobject, + sb->p + i - offset, sb->d[i])) { + swp_pager_update_freerange(&s_free, &n_free, + sb->d[i]); + } sb->d[i] = SWAPBLK_NONE; } pindex = sb->p + SWAP_META_PAGES; if (swp_pager_swblk_empty(sb, 0, start) && swp_pager_swblk_empty(sb, limit, SWAP_META_PAGES)) { - SWAP_PCTRIE_REMOVE(&object->un_pager.swp.swp_blks, + SWAP_PCTRIE_REMOVE(&srcobject->un_pager.swp.swp_blks, sb->p); uma_zfree(swblk_zone, sb); } } swp_pager_freeswapspace(s_free, n_free); +} + +/* + * SWP_PAGER_META_FREE() - free a range of blocks in the object's swap metadata + * + * The requested range of blocks is freed, with any associated swap + * returned to the swap bitmap. + * + * This routine will free swap metadata structures as they are cleaned + * out. This routine does *NOT* operate on swap metadata associated + * with resident pages. + */ +static void +swp_pager_meta_free(vm_object_t object, vm_pindex_t pindex, vm_pindex_t count) +{ + swp_pager_meta_transfer(object, NULL, pindex, count); } /* _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
