Author: jeff Date: Tue Oct 15 03:35:11 2019 New Revision: 353535 URL: https://svnweb.freebsd.org/changeset/base/353535
Log: (1/6) Replace busy checks with acquires where it is trival to do so. This is the first in a series of patches that promotes the page busy field to a first class lock that no longer requires the object lock for consistency. Reviewed by: kib, markj Tested by: pho Sponsored by: Netflix, Intel Differential Revision: https://reviews.freebsd.org/D21548 Modified: head/sys/dev/xen/gntdev/gntdev.c head/sys/dev/xen/privcmd/privcmd.c head/sys/fs/tmpfs/tmpfs_subr.c head/sys/kern/kern_exec.c head/sys/kern/uipc_shm.c head/sys/kern/vfs_bio.c head/sys/kern/vfs_cluster.c head/sys/sys/buf.h head/sys/vm/phys_pager.c head/sys/vm/vm_fault.c head/sys/vm/vm_object.c head/sys/vm/vm_page.c head/sys/vm/vm_page.h head/sys/vm/vm_pageout.c head/sys/vm/vm_swapout.c Modified: head/sys/dev/xen/gntdev/gntdev.c ============================================================================== --- head/sys/dev/xen/gntdev/gntdev.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/dev/xen/gntdev/gntdev.c Tue Oct 15 03:35:11 2019 (r353535) @@ -835,9 +835,9 @@ gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t *mres = NULL; } + vm_page_busy_acquire(page, 0); vm_page_insert(page, object, pidx); page->valid = VM_PAGE_BITS_ALL; - vm_page_xbusy(page); *mres = page; return (VM_PAGER_OK); } Modified: head/sys/dev/xen/privcmd/privcmd.c ============================================================================== --- head/sys/dev/xen/privcmd/privcmd.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/dev/xen/privcmd/privcmd.c Tue Oct 15 03:35:11 2019 (r353535) @@ -178,9 +178,9 @@ privcmd_pg_fault(vm_object_t object, vm_ooffset_t offs *mres = NULL; } + vm_page_busy_acquire(page, 0); vm_page_insert(page, object, pidx); page->valid = VM_PAGE_BITS_ALL; - vm_page_xbusy(page); *mres = page; return (VM_PAGER_OK); } Modified: head/sys/fs/tmpfs/tmpfs_subr.c ============================================================================== --- head/sys/fs/tmpfs/tmpfs_subr.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/fs/tmpfs/tmpfs_subr.c Tue Oct 15 03:35:11 2019 (r353535) @@ -1406,10 +1406,8 @@ tmpfs_reg_resize(struct vnode *vp, off_t newsize, bool if (base != 0) { idx = OFF_TO_IDX(newsize); retry: - m = vm_page_lookup(uobj, idx); + m = vm_page_grab(uobj, idx, VM_ALLOC_NOCREAT); if (m != NULL) { - if (vm_page_sleep_if_busy(m, "tmfssz")) - goto retry; MPASS(m->valid == VM_PAGE_BITS_ALL); } else if (vm_pager_has_page(uobj, idx, NULL, NULL)) { m = vm_page_alloc(uobj, idx, VM_ALLOC_NORMAL | @@ -1430,7 +1428,6 @@ retry: vm_page_lock(m); vm_page_launder(m); vm_page_unlock(m); - vm_page_xunbusy(m); } else { vm_page_free(m); if (ignerr) @@ -1444,6 +1441,7 @@ retry: if (m != NULL) { pmap_zero_page_area(m, base, PAGE_SIZE - base); vm_page_dirty(m); + vm_page_xunbusy(m); vm_pager_page_unswapped(m); } } Modified: head/sys/kern/kern_exec.c ============================================================================== --- head/sys/kern/kern_exec.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/kern/kern_exec.c Tue Oct 15 03:35:11 2019 (r353535) @@ -976,10 +976,14 @@ exec_map_first_page(struct image_params *imgp) #if VM_NRESERVLEVEL > 0 vm_object_color(object, 0); #endif +retry: ma[0] = vm_page_grab(object, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED); if (ma[0]->valid != VM_PAGE_BITS_ALL) { - vm_page_xbusy(ma[0]); + if (vm_page_busy_acquire(ma[0], VM_ALLOC_WAITFAIL) == 0) { + vm_page_unwire_noq(ma[0]); + goto retry; + } if (!vm_pager_has_page(object, 0, NULL, &after)) { if (vm_page_unwire_noq(ma[0])) vm_page_free(ma[0]); Modified: head/sys/kern/uipc_shm.c ============================================================================== --- head/sys/kern/uipc_shm.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/kern/uipc_shm.c Tue Oct 15 03:35:11 2019 (r353535) @@ -457,10 +457,9 @@ shm_dotruncate_locked(struct shmfd *shmfd, off_t lengt if (base != 0) { idx = OFF_TO_IDX(length); retry: - m = vm_page_lookup(object, idx); + m = vm_page_grab(object, idx, VM_ALLOC_NOCREAT); if (m != NULL) { - if (vm_page_sleep_if_busy(m, "shmtrc")) - goto retry; + MPASS(m->valid == VM_PAGE_BITS_ALL); } else if (vm_pager_has_page(object, idx, NULL, NULL)) { m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL); @@ -478,7 +477,6 @@ retry: * as an access. */ vm_page_launder(m); - vm_page_xunbusy(m); } else { vm_page_free(m); VM_OBJECT_WUNLOCK(object); @@ -490,6 +488,7 @@ retry: KASSERT(m->valid == VM_PAGE_BITS_ALL, ("shm_dotruncate: page %p is invalid", m)); vm_page_dirty(m); + vm_page_xunbusy(m); vm_pager_page_unswapped(m); } } Modified: head/sys/kern/vfs_bio.c ============================================================================== --- head/sys/kern/vfs_bio.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/kern/vfs_bio.c Tue Oct 15 03:35:11 2019 (r353535) @@ -2945,10 +2945,10 @@ vfs_vmio_invalidate(struct buf *bp) presid = resid > (PAGE_SIZE - poffset) ? (PAGE_SIZE - poffset) : resid; KASSERT(presid >= 0, ("brelse: extra page")); - while (vm_page_xbusied(m)) - vm_page_sleep_if_xbusy(m, "mbncsh"); + vm_page_busy_acquire(m, VM_ALLOC_SBUSY); if (pmap_page_wired_mappings(m) == 0) vm_page_set_invalid(m, poffset, presid); + vm_page_sunbusy(m); vm_page_release_locked(m, flags); resid -= presid; poffset = 0; @@ -3651,7 +3651,7 @@ vfs_clean_pages_dirty_buf(struct buf *bp) ("vfs_clean_pages_dirty_buf: no buffer offset")); VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); - vfs_drain_busy_pages(bp); + vfs_busy_pages_acquire(bp); vfs_setdirty_locked_object(bp); for (i = 0; i < bp->b_npages; i++) { noff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; @@ -3663,6 +3663,7 @@ vfs_clean_pages_dirty_buf(struct buf *bp) /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */ foff = noff; } + vfs_busy_pages_release(bp); VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); } @@ -4559,28 +4560,25 @@ vfs_page_set_validclean(struct buf *bp, vm_ooffset_t o } /* - * Ensure that all buffer pages are not exclusive busied. If any page is - * exclusive busy, drain it. + * Acquire a shared busy on all pages in the buf. */ void -vfs_drain_busy_pages(struct buf *bp) +vfs_busy_pages_acquire(struct buf *bp) { - vm_page_t m; - int i, last_busied; + int i; VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object); - last_busied = 0; - for (i = 0; i < bp->b_npages; i++) { - m = bp->b_pages[i]; - if (vm_page_xbusied(m)) { - for (; last_busied < i; last_busied++) - vm_page_sbusy(bp->b_pages[last_busied]); - while (vm_page_xbusied(m)) { - vm_page_sleep_if_xbusy(m, "vbpage"); - } - } - } - for (i = 0; i < last_busied; i++) + for (i = 0; i < bp->b_npages; i++) + vm_page_busy_acquire(bp->b_pages[i], VM_ALLOC_SBUSY); +} + +void +vfs_busy_pages_release(struct buf *bp) +{ + int i; + + VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object); + for (i = 0; i < bp->b_npages; i++) vm_page_sunbusy(bp->b_pages[i]); } @@ -4613,17 +4611,17 @@ vfs_busy_pages(struct buf *bp, int clear_modify) KASSERT(bp->b_offset != NOOFFSET, ("vfs_busy_pages: no buffer offset")); VM_OBJECT_WLOCK(obj); - vfs_drain_busy_pages(bp); + if ((bp->b_flags & B_CLUSTER) == 0) { + vm_object_pip_add(obj, bp->b_npages); + vfs_busy_pages_acquire(bp); + } if (bp->b_bufsize != 0) vfs_setdirty_locked_object(bp); bogus = false; for (i = 0; i < bp->b_npages; i++) { m = bp->b_pages[i]; + vm_page_assert_sbusied(m); - if ((bp->b_flags & B_CLUSTER) == 0) { - vm_object_pip_add(obj, 1); - vm_page_sbusy(m); - } /* * When readying a buffer for a read ( i.e * clear_modify == 0 ), it is important to do Modified: head/sys/kern/vfs_cluster.c ============================================================================== --- head/sys/kern/vfs_cluster.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/kern/vfs_cluster.c Tue Oct 15 03:35:11 2019 (r353535) @@ -418,11 +418,9 @@ cluster_rbuild(struct vnode *vp, u_quad_t filesize, da for (bn = blkno, i = 0; i < run; ++i, bn += inc) { if (i == 0) { VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); - vfs_drain_busy_pages(tbp); vm_object_pip_add(tbp->b_bufobj->bo_object, tbp->b_npages); - for (k = 0; k < tbp->b_npages; k++) - vm_page_sbusy(tbp->b_pages[k]); + vfs_busy_pages_acquire(tbp); VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); } else { if ((bp->b_npages * PAGE_SIZE) + @@ -470,10 +468,9 @@ cluster_rbuild(struct vnode *vp, u_quad_t filesize, da if ((tbp->b_pages[j]->valid & vm_page_bits(toff, tinc)) != 0) break; - if (vm_page_xbusied(tbp->b_pages[j])) + if (vm_page_trysbusy(tbp->b_pages[j]) == 0) break; vm_object_pip_add(tbp->b_bufobj->bo_object, 1); - vm_page_sbusy(tbp->b_pages[j]); off += tinc; tsize -= tinc; } @@ -991,11 +988,14 @@ cluster_wbuild(struct vnode *vp, long size, daddr_t st VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); if (i == 0) { - vfs_drain_busy_pages(tbp); + vfs_busy_pages_acquire(tbp); } else { /* if not first buffer */ for (j = 0; j < tbp->b_npages; j += 1) { m = tbp->b_pages[j]; - if (vm_page_xbusied(m)) { + if (vm_page_trysbusy(m) == 0) { + for (j--; j >= 0; j--) + vm_page_sunbusy( + tbp->b_pages[j]); VM_OBJECT_WUNLOCK( tbp->b_object); bqrelse(tbp); @@ -1003,10 +1003,10 @@ cluster_wbuild(struct vnode *vp, long size, daddr_t st } } } + vm_object_pip_add(tbp->b_bufobj->bo_object, + tbp->b_npages); for (j = 0; j < tbp->b_npages; j += 1) { m = tbp->b_pages[j]; - vm_page_sbusy(m); - vm_object_pip_add(m->object, 1); if ((bp->b_npages == 0) || (bp->b_pages[bp->b_npages - 1] != m)) { bp->b_pages[bp->b_npages] = m; Modified: head/sys/sys/buf.h ============================================================================== --- head/sys/sys/buf.h Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/sys/buf.h Tue Oct 15 03:35:11 2019 (r353535) @@ -539,7 +539,8 @@ void bufstrategy(struct bufobj *, struct buf *); void brelse(struct buf *); void bqrelse(struct buf *); int vfs_bio_awrite(struct buf *); -void vfs_drain_busy_pages(struct buf *bp); +void vfs_busy_pages_acquire(struct buf *bp); +void vfs_busy_pages_release(struct buf *bp); struct buf *incore(struct bufobj *, daddr_t); struct buf *gbincore(struct bufobj *, daddr_t); struct buf *getblk(struct vnode *, daddr_t, int, int, int, int); Modified: head/sys/vm/phys_pager.c ============================================================================== --- head/sys/vm/phys_pager.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/vm/phys_pager.c Tue Oct 15 03:35:11 2019 (r353535) @@ -206,29 +206,13 @@ phys_pager_populate(vm_object_t object, vm_pindex_t pi *last = end; for (i = base; i <= end; i++) { -retry: - m = vm_page_lookup(object, i); - if (m == NULL) { - ahead = MIN(end - i, PHYSALLOC); - m = vm_page_alloc(object, i, VM_ALLOC_NORMAL | - VM_ALLOC_ZERO | VM_ALLOC_WAITFAIL | - VM_ALLOC_COUNT(ahead)); - if (m == NULL) - goto retry; - if ((m->flags & PG_ZERO) == 0) - pmap_zero_page(m); + ahead = MIN(end - i, PHYSALLOC); + m = vm_page_grab(object, i, + VM_ALLOC_NORMAL | VM_ALLOC_COUNT(ahead)); + if (m->valid != VM_PAGE_BITS_ALL) { + vm_page_zero_invalid(m, TRUE); m->valid = VM_PAGE_BITS_ALL; - } else if (vm_page_xbusied(m)) { - vm_page_sleep_if_xbusy(m, "physb"); - goto retry; - } else { - vm_page_xbusy(m); - if (m->valid != VM_PAGE_BITS_ALL) - vm_page_zero_invalid(m, TRUE); } - - KASSERT(m->valid == VM_PAGE_BITS_ALL, - ("phys_pager_populate: partially valid page %p", m)); KASSERT(m->dirty == 0, ("phys_pager_populate: dirty page %p", m)); } Modified: head/sys/vm/vm_fault.c ============================================================================== --- head/sys/vm/vm_fault.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/vm/vm_fault.c Tue Oct 15 03:35:11 2019 (r353535) @@ -787,7 +787,7 @@ RetryFault_oom: * around with a shared busied page except, perhaps, * to pmap it. */ - if (vm_page_busied(fs.m)) { + if (vm_page_tryxbusy(fs.m) == 0) { /* * Reference the page before unlocking and * sleeping so that the page daemon is less @@ -819,12 +819,11 @@ RetryFault_oom: } /* - * Mark page busy for other processes, and the + * The page is marked busy for other processes and the * pagedaemon. If it still isn't completely valid * (readable), jump to readrest, else break-out ( we * found the page ). */ - vm_page_xbusy(fs.m); if (fs.m->valid != VM_PAGE_BITS_ALL) goto readrest; break; /* break to PAGE HAS BEEN FOUND */ @@ -1826,16 +1825,17 @@ again: dst_m->dirty = dst_m->valid = src_m->valid; } else { dst_m = src_m; - if (vm_page_sleep_if_busy(dst_m, "fltupg")) + if (vm_page_busy_acquire(dst_m, VM_ALLOC_WAITFAIL) == 0) goto again; - if (dst_m->pindex >= dst_object->size) + if (dst_m->pindex >= dst_object->size) { /* * We are upgrading. Index can occur * out of bounds if the object type is * vnode and the file was truncated. */ + vm_page_xunbusy(dst_m); break; - vm_page_xbusy(dst_m); + } } VM_OBJECT_WUNLOCK(dst_object); Modified: head/sys/vm/vm_object.c ============================================================================== --- head/sys/vm/vm_object.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/vm/vm_object.c Tue Oct 15 03:35:11 2019 (r353535) @@ -838,7 +838,7 @@ rescan: np = TAILQ_NEXT(p, listq); if (p->valid == 0) continue; - if (vm_page_sleep_if_busy(p, "vpcwai")) { + if (vm_page_busy_acquire(p, VM_ALLOC_WAITFAIL) == 0) { if (object->generation != curgeneration) { if ((flags & OBJPC_SYNC) != 0) goto rescan; @@ -848,8 +848,10 @@ rescan: np = vm_page_find_least(object, pi); continue; } - if (!vm_object_page_remove_write(p, flags, &clearobjflags)) + if (!vm_object_page_remove_write(p, flags, &clearobjflags)) { + vm_page_xunbusy(p); continue; + } n = vm_object_page_collect_flush(object, p, pagerflags, flags, &clearobjflags, &eio); @@ -899,6 +901,7 @@ vm_object_page_collect_flush(vm_object_t object, vm_pa int count, i, mreq, runlen; vm_page_lock_assert(p, MA_NOTOWNED); + vm_page_assert_xbusied(p); VM_OBJECT_ASSERT_WLOCKED(object); count = 1; @@ -906,18 +909,22 @@ vm_object_page_collect_flush(vm_object_t object, vm_pa for (tp = p; count < vm_pageout_page_count; count++) { tp = vm_page_next(tp); - if (tp == NULL || vm_page_busied(tp)) + if (tp == NULL || vm_page_tryxbusy(tp) == 0) break; - if (!vm_object_page_remove_write(tp, flags, clearobjflags)) + if (!vm_object_page_remove_write(tp, flags, clearobjflags)) { + vm_page_xunbusy(tp); break; + } } for (p_first = p; count < vm_pageout_page_count; count++) { tp = vm_page_prev(p_first); - if (tp == NULL || vm_page_busied(tp)) + if (tp == NULL || vm_page_tryxbusy(tp) == 0) break; - if (!vm_object_page_remove_write(tp, flags, clearobjflags)) + if (!vm_object_page_remove_write(tp, flags, clearobjflags)) { + vm_page_xunbusy(tp); break; + } p_first = tp; mreq++; } @@ -1158,7 +1165,7 @@ next_page: ("vm_object_madvise: page %p is fictitious", tm)); KASSERT((tm->oflags & VPO_UNMANAGED) == 0, ("vm_object_madvise: page %p is not managed", tm)); - if (vm_page_busied(tm)) { + if (vm_page_tryxbusy(tm) == 0) { if (object != tobject) VM_OBJECT_WUNLOCK(object); if (advice == MADV_WILLNEED) { @@ -1175,6 +1182,7 @@ next_page: vm_page_lock(tm); vm_page_advise(tm, advice); vm_page_unlock(tm); + vm_page_xunbusy(tm); vm_object_madvise_freespace(tobject, advice, tm->pindex, 1); next_pindex: if (tobject != object) @@ -1341,7 +1349,7 @@ retry: * We do not have to VM_PROT_NONE the page as mappings should * not be changed by this operation. */ - if (vm_page_busied(m)) { + if (vm_page_tryxbusy(m) == 0) { VM_OBJECT_WUNLOCK(new_object); vm_page_sleep_if_busy(m, "spltwt"); VM_OBJECT_WLOCK(new_object); @@ -1350,6 +1358,7 @@ retry: /* vm_page_rename() will dirty the page. */ if (vm_page_rename(m, new_object, idx)) { + vm_page_xunbusy(m); VM_OBJECT_WUNLOCK(new_object); VM_OBJECT_WUNLOCK(orig_object); vm_radix_wait(); @@ -1357,6 +1366,8 @@ retry: VM_OBJECT_WLOCK(new_object); goto retry; } + /* Rename released the xbusy lock. */ + #if VM_NRESERVLEVEL > 0 /* * If some of the reservation's allocated pages remain with @@ -1405,7 +1416,6 @@ vm_object_collapse_scan_wait(vm_object_t object, vm_pa backing_object = object->backing_object; VM_OBJECT_ASSERT_WLOCKED(backing_object); - KASSERT(p == NULL || vm_page_busied(p), ("unbusy page %p", p)); KASSERT(p == NULL || p->object == object || p->object == backing_object, ("invalid ownership %p %p %p", p, object, backing_object)); if ((op & OBSC_COLLAPSE_NOWAIT) != 0) @@ -1510,7 +1520,7 @@ vm_object_collapse_scan(vm_object_t object, int op) /* * Check for busy page */ - if (vm_page_busied(p)) { + if (vm_page_tryxbusy(p) == 0) { next = vm_object_collapse_scan_wait(object, p, next, op); continue; } @@ -1532,7 +1542,8 @@ vm_object_collapse_scan(vm_object_t object, int op) } pp = vm_page_lookup(object, new_pindex); - if (pp != NULL && vm_page_busied(pp)) { + if (pp != NULL && vm_page_tryxbusy(pp) == 0) { + vm_page_xunbusy(p); /* * The page in the parent is busy and possibly not * (yet) valid. Until its state is finalized by the @@ -1568,6 +1579,8 @@ vm_object_collapse_scan(vm_object_t object, int op) ("freeing mapped page %p", p)); if (vm_page_remove(p)) vm_page_free(p); + if (pp != NULL) + vm_page_xunbusy(pp); continue; } @@ -1579,10 +1592,14 @@ vm_object_collapse_scan(vm_object_t object, int op) * through the rename. vm_page_rename() will dirty the page. */ if (vm_page_rename(p, object, new_pindex)) { + vm_page_xunbusy(p); + if (pp != NULL) + vm_page_xunbusy(pp); next = vm_object_collapse_scan_wait(object, NULL, next, op); continue; } + /* Rename released the xbusy lock. */ /* Use the old pindex to free the right page. */ if (backing_object->type == OBJT_SWAP) @@ -1859,7 +1876,7 @@ again: * however, be invalidated if the option OBJPR_CLEANONLY is * not specified. */ - if (vm_page_busied(p)) { + if (vm_page_tryxbusy(p) == 0) { vm_page_sleep_if_busy(p, "vmopar"); goto again; } @@ -1872,6 +1889,7 @@ wired: p->valid = 0; vm_page_undirty(p); } + vm_page_xunbusy(p); continue; } KASSERT((p->flags & PG_FICTITIOUS) == 0, @@ -1881,8 +1899,10 @@ wired: object->ref_count != 0 && !vm_page_try_remove_write(p)) goto wired; - if (p->dirty != 0) + if (p->dirty != 0) { + vm_page_xunbusy(p); continue; + } } if ((options & OBJPR_NOTMAPPED) == 0 && object->ref_count != 0 && !vm_page_try_remove_all(p)) @@ -2168,7 +2188,7 @@ again: tm = m; m = TAILQ_NEXT(m, listq); } - if (vm_page_xbusied(tm)) { + if (vm_page_trysbusy(tm) == 0) { for (tobject = object; locked_depth >= 1; locked_depth--) { t1object = tobject->backing_object; @@ -2180,6 +2200,7 @@ again: goto again; } vm_page_unwire(tm, queue); + vm_page_sunbusy(tm); next_page: pindex++; } Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/vm/vm_page.c Tue Oct 15 03:35:11 2019 (r353535) @@ -972,6 +972,32 @@ vm_page_busy_downgrade(vm_page_t m) } /* + * + * vm_page_busy_tryupgrade: + * + * Attempt to upgrade a single shared busy into an exclusive busy. + */ +int +vm_page_busy_tryupgrade(vm_page_t m) +{ + u_int x; + + vm_page_assert_sbusied(m); + + x = m->busy_lock; + for (;;) { + if (VPB_SHARERS(x) > 1) + return (0); + KASSERT((x & ~VPB_BIT_WAITERS) == VPB_SHARERS_WORD(1), + ("vm_page_busy_tryupgrade: invalid lock state")); + if (!atomic_fcmpset_acq_int(&m->busy_lock, &x, + VPB_SINGLE_EXCLUSIVER | (x & VPB_BIT_WAITERS))) + continue; + return (1); + } +} + +/* * vm_page_sbusied: * * Return a positive value if the page is shared busied, 0 otherwise. @@ -2570,7 +2596,12 @@ retry: else if (object->memattr != VM_MEMATTR_DEFAULT) error = EINVAL; else if (vm_page_queue(m) != PQ_NONE && - !vm_page_busied(m) && !vm_page_wired(m)) { + vm_page_tryxbusy(m) != 0) { + if (vm_page_wired(m)) { + vm_page_xunbusy(m); + error = EBUSY; + goto unlock; + } KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, ("page %p has an unexpected memattr", m)); @@ -2616,6 +2647,7 @@ retry: VM_MEMATTR_DEFAULT); } if (m_new == NULL) { + vm_page_xunbusy(m); error = ENOMEM; goto unlock; } @@ -2647,7 +2679,6 @@ retry: m_new->valid = m->valid; m_new->dirty = m->dirty; m->flags &= ~PG_ZERO; - vm_page_xbusy(m); vm_page_dequeue(m); vm_page_replace_checked(m_new, object, m->pindex, m); @@ -4046,8 +4077,8 @@ vm_page_try_blocked_op(vm_page_t m, void (*op)(vm_page KASSERT(m->object != NULL && (m->oflags & VPO_UNMANAGED) == 0, ("vm_page_try_blocked_op: page %p has no object", m)); - KASSERT(!vm_page_busied(m), - ("vm_page_try_blocked_op: page %p is busy", m)); + KASSERT(vm_page_busied(m), + ("vm_page_try_blocked_op: page %p is not busy", m)); VM_OBJECT_ASSERT_LOCKED(m->object); old = m->ref_count; @@ -4163,13 +4194,18 @@ vm_page_grab(vm_object_t object, vm_pindex_t pindex, i (allocflags & VM_ALLOC_IGN_SBUSY) != 0, ("vm_page_grab: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch")); pflags = allocflags & - ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL); + ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL | + VM_ALLOC_NOBUSY); if ((allocflags & VM_ALLOC_NOWAIT) == 0) pflags |= VM_ALLOC_WAITFAIL; + if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) + pflags |= VM_ALLOC_SBUSY; retrylookup: if ((m = vm_page_lookup(object, pindex)) != NULL) { - sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ? - vm_page_xbusied(m) : vm_page_busied(m); + if ((allocflags & (VM_ALLOC_IGN_SBUSY | VM_ALLOC_SBUSY)) != 0) + sleep = !vm_page_trysbusy(m); + else + sleep = !vm_page_tryxbusy(m); if (sleep) { if ((allocflags & VM_ALLOC_NOWAIT) != 0) return (NULL); @@ -4189,12 +4225,7 @@ retrylookup: } else { if ((allocflags & VM_ALLOC_WIRED) != 0) vm_page_wire(m); - if ((allocflags & - (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) == 0) - vm_page_xbusy(m); - else if ((allocflags & VM_ALLOC_SBUSY) != 0) - vm_page_sbusy(m); - return (m); + goto out; } } if ((allocflags & VM_ALLOC_NOCREAT) != 0) @@ -4207,6 +4238,14 @@ retrylookup: } if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0) pmap_zero_page(m); + +out: + if ((allocflags & VM_ALLOC_NOBUSY) != 0) { + if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) + vm_page_sunbusy(m); + else + vm_page_xunbusy(m); + } return (m); } @@ -4359,10 +4398,13 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pin ("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch")); if (count == 0) return (0); - pflags = allocflags & ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | - VM_ALLOC_WAITFAIL | VM_ALLOC_IGN_SBUSY); + pflags = allocflags & + ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL | + VM_ALLOC_NOBUSY); if ((allocflags & VM_ALLOC_NOWAIT) == 0) pflags |= VM_ALLOC_WAITFAIL; + if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) + pflags |= VM_ALLOC_SBUSY; i = 0; retrylookup: m = vm_radix_lookup_le(&object->rtree, pindex + i); @@ -4373,8 +4415,11 @@ retrylookup: mpred = TAILQ_PREV(m, pglist, listq); for (; i < count; i++) { if (m != NULL) { - sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ? - vm_page_xbusied(m) : vm_page_busied(m); + if ((allocflags & + (VM_ALLOC_SBUSY | VM_ALLOC_IGN_SBUSY)) != 0) + sleep = !vm_page_trysbusy(m); + else + sleep = !vm_page_tryxbusy(m); if (sleep) { if ((allocflags & VM_ALLOC_NOWAIT) != 0) break; @@ -4392,11 +4437,6 @@ retrylookup: } if ((allocflags & VM_ALLOC_WIRED) != 0) vm_page_wire(m); - if ((allocflags & (VM_ALLOC_NOBUSY | - VM_ALLOC_SBUSY)) == 0) - vm_page_xbusy(m); - if ((allocflags & VM_ALLOC_SBUSY) != 0) - vm_page_sbusy(m); } else { if ((allocflags & VM_ALLOC_NOCREAT) != 0) break; @@ -4412,6 +4452,12 @@ retrylookup: if ((m->flags & PG_ZERO) == 0) pmap_zero_page(m); m->valid = VM_PAGE_BITS_ALL; + } + if ((allocflags & VM_ALLOC_NOBUSY) != 0) { + if ((allocflags & VM_ALLOC_IGN_SBUSY) != 0) + vm_page_sunbusy(m); + else + vm_page_xunbusy(m); } ma[i] = mpred = m; m = vm_page_next(m); Modified: head/sys/vm/vm_page.h ============================================================================== --- head/sys/vm/vm_page.h Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/vm/vm_page.h Tue Oct 15 03:35:11 2019 (r353535) @@ -542,6 +542,7 @@ malloc2vm_flags(int malloc_flags) int vm_page_busy_acquire(vm_page_t m, int allocflags); void vm_page_busy_downgrade(vm_page_t m); +int vm_page_busy_tryupgrade(vm_page_t m); void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared); void vm_page_free(vm_page_t m); void vm_page_free_zero(vm_page_t m); Modified: head/sys/vm/vm_pageout.c ============================================================================== --- head/sys/vm/vm_pageout.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/vm/vm_pageout.c Tue Oct 15 03:35:11 2019 (r353535) @@ -334,7 +334,7 @@ vm_pageout_cluster(vm_page_t m) VM_OBJECT_ASSERT_WLOCKED(object); pindex = m->pindex; - vm_page_assert_unbusied(m); + vm_page_assert_xbusied(m); mc[vm_pageout_page_count] = pb = ps = m; pageout_count = 1; @@ -360,19 +360,26 @@ more: ib = 0; break; } - if ((p = vm_page_prev(pb)) == NULL || vm_page_busied(p) || - vm_page_wired(p)) { + if ((p = vm_page_prev(pb)) == NULL || + vm_page_tryxbusy(p) == 0) { ib = 0; break; } + if (vm_page_wired(p)) { + ib = 0; + vm_page_xunbusy(p); + break; + } vm_page_test_dirty(p); if (p->dirty == 0) { ib = 0; + vm_page_xunbusy(p); break; } vm_page_lock(p); if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) { vm_page_unlock(p); + vm_page_xunbusy(p); ib = 0; break; } @@ -390,15 +397,22 @@ more: } while (pageout_count < vm_pageout_page_count && pindex + is < object->size) { - if ((p = vm_page_next(ps)) == NULL || vm_page_busied(p) || - vm_page_wired(p)) + if ((p = vm_page_next(ps)) == NULL || + vm_page_tryxbusy(p) == 0) break; + if (vm_page_wired(p)) { + vm_page_xunbusy(p); + break; + } vm_page_test_dirty(p); - if (p->dirty == 0) + if (p->dirty == 0) { + vm_page_xunbusy(p); break; + } vm_page_lock(p); if (!vm_page_in_laundry(p) || !vm_page_try_remove_write(p)) { vm_page_unlock(p); + vm_page_xunbusy(p); break; } vm_page_unlock(p); @@ -445,8 +459,8 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags, VM_OBJECT_ASSERT_WLOCKED(object); /* - * Initiate I/O. Mark the pages busy and verify that they're valid - * and read-only. + * Initiate I/O. Mark the pages shared busy and verify that they're + * valid and read-only. * * We do not have to fixup the clean/dirty bits here... we can * allow the pager to do it after the I/O completes. @@ -460,7 +474,7 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags, mc[i], i, count)); KASSERT((mc[i]->aflags & PGA_WRITEABLE) == 0, ("vm_pageout_flush: writeable page %p", mc[i])); - vm_page_sbusy(mc[i]); + vm_page_busy_downgrade(mc[i]); } vm_object_pip_add(object, count); @@ -598,6 +612,7 @@ vm_pageout_clean(vm_page_t m, int *numpagedout) */ if (object->type == OBJT_VNODE) { vm_page_unlock(m); + vm_page_xunbusy(m); vp = object->handle; if (vp->v_type == VREG && vn_start_write(vp, &mp, V_NOWAIT) != 0) { @@ -648,7 +663,7 @@ vm_pageout_clean(vm_page_t m, int *numpagedout) * The page may have been busied while the object and page * locks were released. */ - if (vm_page_busied(m)) { + if (vm_page_tryxbusy(m) == 0) { vm_page_unlock(m); error = EBUSY; goto unlock_all; @@ -659,6 +674,7 @@ vm_pageout_clean(vm_page_t m, int *numpagedout) * Remove all writeable mappings, failing if the page is wired. */ if (!vm_page_try_remove_write(m)) { + vm_page_xunbusy(m); vm_page_unlock(m); error = EBUSY; goto unlock_all; @@ -792,7 +808,7 @@ recheck: KASSERT(m->object == object, ("page %p does not belong to %p", m, object)); - if (vm_page_busied(m)) + if (vm_page_tryxbusy(m) == 0) continue; /* @@ -804,6 +820,7 @@ recheck: * wire count is guaranteed not to increase. */ if (__predict_false(vm_page_wired(m))) { + vm_page_xunbusy(m); vm_page_dequeue_deferred(m); continue; } @@ -837,6 +854,7 @@ recheck: } if (act_delta != 0) { if (object->ref_count != 0) { + vm_page_xunbusy(m); VM_CNT_INC(v_reactivated); vm_page_activate(m); @@ -861,6 +879,7 @@ recheck: launder--; continue; } else if ((object->flags & OBJ_DEAD) == 0) { + vm_page_xunbusy(m); vm_page_requeue(m); continue; } @@ -876,6 +895,7 @@ recheck: if (object->ref_count != 0) { vm_page_test_dirty(m); if (m->dirty == 0 && !vm_page_try_remove_all(m)) { + vm_page_xunbusy(m); vm_page_dequeue_deferred(m); continue; } @@ -900,6 +920,7 @@ free_page: else pageout_ok = true; if (!pageout_ok) { + vm_page_xunbusy(m); vm_page_requeue(m); continue; } @@ -927,7 +948,8 @@ free_page: } mtx = NULL; object = NULL; - } + } else + vm_page_xunbusy(m); } if (mtx != NULL) { mtx_unlock(mtx); @@ -1507,7 +1529,7 @@ recheck: KASSERT(m->object == object, ("page %p does not belong to %p", m, object)); - if (vm_page_busied(m)) { + if (vm_page_tryxbusy(m) == 0) { /* * Don't mess with busy pages. Leave them at * the front of the queue. Most likely, they @@ -1529,6 +1551,7 @@ recheck: * wire count is guaranteed not to increase. */ if (__predict_false(vm_page_wired(m))) { + vm_page_xunbusy(m); vm_page_dequeue_deferred(m); continue; } @@ -1562,6 +1585,7 @@ recheck: } if (act_delta != 0) { if (object->ref_count != 0) { + vm_page_xunbusy(m); VM_CNT_INC(v_reactivated); vm_page_activate(m); @@ -1575,6 +1599,7 @@ recheck: m->act_count += act_delta + ACT_ADVANCE; continue; } else if ((object->flags & OBJ_DEAD) == 0) { + vm_page_xunbusy(m); vm_page_aflag_set(m, PGA_REQUEUE); goto reinsert; } @@ -1590,6 +1615,7 @@ recheck: if (object->ref_count != 0) { vm_page_test_dirty(m); if (m->dirty == 0 && !vm_page_try_remove_all(m)) { + vm_page_xunbusy(m); vm_page_dequeue_deferred(m); continue; } @@ -1615,7 +1641,10 @@ free_page: m->queue = PQ_NONE; vm_page_free(m); page_shortage--; - } else if ((object->flags & OBJ_DEAD) == 0) + continue; + } + vm_page_xunbusy(m); + if ((object->flags & OBJ_DEAD) == 0) vm_page_launder(m); continue; reinsert: Modified: head/sys/vm/vm_swapout.c ============================================================================== --- head/sys/vm/vm_swapout.c Mon Oct 14 22:29:20 2019 (r353534) +++ head/sys/vm/vm_swapout.c Tue Oct 15 03:35:11 2019 (r353535) @@ -208,6 +208,9 @@ vm_swapout_object_deactivate_pages(pmap_t pmap, vm_obj goto unlock_return; if (should_yield()) goto unlock_return; + if (vm_page_tryxbusy(p) == 0) + continue; + VM_CNT_INC(v_pdpages); /* * The page may acquire a wiring after this check. @@ -215,11 +218,10 @@ vm_swapout_object_deactivate_pages(pmap_t pmap, vm_obj * no harm done if a wiring appears while we are * attempting to deactivate the page. */ - if (vm_page_busied(p) || vm_page_wired(p)) + if (vm_page_wired(p) || !pmap_page_exists_quick(pmap, p)) { + vm_page_xunbusy(p); continue; - VM_CNT_INC(v_pdpages); - if (!pmap_page_exists_quick(pmap, p)) - continue; + } act_delta = pmap_ts_referenced(p); vm_page_lock(p); if ((p->aflags & PGA_REFERENCED) != 0) { @@ -251,6 +253,7 @@ vm_swapout_object_deactivate_pages(pmap_t pmap, vm_obj } else if (vm_page_inactive(p)) (void)vm_page_try_remove_all(p); vm_page_unlock(p); + vm_page_xunbusy(p); *** DIFF OUTPUT TRUNCATED AT 1000 LINES *** _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
