Author: jeff Date: Wed Nov 27 00:39:23 2019 New Revision: 355122 URL: https://svnweb.freebsd.org/changeset/base/355122
Log: Use atomics in more cases for object references. We now can completely omit the object lock if we are above a certain threshold. Hold only a single vnode reference when the vnode object has any ref > 0. This allows us to only lock the object and vnode on 0-1 and 1-0 transitions. Differential Revision: https://reviews.freebsd.org/D22452 Modified: head/sys/sys/refcount.h head/sys/vm/vm_object.c head/sys/vm/vnode_pager.c Modified: head/sys/sys/refcount.h ============================================================================== --- head/sys/sys/refcount.h Tue Nov 26 22:17:02 2019 (r355121) +++ head/sys/sys/refcount.h Wed Nov 27 00:39:23 2019 (r355122) @@ -72,7 +72,7 @@ refcount_init(volatile u_int *count, u_int value) *count = value; } -static __inline void +static __inline u_int refcount_acquire(volatile u_int *count) { u_int old; @@ -80,9 +80,11 @@ refcount_acquire(volatile u_int *count) old = atomic_fetchadd_int(count, 1); if (__predict_false(REFCOUNT_SATURATED(old))) _refcount_update_saturated(count); + + return (old); } -static __inline void +static __inline u_int refcount_acquiren(volatile u_int *count, u_int n) { u_int old; @@ -92,6 +94,8 @@ refcount_acquiren(volatile u_int *count, u_int n) old = atomic_fetchadd_int(count, n); if (__predict_false(REFCOUNT_SATURATED(old))) _refcount_update_saturated(count); + + return (old); } static __inline __result_use_check bool @@ -144,13 +148,13 @@ refcount_wait(volatile u_int *count, const char *wmesg * incremented. Else zero is returned. */ static __inline __result_use_check bool -refcount_acquire_if_not_zero(volatile u_int *count) +refcount_acquire_if_gt(volatile u_int *count, u_int n) { u_int old; old = *count; for (;;) { - if (REFCOUNT_COUNT(old) == 0) + if (REFCOUNT_COUNT(old) <= n) return (false); if (__predict_false(REFCOUNT_SATURATED(old))) return (true); @@ -160,19 +164,10 @@ refcount_acquire_if_not_zero(volatile u_int *count) } static __inline __result_use_check bool -refcount_release_if_not_last(volatile u_int *count) +refcount_acquire_if_not_zero(volatile u_int *count) { - u_int old; - old = *count; - for (;;) { - if (REFCOUNT_COUNT(old) == 1) - return (false); - if (__predict_false(REFCOUNT_SATURATED(old))) - return (true); - if (atomic_fcmpset_int(count, &old, old - 1)) - return (true); - } + return refcount_acquire_if_gt(count, 0); } static __inline __result_use_check bool @@ -193,4 +188,10 @@ refcount_release_if_gt(volatile u_int *count, u_int n) } } +static __inline __result_use_check bool +refcount_release_if_not_last(volatile u_int *count) +{ + + return refcount_release_if_gt(count, 1); +} #endif /* ! __SYS_REFCOUNT_H__ */ Modified: head/sys/vm/vm_object.c ============================================================================== --- head/sys/vm/vm_object.c Tue Nov 26 22:17:02 2019 (r355121) +++ head/sys/vm/vm_object.c Wed Nov 27 00:39:23 2019 (r355122) @@ -468,11 +468,28 @@ vm_object_allocate_anon(vm_pindex_t size) void vm_object_reference(vm_object_t object) { + struct vnode *vp; + u_int old; + if (object == NULL) return; - VM_OBJECT_RLOCK(object); - vm_object_reference_locked(object); - VM_OBJECT_RUNLOCK(object); + + /* + * Many places assume exclusive access to objects with a single + * ref. vm_object_collapse() in particular will directly mainpulate + * references for objects in this state. vnode objects only need + * the lock for the first ref to reference the vnode. + */ + if (!refcount_acquire_if_gt(&object->ref_count, + object->type == OBJT_VNODE ? 0 : 1)) { + VM_OBJECT_RLOCK(object); + old = refcount_acquire(&object->ref_count); + if (object->type == OBJT_VNODE && old == 0) { + vp = object->handle; + vref(vp); + } + VM_OBJECT_RUNLOCK(object); + } } /* @@ -486,10 +503,11 @@ void vm_object_reference_locked(vm_object_t object) { struct vnode *vp; + u_int old; VM_OBJECT_ASSERT_LOCKED(object); - refcount_acquire(&object->ref_count); - if (object->type == OBJT_VNODE) { + old = refcount_acquire(&object->ref_count); + if (object->type == OBJT_VNODE && old == 0) { vp = object->handle; vref(vp); } @@ -507,11 +525,10 @@ vm_object_vndeallocate(vm_object_t object) ("vm_object_vndeallocate: not a vnode object")); KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); - if (refcount_release(&object->ref_count) && - !umtx_shm_vnobj_persistent) + if (!umtx_shm_vnobj_persistent) umtx_shm_object_terminated(object); - VM_OBJECT_RUNLOCK(object); + VM_OBJECT_WUNLOCK(object); /* vrele may need the vnode lock. */ vrele(vp); } @@ -531,15 +548,9 @@ void vm_object_deallocate(vm_object_t object) { vm_object_t robject, temp; - bool released; + bool last, released; while (object != NULL) { - VM_OBJECT_RLOCK(object); - if (object->type == OBJT_VNODE) { - vm_object_vndeallocate(object); - return; - } - /* * If the reference count goes to 0 we start calling * vm_object_terminate() on the object chain. A ref count @@ -551,7 +562,6 @@ vm_object_deallocate(vm_object_t object) released = refcount_release_if_gt(&object->ref_count, 1); else released = refcount_release_if_gt(&object->ref_count, 2); - VM_OBJECT_RUNLOCK(object); if (released) return; @@ -559,7 +569,14 @@ vm_object_deallocate(vm_object_t object) KASSERT(object->ref_count != 0, ("vm_object_deallocate: object deallocated too many times: %d", object->type)); - refcount_release(&object->ref_count); + last = refcount_release(&object->ref_count); + if (object->type == OBJT_VNODE) { + if (last) + vm_object_vndeallocate(object); + else + VM_OBJECT_WUNLOCK(object); + return; + } if (object->ref_count > 1) { VM_OBJECT_WUNLOCK(object); return; @@ -629,7 +646,7 @@ retry: VM_OBJECT_WUNLOCK(object); if (robject->ref_count == 1) { - robject->ref_count--; + refcount_release(&robject->ref_count); object = robject; goto doterm; } @@ -1838,7 +1855,7 @@ vm_object_collapse(vm_object_t object) backing_object)); vm_object_pip_wakeup(backing_object); backing_object->type = OBJT_DEAD; - backing_object->ref_count = 0; + refcount_release(&backing_object->ref_count); VM_OBJECT_WUNLOCK(backing_object); vm_object_destroy(backing_object); Modified: head/sys/vm/vnode_pager.c ============================================================================== --- head/sys/vm/vnode_pager.c Tue Nov 26 22:17:02 2019 (r355121) +++ head/sys/vm/vnode_pager.c Wed Nov 27 00:39:23 2019 (r355122) @@ -150,6 +150,7 @@ vnode_create_vobject(struct vnode *vp, off_t isize, st vm_object_t object; vm_ooffset_t size = isize; struct vattr va; + bool last; if (!vn_isdisk(vp, NULL) && vn_canvmio(vp) == FALSE) return (0); @@ -171,12 +172,15 @@ vnode_create_vobject(struct vnode *vp, off_t isize, st object = vnode_pager_alloc(vp, size, 0, 0, td->td_ucred); /* * Dereference the reference we just created. This assumes - * that the object is associated with the vp. + * that the object is associated with the vp. We still have + * to serialize with vnode_pager_dealloc() for the last + * potential reference. */ VM_OBJECT_RLOCK(object); - refcount_release(&object->ref_count); + last = refcount_release(&object->ref_count); VM_OBJECT_RUNLOCK(object); - vrele(vp); + if (last) + vrele(vp); KASSERT(vp->v_object != NULL, ("vnode_create_vobject: NULL object")); @@ -293,15 +297,17 @@ retry: } vp->v_object = object; VI_UNLOCK(vp); + vrefact(vp); } else { - VM_OBJECT_WLOCK(object); - refcount_acquire(&object->ref_count); + vm_object_reference(object); #if VM_NRESERVLEVEL > 0 - vm_object_color(object, 0); + if ((object->flags & OBJ_COLORED) == 0) { + VM_OBJECT_WLOCK(object); + vm_object_color(object, 0); + VM_OBJECT_WUNLOCK(object); + } #endif - VM_OBJECT_WUNLOCK(object); } - vrefact(vp); return (object); } @@ -345,7 +351,7 @@ vnode_pager_dealloc(vm_object_t object) vp->v_writecount = 0; VI_UNLOCK(vp); VM_OBJECT_WUNLOCK(object); - while (refs-- > 0) + if (refs > 0) vunref(vp); VM_OBJECT_WLOCK(object); } _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
