Author: jeff Date: Tue Feb 4 20:33:01 2020 New Revision: 357528 URL: https://svnweb.freebsd.org/changeset/base/357528
Log: Add an explicit busy state for free pages. This improves behavior with potential bugs that access freed pages as well as providing a path towards lockless page lookup. Reviewed by: kib Differential Revision: https://reviews.freebsd.org/D23444 Modified: head/sys/vm/vm_page.c head/sys/vm/vm_page.h Modified: head/sys/vm/vm_page.c ============================================================================== --- head/sys/vm/vm_page.c Tue Feb 4 20:28:06 2020 (r357527) +++ head/sys/vm/vm_page.c Tue Feb 4 20:33:01 2020 (r357528) @@ -508,7 +508,7 @@ vm_page_init_page(vm_page_t m, vm_paddr_t pa, int segi m->object = NULL; m->ref_count = 0; - m->busy_lock = VPB_UNBUSIED; + m->busy_lock = VPB_FREED; m->flags = m->a.flags = 0; m->phys_addr = pa; m->a.queue = PQ_NONE; @@ -988,6 +988,8 @@ vm_page_sunbusy(vm_page_t m) x = m->busy_lock; for (;;) { + KASSERT(x != VPB_FREED, + ("vm_page_sunbusy: Unlocking freed page.")); if (VPB_SHARERS(x) > 1) { if (atomic_fcmpset_int(&m->busy_lock, &x, x - VPB_ONE_SHARER)) @@ -1155,6 +1157,17 @@ vm_page_xunbusy_hard_unchecked(vm_page_t m) vm_page_xunbusy_hard_tail(m); } +static void +vm_page_busy_free(vm_page_t m) +{ + u_int x; + + atomic_thread_fence_rel(); + x = atomic_swap_int(&m->busy_lock, VPB_FREED); + if ((x & VPB_BIT_WAITERS) != 0) + wakeup(m); +} + /* * vm_page_unhold_pages: * @@ -1249,7 +1262,8 @@ vm_page_putfake(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) != 0, ("managed %p", m)); KASSERT((m->flags & PG_FICTITIOUS) != 0, ("vm_page_putfake: bad page %p", m)); - vm_page_xunbusy(m); + vm_page_assert_xbusied(m); + vm_page_busy_free(m); uma_zfree(fakepg_zone, m); } @@ -2012,11 +2026,12 @@ found: m->a.flags = 0; m->oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0; - m->busy_lock = VPB_UNBUSIED; if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) m->busy_lock = VPB_CURTHREAD_EXCLUSIVE; - if ((req & VM_ALLOC_SBUSY) != 0) + else if ((req & VM_ALLOC_SBUSY) != 0) m->busy_lock = VPB_SHARERS_WORD(1); + else + m->busy_lock = VPB_UNBUSIED; if (req & VM_ALLOC_WIRED) { vm_wire_add(1); m->ref_count = 1; @@ -2202,11 +2217,12 @@ found: flags |= PG_NODUMP; oflags = object == NULL || (object->flags & OBJ_UNMANAGED) != 0 ? VPO_UNMANAGED : 0; - busy_lock = VPB_UNBUSIED; if ((req & (VM_ALLOC_NOBUSY | VM_ALLOC_NOOBJ | VM_ALLOC_SBUSY)) == 0) busy_lock = VPB_CURTHREAD_EXCLUSIVE; - if ((req & VM_ALLOC_SBUSY) != 0) + else if ((req & VM_ALLOC_SBUSY) != 0) busy_lock = VPB_SHARERS_WORD(1); + else + busy_lock = VPB_UNBUSIED; if ((req & VM_ALLOC_WIRED) != 0) vm_wire_add(npages); if (object != NULL) { @@ -2268,7 +2284,7 @@ vm_page_alloc_check(vm_page_t m) ("page %p has unexpected queue %d, flags %#x", m, m->a.queue, (m->a.flags & PGA_QUEUE_STATE_MASK))); KASSERT(m->ref_count == 0, ("page %p has references", m)); - KASSERT(!vm_page_busied(m), ("page %p is busy", m)); + KASSERT(vm_page_busy_freed(m), ("page %p is not freed", m)); KASSERT(m->dirty == 0, ("page %p is dirty", m)); KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, ("page %p has unexpected memattr %d", @@ -3592,9 +3608,6 @@ vm_page_free_prep(vm_page_t m) */ atomic_thread_fence_acq(); - if (vm_page_sbusied(m)) - panic("vm_page_free_prep: freeing shared busy page %p", m); - #if defined(DIAGNOSTIC) && defined(PHYS_TO_DMAP) if (PMAP_HAS_DMAP && (m->flags & PG_ZERO) != 0) { uint64_t *p; @@ -3621,7 +3634,7 @@ vm_page_free_prep(vm_page_t m) ((m->object->flags & OBJ_UNMANAGED) != 0), ("vm_page_free_prep: managed flag mismatch for page %p", m)); - vm_page_object_remove(m); + vm_page_assert_xbusied(m); /* * The object reference can be released without an atomic @@ -3631,13 +3644,13 @@ vm_page_free_prep(vm_page_t m) m->ref_count == VPRC_OBJREF, ("vm_page_free_prep: page %p has unexpected ref_count %u", m, m->ref_count)); + vm_page_object_remove(m); m->object = NULL; m->ref_count -= VPRC_OBJREF; - vm_page_xunbusy(m); - } + } else + vm_page_assert_unbusied(m); - if (vm_page_xbusied(m)) - panic("vm_page_free_prep: freeing exclusive busy page %p", m); + vm_page_busy_free(m); /* * If fictitious remove object association and Modified: head/sys/vm/vm_page.h ============================================================================== --- head/sys/vm/vm_page.h Tue Feb 4 20:28:06 2020 (r357527) +++ head/sys/vm/vm_page.h Tue Feb 4 20:33:01 2020 (r357528) @@ -325,6 +325,9 @@ struct vm_page { #define VPB_UNBUSIED VPB_SHARERS_WORD(0) +/* Freed lock blocks both shared and exclusive. */ +#define VPB_FREED (0xffffffff - VPB_BIT_SHARED) + #define PQ_NONE 255 #define PQ_INACTIVE 0 #define PQ_ACTIVE 1 @@ -700,9 +703,11 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const (m), __FILE__, __LINE__)) #define vm_page_assert_unbusied(m) \ - KASSERT(!vm_page_busied(m), \ - ("vm_page_assert_unbusied: page %p busy @ %s:%d", \ - (m), __FILE__, __LINE__)) + KASSERT((m->busy_lock & ~VPB_BIT_WAITERS) != \ + VPB_CURTHREAD_EXCLUSIVE, \ + ("vm_page_assert_xbusied: page %p busy_lock %#x owned" \ + " by me @ %s:%d", \ + (m), (m)->busy_lock, __FILE__, __LINE__)); \ #define vm_page_assert_xbusied_unchecked(m) do { \ KASSERT(vm_page_xbusied(m), \ @@ -729,6 +734,9 @@ void vm_page_lock_assert_KBI(vm_page_t m, int a, const #define vm_page_xbusied(m) \ (((m)->busy_lock & VPB_SINGLE_EXCLUSIVE) != 0) + +#define vm_page_busy_freed(m) \ + ((m)->busy_lock == VPB_FREED) #define vm_page_xbusy(m) do { \ if (!vm_page_tryxbusy(m)) \ _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
