Author: markj Date: Sun Apr 26 20:08:57 2020 New Revision: 360354 URL: https://svnweb.freebsd.org/changeset/base/360354
Log: Use a single VM object for kernel stacks. Previously we allocated a separate VM object for each kernel stack. However, fully constructed kernel stacks are cached by UMA, so there is no harm in using a single global object for all stacks. This reduces memory consumption and makes it easier to define a memory allocation policy for kernel stack pages, with the aim of reducing physical memory fragmentation. Add a global kstack_object, and use the stack KVA address to index into the object like we do with kernel_object. Reviewed by: kib Tested by: pho Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D24473 Modified: head/sys/kern/kern_thread.c head/sys/sys/proc.h head/sys/vm/vm_extern.h head/sys/vm/vm_glue.c head/sys/vm/vm_kern.h head/sys/vm/vm_swapout.c Modified: head/sys/kern/kern_thread.c ============================================================================== --- head/sys/kern/kern_thread.c Sun Apr 26 20:06:08 2020 (r360353) +++ head/sys/kern/kern_thread.c Sun Apr 26 20:08:57 2020 (r360354) @@ -84,7 +84,7 @@ _Static_assert(offsetof(struct thread, td_pflags) == 0 "struct thread KBI td_pflags"); _Static_assert(offsetof(struct thread, td_frame) == 0x498, "struct thread KBI td_frame"); -_Static_assert(offsetof(struct thread, td_emuldata) == 0x6b0, +_Static_assert(offsetof(struct thread, td_emuldata) == 0x6a0, "struct thread KBI td_emuldata"); _Static_assert(offsetof(struct proc, p_flag) == 0xb0, "struct proc KBI p_flag"); Modified: head/sys/sys/proc.h ============================================================================== --- head/sys/sys/proc.h Sun Apr 26 20:06:08 2020 (r360353) +++ head/sys/sys/proc.h Sun Apr 26 20:08:57 2020 (r360354) @@ -348,7 +348,6 @@ struct thread { /* LP64 hole */ struct callout td_slpcallout; /* (h) Callout for sleep. */ struct trapframe *td_frame; /* (k) */ - struct vm_object *td_kstack_obj;/* (a) Kstack object. */ vm_offset_t td_kstack; /* (a) Kernel VA of kstack. */ int td_kstack_pages; /* (a) Size of the kstack. */ volatile u_int td_critnest; /* (k*) Critical section nest level. */ Modified: head/sys/vm/vm_extern.h ============================================================================== --- head/sys/vm/vm_extern.h Sun Apr 26 20:06:08 2020 (r360353) +++ head/sys/vm/vm_extern.h Sun Apr 26 20:08:57 2020 (r360354) @@ -126,6 +126,8 @@ struct sf_buf *vm_imgact_map_page(vm_object_t object, void vm_imgact_unmap_page(struct sf_buf *sf); void vm_thread_dispose(struct thread *td); int vm_thread_new(struct thread *td, int pages); +void vm_thread_stack_back(struct domainset *ds, vm_offset_t kaddr, + vm_page_t ma[], int npages, int req_class); u_int vm_active_count(void); u_int vm_inactive_count(void); u_int vm_laundry_count(void); Modified: head/sys/vm/vm_glue.c ============================================================================== --- head/sys/vm/vm_glue.c Sun Apr 26 20:06:08 2020 (r360353) +++ head/sys/vm/vm_glue.c Sun Apr 26 20:08:57 2020 (r360354) @@ -264,9 +264,9 @@ vm_sync_icache(vm_map_t map, vm_offset_t va, vm_offset pmap_sync_icache(map->pmap, va, sz); } +vm_object_t kstack_object; static uma_zone_t kstack_cache; static int kstack_cache_size; -static int kstack_domain_iter; static int sysctl_kstack_cache_size(SYSCTL_HANDLER_ARGS) @@ -285,23 +285,15 @@ SYSCTL_PROC(_vm, OID_AUTO, kstack_cache_size, /* * Create the kernel stack (including pcb for i386) for a new thread. - * This routine directly affects the fork perf for a process and - * create performance for a thread. */ static vm_offset_t -vm_thread_stack_create(struct domainset *ds, vm_object_t *ksobjp, int pages) +vm_thread_stack_create(struct domainset *ds, int pages) { vm_page_t ma[KSTACK_MAX_PAGES]; - vm_object_t ksobj; vm_offset_t ks; int i; /* - * Allocate an object for the kstack. - */ - ksobj = vm_object_allocate(OBJT_DEFAULT, pages); - - /* * Get a kernel virtual address for this thread's kstack. */ #if defined(__mips__) @@ -319,54 +311,45 @@ vm_thread_stack_create(struct domainset *ds, vm_object #endif if (ks == 0) { printf("%s: kstack allocation failed\n", __func__); - vm_object_deallocate(ksobj); return (0); } - if (vm_ndomains > 1) { - ksobj->domain.dr_policy = ds; - ksobj->domain.dr_iter = - atomic_fetchadd_int(&kstack_domain_iter, 1); - } if (KSTACK_GUARD_PAGES != 0) { pmap_qremove(ks, KSTACK_GUARD_PAGES); ks += KSTACK_GUARD_PAGES * PAGE_SIZE; } - /* - * For the length of the stack, link in a real page of ram for each - * page of stack. + /* + * Allocate physical pages to back the stack. */ - VM_OBJECT_WLOCK(ksobj); - (void)vm_page_grab_pages(ksobj, 0, VM_ALLOC_NORMAL | VM_ALLOC_WIRED, - ma, pages); + vm_thread_stack_back(ds, ks, ma, pages, VM_ALLOC_NORMAL); for (i = 0; i < pages; i++) vm_page_valid(ma[i]); - VM_OBJECT_WUNLOCK(ksobj); pmap_qenter(ks, ma, pages); - *ksobjp = ksobj; return (ks); } static void -vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages) +vm_thread_stack_dispose(vm_offset_t ks, int pages) { vm_page_t m; + vm_pindex_t pindex; int i; + pindex = atop(ks - VM_MIN_KERNEL_ADDRESS); + pmap_qremove(ks, pages); - VM_OBJECT_WLOCK(ksobj); + VM_OBJECT_WLOCK(kstack_object); for (i = 0; i < pages; i++) { - m = vm_page_lookup(ksobj, i); + m = vm_page_lookup(kstack_object, pindex + i); if (m == NULL) panic("%s: kstack already missing?", __func__); vm_page_xbusy_claim(m); vm_page_unwire_noq(m); vm_page_free(m); } - VM_OBJECT_WUNLOCK(ksobj); - vm_object_deallocate(ksobj); + VM_OBJECT_WUNLOCK(kstack_object); kva_free(ks - (KSTACK_GUARD_PAGES * PAGE_SIZE), (pages + KSTACK_GUARD_PAGES) * PAGE_SIZE); } @@ -377,7 +360,6 @@ vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t int vm_thread_new(struct thread *td, int pages) { - vm_object_t ksobj; vm_offset_t ks; /* Bounds check */ @@ -387,12 +369,8 @@ vm_thread_new(struct thread *td, int pages) pages = KSTACK_MAX_PAGES; ks = 0; - ksobj = NULL; - if (pages == kstack_pages && kstack_cache != NULL) { + if (pages == kstack_pages && kstack_cache != NULL) ks = (vm_offset_t)uma_zalloc(kstack_cache, M_NOWAIT); - if (ks != 0) - ksobj = PHYS_TO_VM_PAGE(pmap_kextract(ks))->object; - } /* * Ensure that kstack objects can draw pages from any memory @@ -401,10 +379,9 @@ vm_thread_new(struct thread *td, int pages) */ if (ks == 0) ks = vm_thread_stack_create(DOMAINSET_PREF(PCPU_GET(domain)), - &ksobj, pages); + pages); if (ks == 0) return (0); - td->td_kstack_obj = ksobj; td->td_kstack = ks; td->td_kstack_pages = pages; return (1); @@ -416,26 +393,52 @@ vm_thread_new(struct thread *td, int pages) void vm_thread_dispose(struct thread *td) { - vm_object_t ksobj; vm_offset_t ks; int pages; pages = td->td_kstack_pages; - ksobj = td->td_kstack_obj; ks = td->td_kstack; td->td_kstack = 0; td->td_kstack_pages = 0; if (pages == kstack_pages) uma_zfree(kstack_cache, (void *)ks); else - vm_thread_stack_dispose(ksobj, ks, pages); + vm_thread_stack_dispose(ks, pages); } +/* + * Allocate physical pages, following the specified NUMA policy, to back a + * kernel stack. + */ +void +vm_thread_stack_back(struct domainset *ds, vm_offset_t ks, vm_page_t ma[], + int npages, int req_class) +{ + vm_pindex_t pindex; + int n; + + pindex = atop(ks - VM_MIN_KERNEL_ADDRESS); + + VM_OBJECT_WLOCK(kstack_object); + for (n = 0; n < npages;) { + if (vm_ndomains > 1) + kstack_object->domain.dr_policy = ds; + + /* + * Use WAITFAIL to force a reset of the domain selection policy + * if we had to sleep for pages. + */ + n += vm_page_grab_pages(kstack_object, pindex + n, + req_class | VM_ALLOC_WIRED | VM_ALLOC_WAITFAIL, + &ma[n], npages - n); + } + VM_OBJECT_WUNLOCK(kstack_object); +} + static int kstack_import(void *arg, void **store, int cnt, int domain, int flags) { struct domainset *ds; - vm_object_t ksobj; int i; if (domain == UMA_ANYDOMAIN) @@ -444,8 +447,7 @@ kstack_import(void *arg, void **store, int cnt, int do ds = DOMAINSET_PREF(domain); for (i = 0; i < cnt; i++) { - store[i] = (void *)vm_thread_stack_create(ds, &ksobj, - kstack_pages); + store[i] = (void *)vm_thread_stack_create(ds, kstack_pages); if (store[i] == NULL) break; } @@ -460,15 +462,15 @@ kstack_release(void *arg, void **store, int cnt) for (i = 0; i < cnt; i++) { ks = (vm_offset_t)store[i]; - vm_thread_stack_dispose( - PHYS_TO_VM_PAGE(pmap_kextract(ks))->object, - ks, kstack_pages); + vm_thread_stack_dispose(ks, kstack_pages); } } static void kstack_cache_init(void *null) { + kstack_object = vm_object_allocate(OBJT_SWAP, + atop(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS)); kstack_cache = uma_zcache_create("kstack_cache", kstack_pages * PAGE_SIZE, NULL, NULL, NULL, NULL, kstack_import, kstack_release, NULL, @@ -476,8 +478,7 @@ kstack_cache_init(void *null) kstack_cache_size = imax(128, mp_ncpus * 4); uma_zone_set_maxcache(kstack_cache, kstack_cache_size); } - -SYSINIT(vm_kstacks, SI_SUB_KTHREAD_INIT, SI_ORDER_ANY, kstack_cache_init, NULL); +SYSINIT(vm_kstacks, SI_SUB_KMEM, SI_ORDER_ANY, kstack_cache_init, NULL); #ifdef KSTACK_USAGE_PROF /* Modified: head/sys/vm/vm_kern.h ============================================================================== --- head/sys/vm/vm_kern.h Sun Apr 26 20:06:08 2020 (r360353) +++ head/sys/vm/vm_kern.h Sun Apr 26 20:08:57 2020 (r360354) @@ -77,5 +77,6 @@ extern struct vmem *memguard_arena; extern u_long vm_kmem_size; extern u_int exec_map_entries; extern u_int exec_map_entry_size; +extern vm_object_t kstack_object; #endif /* _VM_VM_KERN_H_ */ Modified: head/sys/vm/vm_swapout.c ============================================================================== --- head/sys/vm/vm_swapout.c Sun Apr 26 20:06:08 2020 (r360353) +++ head/sys/vm/vm_swapout.c Sun Apr 26 20:08:57 2020 (r360354) @@ -104,6 +104,7 @@ __FBSDID("$FreeBSD$"); #include <vm/vm.h> #include <vm/vm_param.h> +#include <vm/vm_kern.h> #include <vm/vm_object.h> #include <vm/vm_page.h> #include <vm/vm_map.h> @@ -526,24 +527,26 @@ again: static void vm_thread_swapout(struct thread *td) { - vm_object_t ksobj; vm_page_t m; + vm_offset_t kaddr; + vm_pindex_t pindex; int i, pages; cpu_thread_swapout(td); + kaddr = td->td_kstack; pages = td->td_kstack_pages; - ksobj = td->td_kstack_obj; - pmap_qremove(td->td_kstack, pages); - VM_OBJECT_WLOCK(ksobj); + pindex = atop(kaddr - VM_MIN_KERNEL_ADDRESS); + pmap_qremove(kaddr, pages); + VM_OBJECT_WLOCK(kstack_object); for (i = 0; i < pages; i++) { - m = vm_page_lookup(ksobj, i); + m = vm_page_lookup(kstack_object, pindex + i); if (m == NULL) panic("vm_thread_swapout: kstack already missing?"); vm_page_dirty(m); vm_page_xunbusy_unchecked(m); vm_page_unwire(m, PQ_LAUNDRY); } - VM_OBJECT_WUNLOCK(ksobj); + VM_OBJECT_WUNLOCK(kstack_object); } /* @@ -552,38 +555,36 @@ vm_thread_swapout(struct thread *td) static void vm_thread_swapin(struct thread *td, int oom_alloc) { - vm_object_t ksobj; vm_page_t ma[KSTACK_MAX_PAGES]; + vm_offset_t kaddr; int a, count, i, j, pages, rv; + kaddr = td->td_kstack; pages = td->td_kstack_pages; - ksobj = td->td_kstack_obj; - VM_OBJECT_WLOCK(ksobj); - (void)vm_page_grab_pages(ksobj, 0, oom_alloc | VM_ALLOC_WIRED, ma, - pages); - VM_OBJECT_WUNLOCK(ksobj); + vm_thread_stack_back(td->td_domain.dr_policy, kaddr, ma, pages, + oom_alloc); for (i = 0; i < pages;) { vm_page_assert_xbusied(ma[i]); if (vm_page_all_valid(ma[i])) { i++; continue; } - vm_object_pip_add(ksobj, 1); + vm_object_pip_add(kstack_object, 1); for (j = i + 1; j < pages; j++) if (vm_page_all_valid(ma[j])) break; - VM_OBJECT_WLOCK(ksobj); - rv = vm_pager_has_page(ksobj, ma[i]->pindex, NULL, &a); - VM_OBJECT_WUNLOCK(ksobj); + VM_OBJECT_WLOCK(kstack_object); + rv = vm_pager_has_page(kstack_object, ma[i]->pindex, NULL, &a); + VM_OBJECT_WUNLOCK(kstack_object); KASSERT(rv == 1, ("%s: missing page %p", __func__, ma[i])); count = min(a + 1, j - i); - rv = vm_pager_get_pages(ksobj, ma + i, count, NULL, NULL); + rv = vm_pager_get_pages(kstack_object, ma + i, count, NULL, NULL); KASSERT(rv == VM_PAGER_OK, ("%s: cannot get kstack for proc %d", __func__, td->td_proc->p_pid)); - vm_object_pip_wakeup(ksobj); + vm_object_pip_wakeup(kstack_object); i += count; } - pmap_qenter(td->td_kstack, ma, pages); + pmap_qenter(kaddr, ma, pages); cpu_thread_swapin(td); } _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
