Author: mjg
Date: Wed Nov 11 18:45:06 2020
New Revision: 367598
URL: https://svnweb.freebsd.org/changeset/base/367598
Log:
thread: batch tid_free calls in thread_reap
This eliminates the highly pessimal pattern of relocking from multiple
CPUs in quick succession. Note this is still globally serialized.
Modified:
head/sys/kern/kern_thread.c
Modified: head/sys/kern/kern_thread.c
==============================================================================
--- head/sys/kern/kern_thread.c Wed Nov 11 18:43:51 2020 (r367597)
+++ head/sys/kern/kern_thread.c Wed Nov 11 18:45:06 2020 (r367598)
@@ -133,6 +133,7 @@ static __exclusive_cache_line struct thread *thread_zo
static void thread_zombie(struct thread *);
static int thread_unsuspend_one(struct thread *td, struct proc *p,
bool boundary);
+static void thread_free_batched(struct thread *td);
static struct mtx tid_lock;
static bitstr_t *tid_bitmap;
@@ -200,21 +201,41 @@ tid_alloc(void)
}
static void
-tid_free(lwpid_t rtid)
+tid_free_locked(lwpid_t rtid)
{
lwpid_t tid;
+ mtx_assert(&tid_lock, MA_OWNED);
KASSERT(rtid >= NO_PID,
("%s: invalid tid %d\n", __func__, rtid));
tid = rtid - NO_PID;
- mtx_lock(&tid_lock);
KASSERT(bit_test(tid_bitmap, tid) != 0,
("thread ID %d not allocated\n", rtid));
bit_clear(tid_bitmap, tid);
nthreads--;
+}
+
+static void
+tid_free(lwpid_t rtid)
+{
+
+ mtx_lock(&tid_lock);
+ tid_free_locked(rtid);
mtx_unlock(&tid_lock);
}
+static void
+tid_free_batch(lwpid_t *batch, int n)
+{
+ int i;
+
+ mtx_lock(&tid_lock);
+ for (i = 0; i < n; i++) {
+ tid_free_locked(batch[i]);
+ }
+ mtx_unlock(&tid_lock);
+}
+
/*
* Prepare a thread for use.
*/
@@ -440,6 +461,8 @@ void
thread_reap(void)
{
struct thread *itd, *ntd;
+ lwpid_t tidbatch[16];
+ int tidbatchn;
/*
* Reading upfront is pessimal if followed by concurrent atomic_swap,
@@ -450,12 +473,23 @@ thread_reap(void)
itd = (struct thread *)atomic_swap_ptr((uintptr_t *)&thread_zombies,
(uintptr_t)NULL);
+ tidbatchn = 0;
while (itd != NULL) {
ntd = itd->td_zombie;
+ tidbatch[tidbatchn] = itd->td_tid;
+ tidbatchn++;
thread_cow_free(itd);
- thread_free(itd);
+ thread_free_batched(itd);
+ if (tidbatchn == nitems(tidbatch)) {
+ tid_free_batch(tidbatch, tidbatchn);
+ tidbatchn = 0;
+ }
itd = ntd;
}
+
+ if (tidbatchn != 0) {
+ tid_free_batch(tidbatch, tidbatchn);
+ }
}
/*
@@ -502,8 +536,8 @@ thread_alloc_stack(struct thread *td, int pages)
/*
* Deallocate a thread.
*/
-void
-thread_free(struct thread *td)
+static void
+thread_free_batched(struct thread *td)
{
EVENTHANDLER_DIRECT_INVOKE(thread_dtor, td);
@@ -515,9 +549,21 @@ thread_free(struct thread *td)
if (td->td_kstack != 0)
vm_thread_dispose(td);
callout_drain(&td->td_slpcallout);
- tid_free(td->td_tid);
+ /*
+ * Freeing handled by the caller.
+ */
td->td_tid = -1;
uma_zfree(thread_zone, td);
+}
+
+void
+thread_free(struct thread *td)
+{
+ lwpid_t tid;
+
+ tid = td->td_tid;
+ thread_free_batched(td);
+ tid_free(tid);
}
void
_______________________________________________
svn-src-head@freebsd.org mailing list
https://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
