Author: mav
Date: Tue Mar 12 06:58:49 2013
New Revision: 248186
URL: http://svnweb.freebsd.org/changeset/base/248186
Log:
Make kern_nanosleep() and pause_sbt() to use per-CPU sleep queues.
This removes significant sleep queue lock congestion on multithreaded
microbenchmarks, making them scale to multiple CPUs almost linearly.
Modified:
head/sys/kern/kern_synch.c
head/sys/kern/kern_time.c
head/sys/kern/subr_sleepqueue.c
Modified: head/sys/kern/kern_synch.c
==============================================================================
--- head/sys/kern/kern_synch.c Tue Mar 12 06:14:31 2013 (r248185)
+++ head/sys/kern/kern_synch.c Tue Mar 12 06:58:49 2013 (r248186)
@@ -85,7 +85,7 @@ SYSINIT(synch_setup, SI_SUB_KICK_SCHEDUL
NULL);
int hogticks;
-static int pause_wchan;
+static uint8_t pause_wchan[MAXCPU];
static struct callout loadav_callout;
@@ -198,7 +198,8 @@ _sleep(void *ident, struct lock_object *
if (TD_ON_SLEEPQ(td))
sleepq_remove(td, td->td_wchan);
- if (ident == &pause_wchan)
+ if ((uint8_t *)ident >= &pause_wchan[0] &&
+ (uint8_t *)ident <= &pause_wchan[MAXCPU - 1])
sleepq_flags = SLEEPQ_PAUSE;
else
sleepq_flags = SLEEPQ_SLEEP;
@@ -372,7 +373,7 @@ pause_sbt(const char *wmesg, sbintime_t
DELAY((sbt & 0xffffffff) / SBT_1US);
return (0);
}
- return (_sleep(&pause_wchan, NULL, 0, wmesg, sbt, pr, flags));
+ return (_sleep(&pause_wchan[curcpu], NULL, 0, wmesg, sbt, pr, flags));
}
/*
Modified: head/sys/kern/kern_time.c
==============================================================================
--- head/sys/kern/kern_time.c Tue Mar 12 06:14:31 2013 (r248185)
+++ head/sys/kern/kern_time.c Tue Mar 12 06:58:49 2013 (r248186)
@@ -477,7 +477,7 @@ kern_clock_getres(struct thread *td, clo
return (0);
}
-static int nanowait;
+static uint8_t nanowait[MAXCPU];
int
kern_nanosleep(struct thread *td, struct timespec *rqt, struct timespec *rmt)
@@ -503,8 +503,8 @@ kern_nanosleep(struct thread *td, struct
if (TIMESEL(&sbt, tmp))
sbt += tc_tick_sbt;
sbt += tmp;
- error = tsleep_sbt(&nanowait, PWAIT | PCATCH, "nanslp", sbt, prec,
- C_ABSOLUTE);
+ error = tsleep_sbt(&nanowait[curcpu], PWAIT | PCATCH, "nanslp",
+ sbt, prec, C_ABSOLUTE);
if (error != EWOULDBLOCK) {
if (error == ERESTART)
error = EINTR;
Modified: head/sys/kern/subr_sleepqueue.c
==============================================================================
--- head/sys/kern/subr_sleepqueue.c Tue Mar 12 06:14:31 2013
(r248185)
+++ head/sys/kern/subr_sleepqueue.c Tue Mar 12 06:58:49 2013
(r248186)
@@ -88,16 +88,14 @@ __FBSDID("$FreeBSD$");
#endif
/*
- * Constants for the hash table of sleep queue chains. These constants are
- * the same ones that 4BSD (and possibly earlier versions of BSD) used.
- * Basically, we ignore the lower 8 bits of the address since most wait
- * channel pointers are aligned and only look at the next 7 bits for the
- * hash. SC_TABLESIZE must be a power of two for SC_MASK to work properly.
+ * Constants for the hash table of sleep queue chains.
+ * SC_TABLESIZE must be a power of two for SC_MASK to work properly.
*/
-#define SC_TABLESIZE 128 /* Must be power of 2.
*/
+#define SC_TABLESIZE 256 /* Must be power of 2.
*/
#define SC_MASK (SC_TABLESIZE - 1)
#define SC_SHIFT 8
-#define SC_HASH(wc) (((uintptr_t)(wc) >> SC_SHIFT) & SC_MASK)
+#define SC_HASH(wc) ((((uintptr_t)(wc) >> SC_SHIFT) ^
(uintptr_t)(wc)) & \
+ SC_MASK)
#define SC_LOOKUP(wc) &sleepq_chains[SC_HASH(wc)]
#define NR_SLEEPQS 2
/*
_______________________________________________
svn-src-head@freebsd.org mailing list
http://lists.freebsd.org/mailman/listinfo/svn-src-head
To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
