Module Name: src
Committed By: ad
Date: Sun Oct 8 13:23:05 UTC 2023
Modified Files:
src/sys/kern: kern_condvar.c kern_sleepq.c kern_timeout.c
kern_turnstile.c sys_lwp.c sys_select.c
src/sys/rump/librump/rumpkern: sleepq.c
src/sys/sys: sleepq.h syncobj.h
Log Message:
Ensure that an LWP that has taken a legitimate wakeup never produces an
error code from sleepq_block(). Then, it's possible to make cv_signal()
work as expected and only ever wake a singular LWP.
To generate a diff of this commit:
cvs rdiff -u -r1.57 -r1.58 src/sys/kern/kern_condvar.c
cvs rdiff -u -r1.81 -r1.82 src/sys/kern/kern_sleepq.c
cvs rdiff -u -r1.78 -r1.79 src/sys/kern/kern_timeout.c
cvs rdiff -u -r1.52 -r1.53 src/sys/kern/kern_turnstile.c
cvs rdiff -u -r1.86 -r1.87 src/sys/kern/sys_lwp.c
cvs rdiff -u -r1.63 -r1.64 src/sys/kern/sys_select.c
cvs rdiff -u -r1.26 -r1.27 src/sys/rump/librump/rumpkern/sleepq.c
cvs rdiff -u -r1.39 -r1.40 src/sys/sys/sleepq.h
cvs rdiff -u -r1.16 -r1.17 src/sys/sys/syncobj.h
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/kern/kern_condvar.c
diff -u src/sys/kern/kern_condvar.c:1.57 src/sys/kern/kern_condvar.c:1.58
--- src/sys/kern/kern_condvar.c:1.57 Wed Oct 4 20:29:18 2023
+++ src/sys/kern/kern_condvar.c Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_condvar.c,v 1.57 2023/10/04 20:29:18 ad Exp $ */
+/* $NetBSD: kern_condvar.c,v 1.58 2023/10/08 13:23:05 ad Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008, 2019, 2020, 2023
@@ -35,7 +35,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_condvar.c,v 1.57 2023/10/04 20:29:18 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_condvar.c,v 1.58 2023/10/08 13:23:05 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -478,27 +478,13 @@ cv_wakeup_one(kcondvar_t *cv)
kmutex_t *mp;
lwp_t *l;
- /*
- * Keep waking LWPs until a non-interruptable waiter is found. An
- * interruptable waiter could fail to do something useful with the
- * wakeup due to an error return from cv_[timed]wait_sig(), and the
- * caller of cv_signal() may not expect such a scenario.
- *
- * This isn't a problem for non-interruptable waits (untimed and
- * timed), because if such a waiter is woken here it will not return
- * an error.
- */
mp = sleepq_hashlock(cv);
sq = CV_SLEEPQ(cv);
- while ((l = LIST_FIRST(sq)) != NULL) {
+ if (__predict_true((l = LIST_FIRST(sq)) != NULL)) {
KASSERT(l->l_sleepq == sq);
KASSERT(l->l_mutex == mp);
KASSERT(l->l_wchan == cv);
- if ((l->l_flag & LW_SINTR) == 0) {
- sleepq_remove(sq, l);
- break;
- } else
- sleepq_remove(sq, l);
+ sleepq_remove(sq, l, true);
}
mutex_spin_exit(mp);
}
@@ -539,7 +525,7 @@ cv_wakeup_all(kcondvar_t *cv)
KASSERT(l->l_sleepq == sq);
KASSERT(l->l_mutex == mp);
KASSERT(l->l_wchan == cv);
- sleepq_remove(sq, l);
+ sleepq_remove(sq, l, true);
}
mutex_spin_exit(mp);
}
Index: src/sys/kern/kern_sleepq.c
diff -u src/sys/kern/kern_sleepq.c:1.81 src/sys/kern/kern_sleepq.c:1.82
--- src/sys/kern/kern_sleepq.c:1.81 Sun Oct 8 11:12:47 2023
+++ src/sys/kern/kern_sleepq.c Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_sleepq.c,v 1.81 2023/10/08 11:12:47 ad Exp $ */
+/* $NetBSD: kern_sleepq.c,v 1.82 2023/10/08 13:23:05 ad Exp $ */
/*-
* Copyright (c) 2006, 2007, 2008, 2009, 2019, 2020, 2023
@@ -36,7 +36,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.81 2023/10/08 11:12:47 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_sleepq.c,v 1.82 2023/10/08 13:23:05 ad Exp $");
#include <sys/param.h>
#include <sys/kernel.h>
@@ -105,10 +105,17 @@ sleepq_init(sleepq_t *sq)
/*
* sleepq_remove:
*
- * Remove an LWP from a sleep queue and wake it up.
+ * Remove an LWP from a sleep queue and wake it up. Distinguish
+ * between deliberate wakeups (which are a valuable information) and
+ * "unsleep" (an out-of-band action must be taken).
+ *
+ * For wakeup, convert any interruptable wait into non-interruptable
+ * one before waking the LWP. Otherwise, if only one LWP is awoken it
+ * could fail to do something useful with the wakeup due to an error
+ * return and the caller of e.g. cv_signal() may not expect this.
*/
void
-sleepq_remove(sleepq_t *sq, lwp_t *l)
+sleepq_remove(sleepq_t *sq, lwp_t *l, bool wakeup)
{
struct schedstate_percpu *spc;
struct cpu_info *ci;
@@ -125,7 +132,7 @@ sleepq_remove(sleepq_t *sq, lwp_t *l)
l->l_syncobj = &sched_syncobj;
l->l_wchan = NULL;
l->l_sleepq = NULL;
- l->l_flag &= ~LW_SINTR;
+ l->l_flag &= wakeup ? ~(LW_SINTR|LW_CATCHINTR|LW_STIMO) : ~LW_SINTR;
ci = l->l_cpu;
spc = &ci->ci_schedstate;
@@ -409,7 +416,6 @@ sleepq_block(int timo, bool catch_p, syn
*/
flag = atomic_load_relaxed(&l->l_flag);
if (__predict_false((flag & mask) != 0)) {
- p = l->l_proc;
if ((flag & LW_CATCHINTR) == 0 && error != 0)
/* nothing */;
else if ((flag & (LW_CANCELLED | LW_WEXIT | LW_WCORE)) != 0)
@@ -422,6 +428,7 @@ sleepq_block(int timo, bool catch_p, syn
* on locks are non-interruptable and we will
* not recurse again.
*/
+ p = l->l_proc;
mutex_enter(p->p_lock);
if (((sig = sigispending(l, 0)) != 0 &&
(sigprop[sig] & SA_STOP) == 0) ||
@@ -456,7 +463,7 @@ sleepq_wake(sleepq_t *sq, wchan_t wchan,
next = LIST_NEXT(l, l_sleepchain);
if (l->l_wchan != wchan)
continue;
- sleepq_remove(sq, l);
+ sleepq_remove(sq, l, true);
if (--expected == 0)
break;
}
@@ -480,7 +487,7 @@ sleepq_unsleep(lwp_t *l, bool unlock)
KASSERT(lwp_locked(l, mp));
KASSERT(l->l_wchan != NULL);
- sleepq_remove(sq, l);
+ sleepq_remove(sq, l, false);
if (unlock) {
mutex_spin_exit(mp);
}
@@ -503,8 +510,12 @@ sleepq_timeout(void *arg)
*/
lwp_lock(l);
- if (l->l_wchan == NULL) {
- /* Somebody beat us to it. */
+ if (l->l_wchan == NULL || l->l_syncobj == &callout_syncobj) {
+ /*
+ * Somebody beat us to it, or the LWP is blocked in
+ * callout_halt() waiting for us to finish here. In
+ * neither case should the LWP produce EWOULDBLOCK.
+ */
lwp_unlock(l);
return;
}
Index: src/sys/kern/kern_timeout.c
diff -u src/sys/kern/kern_timeout.c:1.78 src/sys/kern/kern_timeout.c:1.79
--- src/sys/kern/kern_timeout.c:1.78 Wed Oct 4 20:29:18 2023
+++ src/sys/kern/kern_timeout.c Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_timeout.c,v 1.78 2023/10/04 20:29:18 ad Exp $ */
+/* $NetBSD: kern_timeout.c,v 1.79 2023/10/08 13:23:05 ad Exp $ */
/*-
* Copyright (c) 2003, 2006, 2007, 2008, 2009, 2019, 2023
@@ -60,7 +60,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.78 2023/10/04 20:29:18 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_timeout.c,v 1.79 2023/10/08 13:23:05 ad Exp $");
/*
* Timeouts are kept in a hierarchical timing wheel. The c_time is the
@@ -246,6 +246,16 @@ SDT_PROBE_DEFINE5(sdt, kernel, callout,
"unsigned"/*flags*/,
"bool"/*expired*/);
+syncobj_t callout_syncobj = {
+ .sobj_name = "callout",
+ .sobj_flag = SOBJ_SLEEPQ_SORTED,
+ .sobj_boostpri = PRI_KERNEL,
+ .sobj_unsleep = sleepq_unsleep,
+ .sobj_changepri = sleepq_changepri,
+ .sobj_lendpri = sleepq_lendpri,
+ .sobj_owner = syncobj_noowner,
+};
+
static inline kmutex_t *
callout_lock(callout_impl_t *c)
{
@@ -612,8 +622,8 @@ callout_wait(callout_impl_t *c, void *in
cc->cc_ev_block.ev_count++;
nlocks = sleepq_enter(&cc->cc_sleepq, l, cc->cc_lock);
sleepq_enqueue(&cc->cc_sleepq, cc, "callout",
- &sleep_syncobj, false);
- sleepq_block(0, false, &sleep_syncobj, nlocks);
+ &callout_syncobj, false);
+ sleepq_block(0, false, &callout_syncobj, nlocks);
}
/*
Index: src/sys/kern/kern_turnstile.c
diff -u src/sys/kern/kern_turnstile.c:1.52 src/sys/kern/kern_turnstile.c:1.53
--- src/sys/kern/kern_turnstile.c:1.52 Wed Oct 4 20:39:35 2023
+++ src/sys/kern/kern_turnstile.c Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: kern_turnstile.c,v 1.52 2023/10/04 20:39:35 ad Exp $ */
+/* $NetBSD: kern_turnstile.c,v 1.53 2023/10/08 13:23:05 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007, 2009, 2019, 2020, 2023
@@ -61,7 +61,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.52 2023/10/04 20:39:35 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: kern_turnstile.c,v 1.53 2023/10/08 13:23:05 ad Exp $");
#include <sys/param.h>
#include <sys/lockdebug.h>
@@ -149,7 +149,7 @@ turnstile_remove(turnstile_t *ts, lwp_t
}
ts->ts_waiters[q]--;
- sleepq_remove(&ts->ts_sleepq[q], l);
+ sleepq_remove(&ts->ts_sleepq[q], l, true);
}
/*
Index: src/sys/kern/sys_lwp.c
diff -u src/sys/kern/sys_lwp.c:1.86 src/sys/kern/sys_lwp.c:1.87
--- src/sys/kern/sys_lwp.c:1.86 Wed Oct 4 20:29:18 2023
+++ src/sys/kern/sys_lwp.c Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: sys_lwp.c,v 1.86 2023/10/04 20:29:18 ad Exp $ */
+/* $NetBSD: sys_lwp.c,v 1.87 2023/10/08 13:23:05 ad Exp $ */
/*-
* Copyright (c) 2001, 2006, 2007, 2008, 2019, 2020, 2023
@@ -36,7 +36,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sys_lwp.c,v 1.86 2023/10/04 20:29:18 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sys_lwp.c,v 1.87 2023/10/08 13:23:05 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -466,6 +466,7 @@ int
lwp_unpark(const lwpid_t *tp, const u_int ntargets)
{
u_int target;
+ kmutex_t *mp;
int error, s;
proc_t *p;
lwp_t *t;
@@ -484,11 +485,10 @@ lwp_unpark(const lwpid_t *tp, const u_in
KASSERT(lwp_locked(t, NULL));
if (__predict_true(t->l_syncobj == &lwp_park_syncobj)) {
- /*
- * As expected it's parked, so wake it up.
- * lwp_unsleep() will release the LWP lock.
- */
- lwp_unsleep(t, true);
+ /* As expected it's parked, so wake it up. */
+ mp = t->l_mutex;
+ sleepq_remove(NULL, t, true);
+ mutex_spin_exit(mp);
} else if (__predict_false(t->l_stat == LSZOMB)) {
lwp_unlock(t);
error = ESRCH;
Index: src/sys/kern/sys_select.c
diff -u src/sys/kern/sys_select.c:1.63 src/sys/kern/sys_select.c:1.64
--- src/sys/kern/sys_select.c:1.63 Wed Oct 4 20:29:18 2023
+++ src/sys/kern/sys_select.c Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: sys_select.c,v 1.63 2023/10/04 20:29:18 ad Exp $ */
+/* $NetBSD: sys_select.c,v 1.64 2023/10/08 13:23:05 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2009, 2010, 2019, 2020, 2023
@@ -85,7 +85,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.63 2023/10/04 20:29:18 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sys_select.c,v 1.64 2023/10/08 13:23:05 ad Exp $");
#include <sys/param.h>
#include <sys/systm.h>
@@ -807,7 +807,7 @@ selnotify(struct selinfo *sip, int event
*/
if (oflag == SEL_BLOCKING && l->l_mutex == lock) {
KASSERT(l->l_wchan == sc);
- sleepq_unsleep(l, false);
+ sleepq_remove(l->l_sleepq, l, true);
}
}
mutex_spin_exit(lock);
Index: src/sys/rump/librump/rumpkern/sleepq.c
diff -u src/sys/rump/librump/rumpkern/sleepq.c:1.26 src/sys/rump/librump/rumpkern/sleepq.c:1.27
--- src/sys/rump/librump/rumpkern/sleepq.c:1.26 Thu Oct 5 19:28:30 2023
+++ src/sys/rump/librump/rumpkern/sleepq.c Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: sleepq.c,v 1.26 2023/10/05 19:28:30 ad Exp $ */
+/* $NetBSD: sleepq.c,v 1.27 2023/10/08 13:23:05 ad Exp $ */
/*
* Copyright (c) 2008 Antti Kantee. All Rights Reserved.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: sleepq.c,v 1.26 2023/10/05 19:28:30 ad Exp $");
+__KERNEL_RCSID(0, "$NetBSD: sleepq.c,v 1.27 2023/10/08 13:23:05 ad Exp $");
#include <sys/param.h>
#include <sys/condvar.h>
@@ -146,6 +146,13 @@ sleepq_unsleep(struct lwp *l, bool clean
}
}
+void
+sleepq_remove(sleepq_t *sq, struct lwp *l, bool wakeup)
+{
+
+ sleepq_unsleep(l, true);
+}
+
/*
* Thread scheduler handles priorities. Therefore no action here.
* (maybe do something if we're deperate?)
Index: src/sys/sys/sleepq.h
diff -u src/sys/sys/sleepq.h:1.39 src/sys/sys/sleepq.h:1.40
--- src/sys/sys/sleepq.h:1.39 Wed Oct 4 20:29:18 2023
+++ src/sys/sys/sleepq.h Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: sleepq.h,v 1.39 2023/10/04 20:29:18 ad Exp $ */
+/* $NetBSD: sleepq.h,v 1.40 2023/10/08 13:23:05 ad Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007, 2008, 2009, 2019, 2020, 2023
@@ -50,7 +50,7 @@ struct syncobj;
typedef struct sleepq sleepq_t;
void sleepq_init(sleepq_t *);
-void sleepq_remove(sleepq_t *, lwp_t *);
+void sleepq_remove(sleepq_t *, lwp_t *, bool);
int sleepq_enter(sleepq_t *, lwp_t *, kmutex_t *);
void sleepq_enqueue(sleepq_t *, wchan_t, const char *,
const struct syncobj *, bool);
Index: src/sys/sys/syncobj.h
diff -u src/sys/sys/syncobj.h:1.16 src/sys/sys/syncobj.h:1.17
--- src/sys/sys/syncobj.h:1.16 Sat Sep 23 18:48:05 2023
+++ src/sys/sys/syncobj.h Sun Oct 8 13:23:05 2023
@@ -1,4 +1,4 @@
-/* $NetBSD: syncobj.h,v 1.16 2023/09/23 18:48:05 ad Exp $ */
+/* $NetBSD: syncobj.h,v 1.17 2023/10/08 13:23:05 ad Exp $ */
/*-
* Copyright (c) 2007, 2008, 2020, 2023 The NetBSD Foundation, Inc.
@@ -57,6 +57,7 @@ struct lwp *syncobj_noowner(wchan_t);
#define SOBJ_SLEEPQ_LIFO 0x04
#define SOBJ_SLEEPQ_NULL 0x08
+extern syncobj_t callout_syncobj;
extern syncobj_t cv_syncobj;
extern syncobj_t kpause_syncobj;
extern syncobj_t lwp_park_syncobj;
