Here's a patch to move the pi_lock out of the "fast path". Thus, only
threads that need to do PI will need to take it.
Comments?
Please look for any race conditions or side effects that I might have
missed.
Thanks,
-- Steve
(Ingo and Thomas, this is what I already sent you)
Signed-off-by: Steven Rostedt <[EMAIL PROTECTED]>
Index: linux_realtime_goliath/kernel/fork.c
===================================================================
--- linux_realtime_goliath/kernel/fork.c (revision 300)
+++ linux_realtime_goliath/kernel/fork.c (working copy)
@@ -1011,6 +1011,7 @@
plist_init(&p->pi_waiters, MAX_PRIO);
preempt_enable();
p->blocked_on = NULL; /* not blocked yet */
+ spin_lock_init(&p->rt_lock);
p->tgid = p->pid;
if (clone_flags & CLONE_THREAD)
Index: linux_realtime_goliath/kernel/rt.c
===================================================================
--- linux_realtime_goliath/kernel/rt.c (revision 301)
+++ linux_realtime_goliath/kernel/rt.c (working copy)
@@ -759,6 +759,7 @@
* If the task is blocked on some other task then boost that
* other task (or tasks) too:
*/
+ __raw_spin_lock(&p->rt_lock);
for (;;) {
struct rt_mutex_waiter *w = p->blocked_on;
#ifdef CONFIG_RT_DEADLOCK_DETECT
@@ -777,7 +778,6 @@
TRACE_BUG_ON_LOCKED(!lock);
TRACE_BUG_ON_LOCKED(!lock_owner(lock));
if (rt_task(p) && plist_empty(&w->pi_list)) {
- TRACE_BUG_ON_LOCKED(was_rt);
plist_init(&w->pi_list, prio);
plist_add(&w->pi_list,
&lock_owner(lock)->task->pi_waiters);
@@ -803,8 +803,10 @@
pi_walk++;
+ __raw_spin_unlock(&p->rt_lock);
p = lock_owner(lock)->task;
TRACE_BUG_ON_LOCKED(!p);
+ __raw_spin_lock(&p->rt_lock);
/*
* If the dependee is already higher-prio then
* no need to boost it, and all further tasks down
@@ -813,6 +815,7 @@
if (p->prio <= prio)
break;
}
+ __raw_spin_unlock(&p->rt_lock);
}
/*
@@ -869,7 +872,7 @@
/* mark the current thread as blocked on the lock */
waiter->eip = eip;
#endif
- __raw_spin_lock(&pi_lock);
+ __raw_spin_lock(&task->rt_lock);
task->blocked_on = waiter;
waiter->lock = lock;
waiter->ti = ti;
@@ -881,15 +884,23 @@
if (!rt_task(task)) {
plist_add(&waiter->list, &lock->wait_list);
set_lock_owner_pending(lock);
- __raw_spin_unlock(&pi_lock);
+ __raw_spin_unlock(&task->rt_lock);
return;
}
#endif
- plist_add(&waiter->pi_list, &lock_owner(lock)->task->pi_waiters);
+ __raw_spin_unlock(&task->rt_lock);
+ __raw_spin_lock(&pi_lock);
/*
- * Add RT tasks to the head:
+ * We could have been added to the pi list from another task
+ * doing a pi_setprio.
*/
- plist_add(&waiter->list, &lock->wait_list);
+ if (likely(plist_empty(&waiter->pi_list))) {
+ plist_add(&waiter->pi_list,
&lock_owner(lock)->task->pi_waiters);
+ /*
+ * Add RT tasks to the head:
+ */
+ plist_add(&waiter->list, &lock->wait_list);
+ }
set_lock_owner_pending(lock);
/*
* If the waiter has higher priority than the owner
Index: linux_realtime_goliath/include/linux/init_task.h
===================================================================
--- linux_realtime_goliath/include/linux/init_task.h (revision 300)
+++ linux_realtime_goliath/include/linux/init_task.h (working copy)
@@ -112,6 +112,7 @@
.proc_lock = SPIN_LOCK_UNLOCKED(tsk.proc_lock), \
.delayed_put = LIST_HEAD_INIT(tsk.delayed_put), \
.pi_waiters = PLIST_INIT(tsk.pi_waiters, MAX_PRIO), \
+ .rt_lock = RAW_SPIN_LOCK_UNLOCKED, \
.journal_info = NULL, \
.cpu_timers = INIT_CPU_TIMERS(tsk.cpu_timers), \
.fs_excl = ATOMIC_INIT(0), \
Index: linux_realtime_goliath/include/linux/sched.h
===================================================================
--- linux_realtime_goliath/include/linux/sched.h (revision 301)
+++ linux_realtime_goliath/include/linux/sched.h (working copy)
@@ -950,6 +950,7 @@
/* RT deadlock detection and priority inheritance handling */
struct rt_mutex_waiter *blocked_on;
struct rt_mutex *pending_owner;
+ raw_spinlock_t rt_lock;
unsigned long rt_flags;
-
To unsubscribe from this list: send the line "unsubscribe linux-kernel" in
the body of a message to [EMAIL PROTECTED]
More majordomo info at http://vger.kernel.org/majordomo-info.html
Please read the FAQ at http://www.tux.org/lkml/
