[RFC 06/60] sched: Add a lock-free variant of resched_cpu()

[Jan H . Schönherr]

Add resched_cpu_locked(), which still works as expected, when it is called
while we already hold a runqueue lock from a different CPU.

There is some optimization potential by merging the logic of resched_curr()
and resched_cpu_locked() to avoid IPIs when calls to both functions happen.

Signed-off-by: Jan H. Schönherr <jscho...@amazon.de>
---
 kernel/sched/core.c  | 21 +++++++++++++++++++--
 kernel/sched/sched.h |  6 ++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index fd1b0abd8474..c38a54f57e90 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -486,6 +486,15 @@ void resched_cpu(int cpu)
 }
 
 #ifdef CONFIG_SMP
+/* resched_cpu() when you're already holding a RQ lock of a different CPU */
+void resched_cpu_locked(int cpu)
+{
+       struct rq *rq = cpu_rq(cpu);
+
+       if (!atomic_read(&rq->resched) && !atomic_xchg(&rq->resched, 1))
+               smp_send_reschedule(cpu);
+}
+
 #ifdef CONFIG_NO_HZ_COMMON
 /*
  * In the semi idle case, use the nearest busy CPU for migrating timers
@@ -1744,6 +1753,14 @@ void sched_ttwu_pending(void)
 
 void scheduler_ipi(void)
 {
+       struct rq *rq = this_rq();
+
+       /* Handle lock-free requests to reschedule the current task */
+       if (atomic_read(&rq->resched)) {
+               atomic_set(&rq->resched, 0);
+               set_thread_flag(TIF_NEED_RESCHED);
+       }
+
        /*
         * Fold TIF_NEED_RESCHED into the preempt_count; anybody setting
         * TIF_NEED_RESCHED remotely (for the first time) will also send
@@ -1751,7 +1768,7 @@ void scheduler_ipi(void)
         */
        preempt_fold_need_resched();
 
-       if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
+       if (llist_empty(&rq->wake_list) && !got_nohz_idle_kick())
                return;
 
        /*
@@ -1774,7 +1791,7 @@ void scheduler_ipi(void)
         * Check if someone kicked us for doing the nohz idle load balance.
         */
        if (unlikely(got_nohz_idle_kick())) {
-               this_rq()->idle_balance = 1;
+               rq->idle_balance = 1;
                raise_softirq_irqoff(SCHED_SOFTIRQ);
        }
        irq_exit();
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index f6da85447f3c..926a26d816a2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -850,6 +850,9 @@ struct rq {
        int                     cpu;
        int                     online;
 
+       /* Lock-free rescheduling request for this runqueue */
+       atomic_t resched;
+
        struct list_head cfs_tasks;
 
        struct sched_avg        avg_rt;
@@ -1647,6 +1650,9 @@ extern void reweight_task(struct task_struct *p, int 
prio);
 
 extern void resched_curr(struct rq *rq);
 extern void resched_cpu(int cpu);
+#ifdef CONFIG_SMP
+void resched_cpu_locked(int cpu);
+#endif
 
 extern struct rt_bandwidth def_rt_bandwidth;
 extern void init_rt_bandwidth(struct rt_bandwidth *rt_b, u64 period, u64 
runtime);
-- 
2.9.3.1.gcba166c.dirty