From: Srivatsa Vaddagiri <va...@codeaurora.org>
Add the necessary hooks to core and the various scheduling
classes that will allow WALT to track CPU utilization and
handle task migration between CPUs as well.
With CONFIG_SCHED_WALT enabled, schedutil will use WALT's cpu
utilization metric by default. This can be switched to PELT's
util_avg at runtime by the following command:
echo 0 > /proc/sys/kernel/sched_use_walt_metrics
Signed-off-by: Srivatsa Vaddagiri <va...@codeaurora.org>
Signed-off-by: Vikram Mulukutla <mark...@codeaurora.org>
---
kernel/sched/core.c | 29 ++++++++++++++++++++++++++++-
kernel/sched/deadline.c | 7 +++++++
kernel/sched/debug.c | 9 +++++++++
kernel/sched/fair.c | 9 +++++++--
kernel/sched/rt.c | 6 ++++++
5 files changed, 57 insertions(+), 3 deletions(-)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 44817c6..3b7f67d 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -91,6 +91,8 @@
#define CREATE_TRACE_POINTS
#include <trace/events/sched.h>
+#include "walt.h"
+
DEFINE_MUTEX(sched_domains_mutex);
DEFINE_PER_CPU_SHARED_ALIGNED(struct rq, runqueues);
@@ -991,6 +993,7 @@ static struct rq *move_queued_task(struct rq *rq, struct
task_struct *p, int new
p->on_rq = TASK_ON_RQ_MIGRATING;
dequeue_task(rq, p, 0);
+ walt_prepare_migrate(p, rq, true);
set_task_cpu(p, new_cpu);
raw_spin_unlock(&rq->lock);
@@ -998,6 +1001,7 @@ static struct rq *move_queued_task(struct rq *rq, struct
task_struct *p, int new
raw_spin_lock(&rq->lock);
BUG_ON(task_cpu(p) != new_cpu);
+ walt_finish_migrate(p, rq, true);
enqueue_task(rq, p, 0);
p->on_rq = TASK_ON_RQ_QUEUED;
check_preempt_curr(rq, p, 0);
@@ -1257,7 +1261,9 @@ static void __migrate_swap_task(struct task_struct *p,
int cpu)
p->on_rq = TASK_ON_RQ_MIGRATING;
deactivate_task(src_rq, p, 0);
+ walt_prepare_migrate(p, src_rq, true);
set_task_cpu(p, cpu);
+ walt_finish_migrate(p, dst_rq, true);
activate_task(dst_rq, p, 0);
p->on_rq = TASK_ON_RQ_QUEUED;
check_preempt_curr(dst_rq, p, 0);
@@ -2072,13 +2078,19 @@ try_to_wake_up(struct task_struct *p, unsigned int
state, int wake_flags)
*/
smp_cond_load_acquire(&p->on_cpu, !VAL);
+ raw_spin_lock(&task_rq(p)->lock);
+ walt_update_task_ravg(p, task_rq(p), TASK_WAKE, walt_ktime_clock(), 0);
+ raw_spin_unlock(&task_rq(p)->lock);
+
p->sched_contributes_to_load = !!task_contributes_to_load(p);
p->state = TASK_WAKING;
cpu = select_task_rq(p, p->wake_cpu, SD_BALANCE_WAKE, wake_flags);
if (task_cpu(p) != cpu) {
wake_flags |= WF_MIGRATED;
+ walt_prepare_migrate(p, task_rq(p), false);
set_task_cpu(p, cpu);
+ walt_finish_migrate(p, cpu_rq(cpu), false);
}
#endif /* CONFIG_SMP */
@@ -2129,8 +2141,10 @@ static void try_to_wake_up_local(struct task_struct *p,
struct pin_cookie cookie
trace_sched_waking(p);
- if (!task_on_rq_queued(p))
+ if (!task_on_rq_queued(p)) {
+ walt_update_task_ravg(p, rq, TASK_WAKE, walt_ktime_clock(), 0);
ttwu_activate(rq, p, ENQUEUE_WAKEUP);
+ }
ttwu_do_wakeup(rq, p, 0, cookie);
if (schedstat_enabled())
@@ -2196,6 +2210,7 @@ static void __sched_fork(unsigned long clone_flags,
struct task_struct *p)
p->se.nr_migrations = 0;
p->se.vruntime = 0;
INIT_LIST_HEAD(&p->se.group_node);
+ walt_init_new_task_load(p);
#ifdef CONFIG_FAIR_GROUP_SCHED
p->se.cfs_rq = NULL;
@@ -2570,6 +2585,8 @@ void wake_up_new_task(struct task_struct *p)
rq = __task_rq_lock(p, &rf);
post_init_entity_util_avg(&p->se);
+ walt_mark_task_starting(p);
+
activate_task(rq, p, 0);
p->on_rq = TASK_ON_RQ_QUEUED;
trace_sched_wakeup_new(p);
@@ -3071,6 +3088,7 @@ void scheduler_tick(void)
update_rq_clock(rq);
curr->sched_class->task_tick(rq, curr, 0);
cpu_load_update_active(rq);
+ walt_update_task_ravg(rq->curr, rq, TASK_UPDATE, walt_ktime_clock(), 0);
calc_global_load_tick(rq);
raw_spin_unlock(&rq->lock);
@@ -3322,6 +3340,7 @@ static void __sched notrace __schedule(bool preempt)
struct pin_cookie cookie;
struct rq *rq;
int cpu;
+ u64 wallclock;
cpu = smp_processor_id();
rq = cpu_rq(cpu);
@@ -3385,6 +3404,9 @@ static void __sched notrace __schedule(bool preempt)
update_rq_clock(rq);
next = pick_next_task(rq, prev, cookie);
+ wallclock = walt_ktime_clock();
+ walt_update_task_ravg(prev, rq, PUT_PREV_TASK, wallclock, 0);
+ walt_update_task_ravg(next, rq, PICK_NEXT_TASK, wallclock, 0);
clear_tsk_need_resched(prev);
clear_preempt_need_resched();
rq->clock_skip_update = 0;
@@ -7284,6 +7306,8 @@ static void sched_rq_cpu_starting(unsigned int cpu)
{
struct rq *rq = cpu_rq(cpu);
+ walt_set_window_start(rq);
+
rq->calc_load_update = calc_load_update;
update_max_interval();
}
@@ -7304,6 +7328,9 @@ int sched_cpu_dying(unsigned int cpu)
/* Handle pending wakeups and then migrate everything off */
sched_ttwu_pending();
raw_spin_lock_irqsave(&rq->lock, flags);
+
+ walt_migrate_sync_cpu(cpu);
+
if (rq->rd) {
BUG_ON(!cpumask_test_cpu(cpu, rq->rd->span));
set_rq_offline(rq);
diff --git a/kernel/sched/deadline.c b/kernel/sched/deadline.c
index 1ce8867..0dd3c1f 100644
--- a/kernel/sched/deadline.c
+++ b/kernel/sched/deadline.c
@@ -15,6 +15,7 @@
* Fabio Checconi <fchecc...@gmail.com>
*/
#include "sched.h"
+#include "walt.h"
#include <linux/slab.h>
@@ -278,7 +279,9 @@ static struct rq *dl_task_offline_migration(struct rq *rq,
struct task_struct *p
* By now the task is replenished and enqueued; migrate it.
*/
deactivate_task(rq, p, 0);
+ walt_prepare_migrate(p, rq, true);
set_task_cpu(p, later_rq->cpu);
+ walt_finish_migrate(p, later_rq, true);
activate_task(later_rq, p, 0);
if (!fallback)
@@ -1512,7 +1515,9 @@ retry:
}
deactivate_task(rq, next_task, 0);
+ walt_prepare_migrate(next_task, rq, true);
set_task_cpu(next_task, later_rq->cpu);
+ walt_finish_migrate(next_task, later_rq, true);
activate_task(later_rq, next_task, 0);
ret = 1;
@@ -1600,7 +1605,9 @@ static void pull_dl_task(struct rq *this_rq)
resched = true;
deactivate_task(src_rq, p, 0);
+ walt_prepare_migrate(p, src_rq, true);
set_task_cpu(p, this_cpu);
+ walt_finish_migrate(p, this_rq, true);
activate_task(this_rq, p, 0);
dmin = p->dl.deadline;
diff --git a/kernel/sched/debug.c b/kernel/sched/debug.c
index 2a0a999..ab10031 100644
--- a/kernel/sched/debug.c
+++ b/kernel/sched/debug.c
@@ -607,6 +607,15 @@ do {
\
P(nr_switches);
P(nr_load_updates);
P(nr_uninterruptible);
+#ifdef CONFIG_SMP
+ P(cpu_capacity_orig);
+ P(cpu_capacity);
+#ifdef CONFIG_SCHED_WALT
+ P(window_start);
+ P(curr_runnable_sum);
+ P(prev_runnable_sum);
+#endif
+#endif
PN(next_balance);
SEQ_printf(m, " .%-30s: %ld\n", "curr->pid",
(long)(task_pid_nr(rq->curr)));
PN(clock);
diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 39c826d..182dcd3 100644
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -34,6 +34,7 @@
#include <trace/events/sched.h>
#include "sched.h"
+#include "walt.h"
/*
* Targeted preemption latency for CPU-bound tasks:
@@ -2885,6 +2886,7 @@ static inline void cfs_rq_util_change(struct cfs_rq
*cfs_rq)
if (cpu == smp_processor_id() && &rq->cfs == cfs_rq) {
unsigned long max = rq->cpu_capacity_orig;
+ unsigned long util = cpu_walt_util(rq);
/*
* There are a few boundary cases this might miss but it should
@@ -2902,8 +2904,8 @@ static inline void cfs_rq_util_change(struct cfs_rq
*cfs_rq)
*
* See cpu_util().
*/
- cpufreq_update_util(rq_clock(rq),
- min(cfs_rq->avg.util_avg, max), max);
+
+ cpufreq_update_util(rq_clock(rq), min(util, max), max);
}
}
@@ -6205,7 +6207,9 @@ static void detach_task(struct task_struct *p, struct
lb_env *env)
p->on_rq = TASK_ON_RQ_MIGRATING;
deactivate_task(env->src_rq, p, 0);
+ walt_prepare_migrate(p, env->src_rq, true);
set_task_cpu(p, env->dst_cpu);
+ /* update WALT later under the dest rq's lock */
}
/*
@@ -6337,6 +6341,7 @@ static void attach_task(struct rq *rq, struct task_struct
*p)
lockdep_assert_held(&rq->lock);
BUG_ON(task_rq(p) != rq);
+ walt_finish_migrate(p, rq, true);
activate_task(rq, p, 0);
p->on_rq = TASK_ON_RQ_QUEUED;
check_preempt_curr(rq, p, 0);
diff --git a/kernel/sched/rt.c b/kernel/sched/rt.c
index d5690b7..130040c 100644
--- a/kernel/sched/rt.c
+++ b/kernel/sched/rt.c
@@ -8,6 +8,8 @@
#include <linux/slab.h>
#include <linux/irq_work.h>
+#include "walt.h"
+
int sched_rr_timeslice = RR_TIMESLICE;
static int do_sched_rt_period_timer(struct rt_bandwidth *rt_b, int overrun);
@@ -1843,7 +1845,9 @@ retry:
}
deactivate_task(rq, next_task, 0);
+ walt_prepare_migrate(next_task, rq, true);
set_task_cpu(next_task, lowest_rq->cpu);
+ walt_finish_migrate(next_task, lowest_rq, true);
activate_task(lowest_rq, next_task, 0);
ret = 1;
@@ -2097,7 +2101,9 @@ static void pull_rt_task(struct rq *this_rq)
resched = true;
deactivate_task(src_rq, p, 0);
+ walt_prepare_migrate(p, src_rq, true);
set_task_cpu(p, this_cpu);
+ walt_finish_migrate(p, this_rq, true);
activate_task(this_rq, p, 0);
/*
* We continue with the search, just in
--
TheMan
