Inserts call to main logic from 'load_balance', balance parameters calculation during enqueue/dequeue task from runqueue and affinity mask change callback for fair scheduling class.
Signed-off-by: Tarek Dakhran <[email protected]> Signed-off-by: Sergey Dyasly <[email protected]> Signed-off-by: Dmitriy Safonov <[email protected]> Signed-off-by: Arseniy Krasnov <[email protected]> Signed-off-by: Ilya Maximets <[email protected]> --- kernel/sched/fair.c | 204 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 202 insertions(+), 2 deletions(-) diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c index 79be023..06f6518 100644 --- a/kernel/sched/fair.c +++ b/kernel/sched/fair.c @@ -677,6 +677,16 @@ static u64 sched_vslice(struct cfs_rq *cfs_rq, struct sched_entity *se) } #ifdef CONFIG_HPERF_HMP +static void hmp_calculate_imbalance(void) +{ + if (atomic_long_read(&a7_total_weight) == 0) { + atomic_set(&hmp_imbalance, 0); + return; + } + + atomic_set(&hmp_imbalance, 1); +} + static bool is_task_hmp(struct task_struct *task, const struct cpumask *task_cpus) { @@ -711,6 +721,13 @@ static inline void add_druntime_sum(struct rq *rq, long delta) rq->druntime_sum += delta; check_druntime_sum(rq, rq->druntime_sum); } + +static inline void sub_druntime_sum(struct rq *rq, long delta) +{ + rq->druntime_sum -= delta; + check_druntime_sum(rq, rq->druntime_sum); +} + /* Updates druntime for a task */ static inline void update_hmp_stat(struct cfs_rq *cfs_rq, struct sched_entity *curr, @@ -861,7 +878,9 @@ static void update_curr(struct cfs_rq *cfs_rq) account_cfs_rq_runtime(cfs_rq, delta_exec); +#ifdef CONFIG_HPERF_HMP update_hmp_stat(cfs_rq, curr, delta_exec); +#endif } static void update_curr_fair(struct rq *rq) @@ -4200,6 +4219,66 @@ static inline void hrtick_update(struct rq *rq) } #endif +#ifdef CONFIG_HPERF_HMP +#ifdef CONFIG_HPERF_HMP_DEBUG +static void check_nr_hmp_tasks(struct rq *rq) +{ + if (rq->nr_hmp_tasks > rq->cfs.h_nr_running) { + pr_emerg("HMP BUG: rq->nr_hmp_tasks = %u, " + "rq->cfs.h_nr_running = %u\n", rq->nr_hmp_tasks, + rq->cfs.h_nr_running); + BUG(); + } +} +#else +static void check_nr_hmp_tasks(struct rq *rq) { } +#endif + +static void nr_hmp_tasks_inc(struct rq *rq) +{ + if (!rq->nr_hmp_tasks) { + if (cpu_is_fastest(rq->cpu)) + atomic_inc(&a15_nr_hmp_busy); + else + atomic_inc(&a7_nr_hmp_busy); + } + + rq->nr_hmp_tasks++; + check_nr_hmp_tasks(rq); +} + +static void nr_hmp_tasks_dec(struct rq *rq) +{ + rq->nr_hmp_tasks--; + + if (!rq->nr_hmp_tasks) { + if (cpu_is_fastest(rq->cpu)) + atomic_dec(&a15_nr_hmp_busy); + else + atomic_dec(&a7_nr_hmp_busy); + } + check_nr_hmp_tasks(rq); +} + +static void +set_cpus_allowed_hmp(struct task_struct *p, const struct cpumask *new_mask) +{ + bool is_hmp_before, is_hmp_after; + + cpumask_copy(&p->cpus_allowed, new_mask); + p->nr_cpus_allowed = cpumask_weight(new_mask); + is_hmp_before = is_task_hmp(p, NULL); + is_hmp_after = is_task_hmp(p, new_mask); + + if (!p->on_cpu && p->se.on_rq && (is_hmp_before != is_hmp_after)) { + if (is_hmp_after) + nr_hmp_tasks_inc(rq_of(cfs_rq_of(&p->se))); + else + nr_hmp_tasks_dec(rq_of(cfs_rq_of(&p->se))); + } +} +#endif + /* * The enqueue_task method is called before nr_running is * increased. Here we update the fair scheduling stats and @@ -4241,8 +4320,24 @@ enqueue_task_fair(struct rq *rq, struct task_struct *p, int flags) update_cfs_shares(cfs_rq); } - if (!se) + if (!se) { add_nr_running(rq, 1); +#ifdef CONFIG_HPERF_HMP + if (is_task_hmp(p, NULL)) + nr_hmp_tasks_inc(rq); + + if (cpu_is_fastest(rq->cpu)) { + atomic_long_add(p->se.load.weight, &a15_total_weight); + if (p->se.druntime < 0) + add_druntime_sum(rq, p->se.druntime); + } else { + atomic_long_add(p->se.load.weight, &a7_total_weight); + if (p->se.druntime > 0) + add_druntime_sum(rq, p->se.druntime); + } + hmp_calculate_imbalance(); +#endif + } hrtick_update(rq); } @@ -4301,8 +4396,30 @@ static void dequeue_task_fair(struct rq *rq, struct task_struct *p, int flags) update_cfs_shares(cfs_rq); } - if (!se) + if (!se) { sub_nr_running(rq, 1); +#ifdef CONFIG_HPERF_HMP + if (is_task_hmp(p, NULL)) + nr_hmp_tasks_dec(rq); + + /* + * Set this field to 0 because if task selected for migration + * fall asleep it will never be selected again for migration. + */ + p->se.migrate_candidate = 0; + + if (cpu_is_fastest(rq->cpu)) { + atomic_long_sub(p->se.load.weight, &a15_total_weight); + if (p->se.druntime < 0) + sub_druntime_sum(rq, p->se.druntime); + } else { + atomic_long_sub(p->se.load.weight, &a7_total_weight); + if (p->se.druntime > 0) + sub_druntime_sum(rq, p->se.druntime); + } + hmp_calculate_imbalance(); +#endif + } hrtick_update(rq); } @@ -5848,6 +5965,11 @@ int can_migrate_task(struct task_struct *p, struct lb_env *env) } schedstat_inc(p, se.statistics.nr_failed_migrations_hot); + +#ifdef CONFIG_HPERF_HMP + if (env->src_rq->migrate_task) /*idle pull*/ + return 1; +#endif return 0; } @@ -5879,6 +6001,10 @@ static struct task_struct *detach_one_task(struct lb_env *env) if (!can_migrate_task(p, env)) continue; +#ifdef CONFIG_HPERF_HMP + if (p->se.migrate_candidate) + continue; +#endif detach_task(p, env); /* @@ -5946,6 +6072,10 @@ static int detach_tasks(struct lb_env *env) if ((load / 2) > env->imbalance) goto next; +#ifdef CONFIG_HPERF_HMP + if (p->se.migrate_candidate) + goto next; +#endif detach_task(p, env); list_add(&p->se.group_node, &env->tasks); @@ -7909,6 +8039,44 @@ unlock: return ld_moved; } + +/** + * hmp_do_rebalance(): Checks imbalance in HMP domain and performs balancing. + * + * @sd: Current sched domain. + * @this_cpu: without NO_HZ same as smp_processor_id(). + * + * Returns moved weight. + */ +static unsigned int hmp_do_rebalance(struct sched_domain *sd, int this_cpu) +{ + unsigned int ld_moved = 0; + switch (is_hmp_imbalance(sd)) { + case SWAP_TASKS: + ld_moved = swap_tasks(sd, this_cpu); + break; + case A15_TO_A7: + ld_moved = move_a15_to_a7(sd, this_cpu); + break; + case A7_TO_A15: + ld_moved = move_a7_to_a15(sd, this_cpu); + break; + case SKIP_REBALANCE: + default: + break; + } + return ld_moved; +} + +/* HMP balancing entry point */ +static unsigned int hmp_load_balance(struct sched_domain *sd, + enum cpu_idle_type idle, int this_cpu) +{ + if (idle == CPU_NEWLY_IDLE || idle == CPU_IDLE) + return hmp_idle_pull(sd, this_cpu); + else + return hmp_do_rebalance(sd, this_cpu); +} #endif /* CONFIG_HPERF_HMP */ /* @@ -7938,6 +8106,11 @@ static int load_balance(int this_cpu, struct rq *this_rq, .tasks = LIST_HEAD_INIT(env.tasks), }; +#ifdef CONFIG_HPERF_HMP + /* It is HMP domain, so branch to HPERF_HMP logic */ + if (sd->flags & SD_HMP_BALANCE) + return hmp_load_balance(sd, idle, this_cpu); +#endif /* * For NEWLY_IDLE load_balancing, we don't need to consider * other cpus in our group @@ -8322,6 +8495,14 @@ static int active_load_balance_cpu_stop(void *data) struct sched_domain *sd; struct task_struct *p = NULL; +#ifdef CONFIG_HPERF_HMP_DEBUG + if (!cpumask_test_cpu(target_cpu, cpu_fastest_mask) && + !cpumask_test_cpu(target_cpu, cpu_slowest_mask)) { + pr_emerg("hperf_hmp: %s: for CPU#%i target_cpu is invalid: %i!\n", + __func__, busiest_cpu, target_cpu); + BUG(); + } +#endif raw_spin_lock_irq(&busiest_rq->lock); /* make sure the requested cpu hasn't gone down in the meantime */ @@ -8349,6 +8530,9 @@ static int active_load_balance_cpu_stop(void *data) } if (likely(sd)) { +#ifdef CONFIG_HPERF_HMP + struct task_struct *migrate_task; +#endif struct lb_env env = { .sd = sd, .dst_cpu = target_cpu, @@ -8360,7 +8544,19 @@ static int active_load_balance_cpu_stop(void *data) schedstat_inc(sd, alb_count); +#ifdef CONFIG_HPERF_HMP + if (env.src_rq->migrate_task) { + migrate_task = env.src_rq->migrate_task; + p = detach_specified_task(migrate_task, &env); + if (p) + migrate_task->se.last_migration = jiffies; + env.src_rq->migrate_task = NULL; + } else { + p = detach_one_task(&env); + } +#else p = detach_one_task(&env); +#endif if (p) schedstat_inc(sd, alb_pushed); else @@ -9267,8 +9463,12 @@ const struct sched_class fair_sched_class = { .task_waking = task_waking_fair, .task_dead = task_dead_fair, +#ifdef CONFIG_HPERF_HMP + .set_cpus_allowed = set_cpus_allowed_hmp, +#else .set_cpus_allowed = set_cpus_allowed_common, #endif +#endif .set_curr_task = set_curr_task_fair, .task_tick = task_tick_fair, -- 1.9.1 -- To unsubscribe from this list: send the line "unsubscribe linux-kernel" in the body of a message to [email protected] More majordomo info at http://vger.kernel.org/majordomo-info.html Please read the FAQ at http://www.tux.org/lkml/
