From: Peter Zijlstra <pet...@infradead.org>
Marks all tasks in a cgroup as matching for core-scheduling.
Signed-off-by: Peter Zijlstra (Intel) <pet...@infradead.org>
Signed-off-by: Julien Desfossez <jdesfos...@digitalocean.com>
Signed-off-by: Vineeth Remanan Pillai <vpil...@digitalocean.com>
---
Changes in v3
-------------
- Fixes the refcount management when deleting a tagged cgroup.
- Julien Desfossez
---
kernel/sched/core.c | 78 ++++++++++++++++++++++++++++++++++++++++++++
kernel/sched/sched.h | 4 +++
2 files changed, 82 insertions(+)
diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 112d70f2b1e5..3164c6b33553 100644
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -6658,6 +6658,15 @@ static void sched_change_group(struct task_struct *tsk,
int type)
tg = container_of(task_css_check(tsk, cpu_cgrp_id, true),
struct task_group, css);
tg = autogroup_task_group(tsk, tg);
+
+#ifdef CONFIG_SCHED_CORE
+ if ((unsigned long)tsk->sched_task_group == tsk->core_cookie)
+ tsk->core_cookie = 0UL;
+
+ if (tg->tagged /* && !tsk->core_cookie ? */)
+ tsk->core_cookie = (unsigned long)tg;
+#endif
+
tsk->sched_task_group = tg;
#ifdef CONFIG_FAIR_GROUP_SCHED
@@ -6737,6 +6746,18 @@ static int cpu_cgroup_css_online(struct
cgroup_subsys_state *css)
return 0;
}
+static void cpu_cgroup_css_offline(struct cgroup_subsys_state *css)
+{
+#ifdef CONFIG_SCHED_CORE
+ struct task_group *tg = css_tg(css);
+
+ if (tg->tagged) {
+ sched_core_put();
+ tg->tagged = 0;
+ }
+#endif
+}
+
static void cpu_cgroup_css_released(struct cgroup_subsys_state *css)
{
struct task_group *tg = css_tg(css);
@@ -7117,6 +7138,46 @@ static u64 cpu_rt_period_read_uint(struct
cgroup_subsys_state *css,
}
#endif /* CONFIG_RT_GROUP_SCHED */
+#ifdef CONFIG_SCHED_CORE
+static u64 cpu_core_tag_read_u64(struct cgroup_subsys_state *css, struct
cftype *cft)
+{
+ struct task_group *tg = css_tg(css);
+
+ return !!tg->tagged;
+}
+
+static int cpu_core_tag_write_u64(struct cgroup_subsys_state *css, struct
cftype *cft, u64 val)
+{
+ struct task_group *tg = css_tg(css);
+ struct css_task_iter it;
+ struct task_struct *p;
+
+ if (val > 1)
+ return -ERANGE;
+
+ if (!static_branch_likely(&sched_smt_present))
+ return -EINVAL;
+
+ if (tg->tagged == !!val)
+ return 0;
+
+ tg->tagged = !!val;
+
+ if (!!val)
+ sched_core_get();
+
+ css_task_iter_start(css, 0, &it);
+ while ((p = css_task_iter_next(&it)))
+ p->core_cookie = !!val ? (unsigned long)tg : 0UL;
+ css_task_iter_end(&it);
+
+ if (!val)
+ sched_core_put();
+
+ return 0;
+}
+#endif
+
static struct cftype cpu_legacy_files[] = {
#ifdef CONFIG_FAIR_GROUP_SCHED
{
@@ -7152,6 +7213,14 @@ static struct cftype cpu_legacy_files[] = {
.read_u64 = cpu_rt_period_read_uint,
.write_u64 = cpu_rt_period_write_uint,
},
+#endif
+#ifdef CONFIG_SCHED_CORE
+ {
+ .name = "tag",
+ .flags = CFTYPE_NOT_ON_ROOT,
+ .read_u64 = cpu_core_tag_read_u64,
+ .write_u64 = cpu_core_tag_write_u64,
+ },
#endif
{ } /* Terminate */
};
@@ -7319,6 +7388,14 @@ static struct cftype cpu_files[] = {
.seq_show = cpu_max_show,
.write = cpu_max_write,
},
+#endif
+#ifdef CONFIG_SCHED_CORE
+ {
+ .name = "tag",
+ .flags = CFTYPE_NOT_ON_ROOT,
+ .read_u64 = cpu_core_tag_read_u64,
+ .write_u64 = cpu_core_tag_write_u64,
+ },
#endif
{ } /* terminate */
};
@@ -7326,6 +7403,7 @@ static struct cftype cpu_files[] = {
struct cgroup_subsys cpu_cgrp_subsys = {
.css_alloc = cpu_cgroup_css_alloc,
.css_online = cpu_cgroup_css_online,
+ .css_offline = cpu_cgroup_css_offline,
.css_released = cpu_cgroup_css_released,
.css_free = cpu_cgroup_css_free,
.css_extra_stat_show = cpu_extra_stat_show,
diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index 0cbcfb6c8ee4..bd9b473ebde2 100644
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -363,6 +363,10 @@ struct cfs_bandwidth {
struct task_group {
struct cgroup_subsys_state css;
+#ifdef CONFIG_SCHED_CORE
+ int tagged;
+#endif
+
#ifdef CONFIG_FAIR_GROUP_SCHED
/* schedulable entities of this group on each CPU */
struct sched_entity **se;
--
2.17.1
