Track A (cpuset_update_sd_hk_unlock) updates the HK_TYPE_KERNEL_NOISE
and HK_TYPE_MANAGED_IRQ cpumasks but performs no per-CPU reconfiguration.
Tick suppression, RCU callback offloading and managed-IRQ remapping only
take effect when the affected CPUs pass through the CPU hotplug machinery.
Implement dhm_cycle_isolated_cpus() and call it from
cpuset_update_sd_hk_unlock() after all cpuset and hotplug locks are
released so that remove_cpu()/add_cpu() may acquire cpus_write_lock
without violating the cpu_hotplug_lock > cpuset_top_mutex order.
On isolation, for each newly-isolated CPU:
1. remove_cpu() - offline; dying callbacks migrate IRQs
2. tick_nohz_cpu_isolate() - add to tick_nohz_full_mask, enable
context tracking (B0/B3)
3. rcu_nocb_cpu_isolate() - lazy nocb init, spawn kthreads, offload
callbacks (B1)
4. add_cpu() - online; tick and IRQ online callbacks
reconfigure against updated HK masks
On de-isolation, the reverse order is applied. The managed-IRQ
remapping requires no explicit call: irq_migrate_all_off_this_cpu()
(dying callback) and irq_affinity_online_cpu() (online callback)
already consult the updated HK_TYPE_MANAGED_IRQ mask.
dhm_prev_isolated tracks the previous isolation set so that only CPUs
whose state changed are cycled rather than the full isolation set.
lockup_detector_hk_update() (B2) is called once after all CPUs are
cycled to update the watchdog mask.
CPUs with hotplug disabled (e.g. x86-64 boot CPU) cannot be taken
offline and are skipped. On any remove_cpu() failure the corresponding
CPU is cleared from dhm_prev_isolated so the next isolation attempt
will retry rather than silently treating it as already isolated.
Symmetrically, a de-isolation remove_cpu() failure re-sets the bit in
dhm_prev_isolated so the CPU remains tracked as still isolated.
Co-developed-by: Qiliang Yuan <[email protected]>
Signed-off-by: Qiliang Yuan <[email protected]>
Signed-off-by: Jing Wu <[email protected]>
---
kernel/cgroup/cpuset.c | 98 ++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 98 insertions(+)
diff --git a/kernel/cgroup/cpuset.c b/kernel/cgroup/cpuset.c
index 62eb6798a0c3e..d9e121bf14292 100644
--- a/kernel/cgroup/cpuset.c
+++ b/kernel/cgroup/cpuset.c
@@ -20,6 +20,8 @@
*/
#include "cpuset-internal.h"
+#include <linux/cpu.h>
+#include <linux/cpuhplock.h>
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/kernel.h>
@@ -33,7 +35,9 @@
#include <linux/sched/task.h>
#include <linux/security.h>
#include <linux/oom.h>
+#include <linux/nmi.h>
#include <linux/sched/isolation.h>
+#include <linux/tick.h>
#include <linux/wait.h>
#include <linux/workqueue.h>
#include <linux/task_work.h>
@@ -164,6 +168,14 @@ static cpumask_var_t isolated_hk_cpus; /* T */
static DEFINE_SPINLOCK(dhm_cycling_lock);
static cpumask_var_t dhm_cycling_cpus;
+/*
+ * Snapshot of the isolated CPUs from the previous housekeeping update.
+ * Used to compute the delta (newly isolated / newly de-isolated) so that
+ * only the changed CPUs are cycled rather than the full isolation set.
+ * Protected by cpuset_top_mutex.
+ */
+static cpumask_var_t dhm_prev_isolated;
+
/*
* A flag to force sched domain rebuild at the end of an operation.
* It can be set in
@@ -1339,6 +1351,84 @@ static bool prstate_housekeeping_conflict(int prstate,
struct cpumask *new_cpus)
return false;
}
+/*
+ * dhm_cycle_isolated_cpus - Apply kernel-noise isolation via hotplug cycling
+ *
+ * For each CPU newly entering isolation: cycle it offline, configure tick
+ * suppression and RCU callback offloading while it is offline, then bring
+ * it back online. The managed-IRQ state is handled automatically by the
+ * existing irq_migrate_all_off_this_cpu() dying callback and the
+ * irq_affinity_online_cpu() online callback which both consult the
+ * already-updated HK_TYPE_MANAGED_IRQ mask.
+ *
+ * For each CPU leaving isolation: cycle it offline, de-offload RCU and
+ * restore the tick, then bring it back online.
+ *
+ * Must be called without any cpuset or hotplug locks held.
+ */
+static void dhm_cycle_isolated_cpus(const struct cpumask *new_isolated)
+{
+ cpumask_var_t newly_isolated, newly_deisolated;
+ int cpu;
+
+ if (!alloc_cpumask_var(&newly_isolated, GFP_KERNEL) ||
+ !alloc_cpumask_var(&newly_deisolated, GFP_KERNEL)) {
+ free_cpumask_var(newly_isolated);
+ return;
+ }
+
+ cpumask_andnot(newly_isolated, new_isolated, dhm_prev_isolated);
+ cpumask_andnot(newly_deisolated, dhm_prev_isolated, new_isolated);
+ cpumask_copy(dhm_prev_isolated, new_isolated);
+
+ if (cpumask_empty(newly_isolated) && cpumask_empty(newly_deisolated))
+ return;
+
+ /* Mark cycling CPUs so cpuset_hotplug_update_tasks skips invalidation
*/
+ spin_lock(&dhm_cycling_lock);
+ cpumask_or(dhm_cycling_cpus, newly_isolated, newly_deisolated);
+ spin_unlock(&dhm_cycling_lock);
+
+ for_each_cpu(cpu, newly_isolated) {
+ if (!cpu_is_hotpluggable(cpu)) {
+ pr_warn_once("cpuset: CPU%d cannot be isolated (hotplug
disabled)\n",
+ cpu);
+ cpumask_clear_cpu(cpu, dhm_prev_isolated);
+ continue;
+ }
+ if (remove_cpu(cpu)) {
+ pr_warn_once("cpuset: failed to offline CPU%d for
isolation\n",
+ cpu);
+ cpumask_clear_cpu(cpu, dhm_prev_isolated);
+ continue;
+ }
+ WARN_ON_ONCE(tick_nohz_cpu_isolate(cpu));
+ WARN_ON_ONCE(rcu_nocb_cpu_isolate(cpu));
+ WARN_ON_ONCE(add_cpu(cpu));
+ }
+
+ for_each_cpu(cpu, newly_deisolated) {
+ if (remove_cpu(cpu)) {
+ pr_warn_once("cpuset: failed to offline CPU%d for
de-isolation\n",
+ cpu);
+ cpumask_set_cpu(cpu, dhm_prev_isolated);
+ continue;
+ }
+ WARN_ON_ONCE(rcu_nocb_cpu_deoffload(cpu));
+ tick_nohz_cpu_deisolate(cpu);
+ WARN_ON_ONCE(add_cpu(cpu));
+ }
+
+ spin_lock(&dhm_cycling_lock);
+ cpumask_clear(dhm_cycling_cpus);
+ spin_unlock(&dhm_cycling_lock);
+
+ lockup_detector_hk_update();
+
+ free_cpumask_var(newly_isolated);
+ free_cpumask_var(newly_deisolated);
+}
+
/*
* cpuset_update_sd_hk_unlock - Rebuild sched domains, update HK & unlock
*
@@ -1386,6 +1476,13 @@ static void cpuset_update_sd_hk_unlock(void)
WARN_ON_ONCE(housekeeping_update_types(noise_types,
isolated_hk_cpus));
mutex_unlock(&cpuset_top_mutex);
+
+ /*
+ * All cpuset and hotplug locks are released. Cycle each
+ * affected CPU through hotplug to activate tick suppression,
+ * RCU callback offloading and managed-IRQ remapping.
+ */
+ dhm_cycle_isolated_cpus(isolated_hk_cpus);
} else {
cpuset_full_unlock();
}
@@ -3717,6 +3814,7 @@ int __init cpuset_init(void)
BUG_ON(!zalloc_cpumask_var(&isolated_cpus, GFP_KERNEL));
BUG_ON(!zalloc_cpumask_var(&isolated_hk_cpus, GFP_KERNEL));
BUG_ON(!zalloc_cpumask_var(&dhm_cycling_cpus, GFP_KERNEL));
+ BUG_ON(!zalloc_cpumask_var(&dhm_prev_isolated, GFP_KERNEL));
cpumask_setall(top_cpuset.cpus_allowed);
nodes_setall(top_cpuset.mems_allowed);
--
2.43.0
