Clean up a variety of synchronization issues and related bandaids by having a per-cpu mutex that guards changes to kthread, and fd open/close/revoke.
Replace the SIGKILL hack for userspace timerlat threads (that doesn't even work, because we don't wait for the process to actually die) with a mutex-protected detachment mechanism. The mutex should be uncontended during normal timerlat_fd_read() usage. Signed-off-by: Crystal Wood <[email protected]> --- kernel/trace/trace_osnoise.c | 239 +++++++++++++++++++++-------------- 1 file changed, 144 insertions(+), 95 deletions(-) diff --git a/kernel/trace/trace_osnoise.c b/kernel/trace/trace_osnoise.c index 0e1265acd1cc..e2e1ef3f5a61 100644 --- a/kernel/trace/trace_osnoise.c +++ b/kernel/trace/trace_osnoise.c @@ -222,19 +222,30 @@ struct osn_thread { u64 delta_start; }; +struct fd_data { + struct task_struct *thread; + void (*detach)(struct fd_data *fdd); + int cpu; +}; + /* * Runtime information: this structure saves the runtime information used by * one sampling thread. */ struct osnoise_variables { + struct mutex lock; /* covers kthread and fdd changes */ struct task_struct *kthread; - bool sampling; - pid_t pid; - struct osn_nmi nmi; - struct osn_irq irq; - struct osn_softirq softirq; - struct osn_thread thread; - local_t int_counter; + struct fd_data *fdd; + + struct_group(zero, + bool sampling; + pid_t pid; + struct osn_nmi nmi; + struct osn_irq irq; + struct osn_softirq softirq; + struct osn_thread thread; + local_t int_counter; + ); }; /* @@ -250,6 +261,11 @@ static inline struct osnoise_variables *this_cpu_osn_var(void) return this_cpu_ptr(&per_cpu_osnoise_var); } +static inline struct osnoise_variables *cpu_osn_var(int cpu) +{ + return per_cpu_ptr(&per_cpu_osnoise_var, cpu); +} + /* * Protect the interface. */ @@ -292,9 +308,6 @@ static inline void tlat_var_reset(void) struct timerlat_variables *tlat_var; int cpu; - /* Synchronize with the timerlat interfaces */ - mutex_lock(&interface_lock); - /* * So far, all the values are initialized as 0, so * zeroing the structure is perfect. @@ -310,8 +323,6 @@ static inline void tlat_var_reset(void) * thread that wakes up, if TIMERLAT_ALIGN is set. */ atomic64_set(&align_next, 0); - - mutex_unlock(&interface_lock); } #else /* CONFIG_TIMERLAT_TRACER */ #define tlat_var_reset() do {} while (0) @@ -330,8 +341,11 @@ static inline void osn_var_reset(void) * zeroing the structure is perfect. */ for_each_online_cpu(cpu) { - osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu); - memset(osn_var, 0, sizeof(*osn_var)); + osn_var = cpu_osn_var(cpu); + + WARN_ON(osn_var->kthread); + WARN_ON(osn_var->fdd); + memset(&osn_var->zero, 0, sizeof(osn_var->zero)); } } @@ -1673,6 +1687,8 @@ static void osnoise_sleep(bool skip_period) */ static inline int osnoise_migration_pending(void) { + struct osnoise_variables *osn = this_cpu_osn_var(); + if (!current->migration_pending) return 0; @@ -1688,10 +1704,10 @@ static inline int osnoise_migration_pending(void) * The tracers are responsible for cleaning their env before * exiting. */ - mutex_lock(&interface_lock); + mutex_lock(&osn->lock); this_cpu_osn_var()->kthread = NULL; cpumask_clear_cpu(smp_processor_id(), &kthread_cpumask); - mutex_unlock(&interface_lock); + mutex_unlock(&osn->lock); return 1; } @@ -1978,32 +1994,22 @@ static int timerlat_main(void *data) */ static void stop_kthread(unsigned int cpu) { - struct task_struct *kthread; + struct osnoise_variables *osn_var = cpu_osn_var(cpu); - kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL); - if (kthread) { - if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask) && - !WARN_ON(!test_bit(OSN_WORKLOAD, &osnoise_options))) { - kthread_stop(kthread); - } else if (!WARN_ON(test_bit(OSN_WORKLOAD, &osnoise_options))) { - /* - * This is a user thread waiting on the timerlat_fd. We need - * to close all users, and the best way to guarantee this is - * by killing the thread. NOTE: this is a purpose specific file. - */ - kill_pid(kthread->thread_pid, SIGKILL, 1); - put_task_struct(kthread); - } - } else { - /* if no workload, just return */ - if (!test_bit(OSN_WORKLOAD, &osnoise_options)) { - /* - * This is set in the osnoise tracer case. - */ - per_cpu(per_cpu_osnoise_var, cpu).sampling = false; - barrier(); - } + mutex_lock(&osn_var->lock); + + if (osn_var->fdd) { + WARN_ON(test_bit(OSN_WORKLOAD, &osnoise_options)); + WARN_ON(osn_var->kthread); + osn_var->fdd->detach(osn_var->fdd); + } else if (osn_var->kthread) { + WARN_ON(!cpumask_test_and_clear_cpu(cpu, &kthread_cpumask)); + WARN_ON(!test_bit(OSN_WORKLOAD, &osnoise_options)); + kthread_stop(osn_var->kthread); + osn_var->kthread = NULL; } + + mutex_unlock(&osn_var->lock); } /* @@ -2029,13 +2035,18 @@ static void stop_per_cpu_kthreads(void) */ static int start_kthread(unsigned int cpu) { + struct osnoise_variables *osn = cpu_osn_var(cpu); struct task_struct *kthread; void *main = osnoise_main; char comm[24]; + int ret = 0; + + lockdep_assert_held(&trace_types_lock); + mutex_lock(&osn->lock); /* Do not start a new thread if it is already running */ - if (per_cpu(per_cpu_osnoise_var, cpu).kthread) - return 0; + if (osn->kthread) + goto out; if (timerlat_enabled()) { snprintf(comm, 24, "timerlat/%d", cpu); @@ -2045,7 +2056,7 @@ static int start_kthread(unsigned int cpu) if (!test_bit(OSN_WORKLOAD, &osnoise_options)) { per_cpu(per_cpu_osnoise_var, cpu).sampling = true; barrier(); - return 0; + goto out; } snprintf(comm, 24, "osnoise/%d", cpu); } @@ -2054,13 +2065,16 @@ static int start_kthread(unsigned int cpu) if (IS_ERR(kthread)) { pr_err(BANNER "could not start sampling thread\n"); - return -ENOMEM; + ret = -ENOMEM; + goto out; } - per_cpu(per_cpu_osnoise_var, cpu).kthread = kthread; + osn->kthread = kthread; cpumask_set_cpu(cpu, &kthread_cpumask); - return 0; +out: + mutex_unlock(&osn->lock); + return ret; } /* @@ -2086,15 +2100,9 @@ static int start_per_cpu_kthreads(void) */ cpumask_and(current_mask, cpu_online_mask, &osnoise_cpumask); - for_each_possible_cpu(cpu) { - if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask)) { - struct task_struct *kthread; - - kthread = xchg_relaxed(&(per_cpu(per_cpu_osnoise_var, cpu).kthread), NULL); - if (!WARN_ON(!kthread)) - kthread_stop(kthread); - } - } + for_each_possible_cpu(cpu) + if (cpumask_test_and_clear_cpu(cpu, &kthread_cpumask)) + stop_kthread(cpu); for_each_cpu(cpu, current_mask) { retval = start_kthread(cpu); @@ -2417,11 +2425,37 @@ osnoise_cpus_write(struct file *filp, const char __user *ubuf, size_t count, } #ifdef CONFIG_TIMERLAT_TRACER +static void timerlat_fd_detach(struct fd_data *fdd) +{ + struct osnoise_variables *osn_var; + struct timerlat_variables *tlat_var; + + osn_var = cpu_osn_var(fdd->cpu); + tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, fdd->cpu); + + lockdep_assert_held(&osn_var->lock); + if (WARN_ON(fdd != osn_var->fdd)) + return; + + WARN_ON(osn_var->kthread); + + hrtimer_cancel(&tlat_var->timer); + memset(tlat_var, 0, sizeof(*tlat_var)); + + osn_var->sampling = 0; + osn_var->pid = 0; + + put_task_struct(fdd->thread); + osn_var->fdd = NULL; +} + static int timerlat_fd_open(struct inode *inode, struct file *file) { + struct fd_data *fdd; struct osnoise_variables *osn_var; struct timerlat_variables *tlat; long cpu = (long) inode->i_cdev; + int ret = 0; mutex_lock(&interface_lock); @@ -2437,16 +2471,15 @@ static int timerlat_fd_open(struct inode *inode, struct file *file) migrate_disable(); osn_var = this_cpu_osn_var(); + mutex_lock(&osn_var->lock); - /* - * The osn_var->pid holds the single access to this file. - */ - if (osn_var->pid) { - mutex_unlock(&interface_lock); - migrate_enable(); - return -EBUSY; + if (osn_var->fdd) { + ret = -EBUSY; + goto err; } + WARN_ON_ONCE(osn_var->kthread); + /* * timerlat tracer is a per-cpu tracer. Check if the user-space too * is pinned to a single CPU. The tracer laters monitor if the task @@ -2455,24 +2488,33 @@ static int timerlat_fd_open(struct inode *inode, struct file *file) * setup. */ if (current->nr_cpus_allowed > 1 || cpu != smp_processor_id()) { - mutex_unlock(&interface_lock); - migrate_enable(); - return -EPERM; + ret = -EPERM; + goto err; + } + + fdd = kmalloc(sizeof(*fdd), GFP_KERNEL); + if (!fdd) { + ret = -ENOMEM; + goto err; } /* * From now on, it is good to go. */ - file->private_data = inode->i_cdev; + fdd->thread = current; + fdd->detach = timerlat_fd_detach; + fdd->cpu = cpu; + file->private_data = fdd; get_task_struct(current); - osn_var->kthread = current; osn_var->pid = current->pid; + osn_var->fdd = fdd; /* * Setup is done. */ + mutex_unlock(&osn_var->lock); mutex_unlock(&interface_lock); tlat = this_cpu_tmr_var(); @@ -2482,6 +2524,12 @@ static int timerlat_fd_open(struct inode *inode, struct file *file) migrate_enable(); return 0; + +err: + mutex_unlock(&osn_var->lock); + mutex_unlock(&interface_lock); + migrate_enable(); + return ret; }; /* @@ -2497,12 +2545,13 @@ static ssize_t timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, loff_t *ppos) { - long cpu = (long) file->private_data; + struct fd_data *fdd = file->private_data; struct osnoise_variables *osn_var; struct timerlat_variables *tlat; struct timerlat_sample s; s64 diff; u64 now; + int ret = 0; migrate_disable(); @@ -2513,13 +2562,13 @@ timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, * we can do about it. * So, if the thread is running on another CPU, stop the machinery. */ - if (cpu == smp_processor_id()) { + if (fdd->cpu == smp_processor_id()) { if (tlat->uthread_migrate) { migrate_enable(); return -EINVAL; } } else { - per_cpu_ptr(&per_cpu_timerlat_var, cpu)->uthread_migrate = 1; + per_cpu_ptr(&per_cpu_timerlat_var, fdd->cpu)->uthread_migrate = 1; osnoise_taint("timerlat user thread migrate\n"); osnoise_stop_tracing(); migrate_enable(); @@ -2528,6 +2577,13 @@ timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, osn_var = this_cpu_osn_var(); + /* In normal usage, this should always be uncontended. */ + mutex_lock(&osn_var->lock); + if (fdd != osn_var->fdd || current != fdd->thread) { + ret = -EINVAL; + goto out; + } + /* * The timerlat in user-space runs in a different order: * the read() starts from the execution of the previous occurrence, @@ -2574,6 +2630,11 @@ timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, /* wait for the next period */ wait_next_period(tlat); + if (fdd != osn_var->fdd) { + ret = -EINVAL; + goto out; + } + /* This is the wakeup from this cycle */ now = ktime_to_ns(hrtimer_cb_get_time(&tlat->timer)); diff = now - tlat->abs_period; @@ -2599,39 +2660,24 @@ timerlat_fd_read(struct file *file, char __user *ubuf, size_t count, } out: + mutex_unlock(&osn_var->lock); migrate_enable(); - return 0; + return ret; } static int timerlat_fd_release(struct inode *inode, struct file *file) { + struct fd_data *fdd = file->private_data; struct osnoise_variables *osn_var; - struct timerlat_variables *tlat_var; - long cpu = (long) file->private_data; - - migrate_disable(); - mutex_lock(&interface_lock); - - osn_var = per_cpu_ptr(&per_cpu_osnoise_var, cpu); - tlat_var = per_cpu_ptr(&per_cpu_timerlat_var, cpu); - if (tlat_var->kthread) - hrtimer_cancel(&tlat_var->timer); - memset(tlat_var, 0, sizeof(*tlat_var)); + osn_var = per_cpu_ptr(&per_cpu_osnoise_var, fdd->cpu); - osn_var->sampling = 0; - osn_var->pid = 0; + mutex_lock(&osn_var->lock); + if (fdd == osn_var->fdd) + timerlat_fd_detach(fdd); + mutex_unlock(&osn_var->lock); - /* - * We are leaving, not being stopped... see stop_kthread(); - */ - if (osn_var->kthread) { - put_task_struct(osn_var->kthread); - osn_var->kthread = NULL; - } - - mutex_unlock(&interface_lock); - migrate_enable(); + kfree(fdd); return 0; } #endif @@ -3157,10 +3203,13 @@ __init static int init_timerlat_tracer(void) __init static int init_osnoise_tracer(void) { - int ret; + int ret, cpu; mutex_init(&interface_lock); + for_each_online_cpu(cpu) + mutex_init(&cpu_osn_var(cpu)->lock); + cpumask_copy(&osnoise_cpumask, cpu_all_mask); ret = register_tracer(&osnoise_tracer); -- 2.54.0
