On context switch we are locking the vtime seqcount of the scheduling-out task twice:
* On vtime_task_switch_common(), when we flush the pending vtime through vtime_account_system() / vtime_account_idle() * On arch_vtime_task_switch() to reset the vtime state. This is pointless as these actions can be performed without the need to unlock/lock in the middle. The reason these steps are separated is to consolidate a very small amount of common code between CONFIG_VIRT_CPU_ACCOUNTING_GEN and CONFIG_VIRT_CPU_ACCOUNTING_NATIVE. Performance in this fast path is definetly a priority over artificial code factorization so split the task switch code between GEN and NATIVE and mutualize the parts than can run under a single seqcount locked block. Signed-off-by: Frederic Weisbecker <frede...@kernel.org> Cc: Yauheni Kaliuta <yauheni.kali...@redhat.com> Cc: Thomas Gleixner <t...@linutronix.de> Cc: Rik van Riel <r...@redhat.com> Cc: Peter Zijlstra <pet...@infradead.org> Cc: Wanpeng Li <wanpen...@tencent.com> Cc: Ingo Molnar <mi...@kernel.org> --- include/linux/vtime.h | 32 ++++++++++++++++---------------- kernel/sched/cputime.c | 38 +++++++++++++++++++++----------------- 2 files changed, 37 insertions(+), 33 deletions(-) diff --git a/include/linux/vtime.h b/include/linux/vtime.h index 2fd247f..d9160ab 100644 --- a/include/linux/vtime.h +++ b/include/linux/vtime.h @@ -14,8 +14,12 @@ struct task_struct; * vtime_accounting_cpu_enabled() definitions/declarations */ #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) + static inline bool vtime_accounting_cpu_enabled(void) { return true; } +extern void vtime_task_switch(struct task_struct *prev); + #elif defined(CONFIG_VIRT_CPU_ACCOUNTING_GEN) + /* * Checks if vtime is enabled on some CPU. Cputime readers want to be careful * in that case and compute the tickless cputime. @@ -36,33 +40,29 @@ static inline bool vtime_accounting_cpu_enabled(void) return false; } + +extern void vtime_task_switch_generic(struct task_struct *prev); + +static inline void vtime_task_switch(struct task_struct *prev) +{ + if (vtime_accounting_cpu_enabled()) + vtime_task_switch_generic(prev); +} + #else /* !CONFIG_VIRT_CPU_ACCOUNTING */ + static inline bool vtime_accounting_cpu_enabled(void) { return false; } -#endif +static inline void vtime_task_switch(struct task_struct *prev) { } +#endif /* * Common vtime APIs */ #ifdef CONFIG_VIRT_CPU_ACCOUNTING - -#ifdef __ARCH_HAS_VTIME_TASK_SWITCH -extern void vtime_task_switch(struct task_struct *prev); -#else -extern void vtime_common_task_switch(struct task_struct *prev); -static inline void vtime_task_switch(struct task_struct *prev) -{ - if (vtime_accounting_cpu_enabled()) - vtime_common_task_switch(prev); -} -#endif /* __ARCH_HAS_VTIME_TASK_SWITCH */ - extern void vtime_account_kernel(struct task_struct *tsk); extern void vtime_account_idle(struct task_struct *tsk); - #else /* !CONFIG_VIRT_CPU_ACCOUNTING */ - -static inline void vtime_task_switch(struct task_struct *prev) { } static inline void vtime_account_kernel(struct task_struct *tsk) { } #endif /* !CONFIG_VIRT_CPU_ACCOUNTING */ diff --git a/kernel/sched/cputime.c b/kernel/sched/cputime.c index 63f10ac..facc665 100644 --- a/kernel/sched/cputime.c +++ b/kernel/sched/cputime.c @@ -404,9 +404,10 @@ static inline void irqtime_account_process_tick(struct task_struct *p, int user_ /* * Use precise platform statistics if available: */ -#ifdef CONFIG_VIRT_CPU_ACCOUNTING +#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE + # ifndef __ARCH_HAS_VTIME_TASK_SWITCH -void vtime_common_task_switch(struct task_struct *prev) +void vtime_task_switch(struct task_struct *prev) { if (is_idle_task(prev)) vtime_account_idle(prev); @@ -417,10 +418,7 @@ void vtime_common_task_switch(struct task_struct *prev) arch_vtime_task_switch(prev); } # endif -#endif /* CONFIG_VIRT_CPU_ACCOUNTING */ - -#ifdef CONFIG_VIRT_CPU_ACCOUNTING_NATIVE /* * Archs that account the whole time spent in the idle task * (outside irq) as idle time can rely on this and just implement @@ -730,19 +728,25 @@ static void vtime_account_guest(struct task_struct *tsk, } } -void vtime_account_kernel(struct task_struct *tsk) +static void __vtime_account_kernel(struct task_struct *tsk, + struct vtime *vtime) { - struct vtime *vtime = &tsk->vtime; - - if (!vtime_delta(vtime)) - return; - - write_seqcount_begin(&vtime->seqcount); /* We might have scheduled out from guest path */ if (tsk->flags & PF_VCPU) vtime_account_guest(tsk, vtime); else vtime_account_system(tsk, vtime); +} + +void vtime_account_kernel(struct task_struct *tsk) +{ + struct vtime *vtime = &tsk->vtime; + + if (!vtime_delta(vtime)) + return; + + write_seqcount_begin(&vtime->seqcount); + __vtime_account_kernel(tsk, vtime); write_seqcount_end(&vtime->seqcount); } @@ -800,18 +804,18 @@ EXPORT_SYMBOL_GPL(vtime_guest_exit); void vtime_account_idle(struct task_struct *tsk) { - struct vtime *vtime = &tsk->vtime; - - write_seqcount_begin(&vtime->seqcount); account_idle_time(get_vtime_delta(&tsk->vtime)); - write_seqcount_end(&vtime->seqcount); } -void arch_vtime_task_switch(struct task_struct *prev) +void vtime_task_switch_generic(struct task_struct *prev) { struct vtime *vtime = &prev->vtime; write_seqcount_begin(&vtime->seqcount); + if (is_idle_task(prev)) + vtime_account_idle(prev); + else + __vtime_account_kernel(prev, vtime); vtime->state = VTIME_INACTIVE; write_seqcount_end(&vtime->seqcount); -- 2.7.4