On 09/01/19 22:59, Josh Poimboeuf wrote: > Static calls are a replacement for global function pointers. They use > code patching to allow direct calls to be used instead of indirect > calls. They give the flexibility of function pointers, but with > improved performance. This is especially important for cases where > retpolines would otherwise be used, as retpolines can significantly > impact performance. > > The concept and code are an extension of previous work done by Ard > Biesheuvel and Steven Rostedt: > > https://lkml.kernel.org/r/20181005081333.15018-1-ard.biesheu...@linaro.org > https://lkml.kernel.org/r/20181006015110.653946...@goodmis.org > > There are two implementations, depending on arch support: > > 1) out-of-line: patched trampolines (CONFIG_HAVE_STATIC_CALL) > 2) basic function pointers > > For more details, see the comments in include/linux/static_call.h. > > Signed-off-by: Josh Poimboeuf <jpoim...@redhat.com> > --- > arch/Kconfig | 3 + > include/linux/static_call.h | 135 ++++++++++++++++++++++++++++++ > include/linux/static_call_types.h | 13 +++ > 3 files changed, 151 insertions(+) > create mode 100644 include/linux/static_call.h > create mode 100644 include/linux/static_call_types.h > > diff --git a/arch/Kconfig b/arch/Kconfig > index 4cfb6de48f79..7e469a693da3 100644 > --- a/arch/Kconfig > +++ b/arch/Kconfig > @@ -885,6 +885,9 @@ config HAVE_ARCH_PREL32_RELOCATIONS > architectures, and don't require runtime relocation on relocatable > kernels. > > +config HAVE_STATIC_CALL > + bool > + > source "kernel/gcov/Kconfig" > > source "scripts/gcc-plugins/Kconfig" > diff --git a/include/linux/static_call.h b/include/linux/static_call.h > new file mode 100644 > index 000000000000..9e85c479cd11 > --- /dev/null > +++ b/include/linux/static_call.h > @@ -0,0 +1,135 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +#ifndef _LINUX_STATIC_CALL_H > +#define _LINUX_STATIC_CALL_H > + > +/* > + * Static call support > + * > + * Static calls use code patching to hard-code function pointers into direct > + * branch instructions. They give the flexibility of function pointers, but > + * with improved performance. This is especially important for cases where > + * retpolines would otherwise be used, as retpolines can significantly impact > + * performance. > + * > + * > + * API overview: > + * > + * DECLARE_STATIC_CALL(key, func); > + * DEFINE_STATIC_CALL(key, func); > + * static_call(key, args...); > + * static_call_update(key, func); > + * > + * > + * Usage example: > + * > + * # Start with the following functions (with identical prototypes): > + * int func_a(int arg1, int arg2); > + * int func_b(int arg1, int arg2); > + * > + * # Define a 'my_key' reference, associated with func_a() by default > + * DEFINE_STATIC_CALL(my_key, func_a); > + * > + * # Call func_a() > + * static_call(my_key, arg1, arg2); > + * > + * # Update 'my_key' to point to func_b() > + * static_call_update(my_key, func_b); > + * > + * # Call func_b() > + * static_call(my_key, arg1, arg2); > + * > + * > + * Implementation details: > + * > + * This requires some arch-specific code (CONFIG_HAVE_STATIC_CALL). > + * Otherwise basic indirect calls are used (with function pointers). > + * > + * Each static_call() site calls into a trampoline associated with the > key. > + * The trampoline has a direct branch to the default function. Updates > to a > + * key will modify the trampoline's branch destination. > + */ > + > +#include <linux/types.h> > +#include <linux/cpu.h> > +#include <linux/static_call_types.h> > + > +#ifdef CONFIG_HAVE_STATIC_CALL > +#include <asm/static_call.h> > +extern void arch_static_call_transform(void *site, void *tramp, void *func); > +#endif > + > + > +#define DECLARE_STATIC_CALL(key, func) > \ > + extern struct static_call_key key; \ > + extern typeof(func) STATIC_CALL_TRAMP(key) > + > + > +#if defined(CONFIG_HAVE_STATIC_CALL) > + > +struct static_call_key { > + void *func, *tramp; > +}; > + > +#define DEFINE_STATIC_CALL(key, _func) > \ > + DECLARE_STATIC_CALL(key, _func); \ > + struct static_call_key key = { \ > + .func = _func, \ > + .tramp = STATIC_CALL_TRAMP(key), \ > + }; \ > + ARCH_DEFINE_STATIC_CALL_TRAMP(key, _func) > + > +#define static_call(key, args...) STATIC_CALL_TRAMP(key)(args) > + > +static inline void __static_call_update(struct static_call_key *key, void > *func) > +{ > + cpus_read_lock(); > + WRITE_ONCE(key->func, func); > + arch_static_call_transform(NULL, key->tramp, func); > + cpus_read_unlock(); > +} > + > +#define static_call_update(key, func) > \ > +({ \ > + BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key))); \ > + __static_call_update(&key, func); \ > +}) > + > +#define EXPORT_STATIC_CALL(key) > \ > + EXPORT_SYMBOL(STATIC_CALL_TRAMP(key)) > + > +#define EXPORT_STATIC_CALL_GPL(key) \ > + EXPORT_SYMBOL_GPL(STATIC_CALL_TRAMP(key)) > + > + > +#else /* Generic implementation */ > + > +struct static_call_key { > + void *func; > +}; > + > +#define DEFINE_STATIC_CALL(key, _func) > \ > + DECLARE_STATIC_CALL(key, _func); \ > + struct static_call_key key = { \ > + .func = _func, \ > + } > + > +#define static_call(key, args...) \ > + ((typeof(STATIC_CALL_TRAMP(key))*)(key.func))(args) > + > +static inline void __static_call_update(struct static_call_key *key, void > *func) > +{ > + WRITE_ONCE(key->func, func); > +} > + > +#define static_call_update(key, func) > \ > +({ \ > + BUILD_BUG_ON(!__same_type(func, STATIC_CALL_TRAMP(key))); \ > + __static_call_update(&key, func); \ > +}) > + > +#define EXPORT_STATIC_CALL(key) EXPORT_SYMBOL(key) > +#define EXPORT_STATIC_CALL_GPL(key) EXPORT_SYMBOL_GPL(key) > + > +#endif /* CONFIG_HAVE_STATIC_CALL_INLINE */ CONFIG_HAVE_STATIC_CALL?
-Ed > + > +#endif /* _LINUX_STATIC_CALL_H */ > diff --git a/include/linux/static_call_types.h > b/include/linux/static_call_types.h > new file mode 100644 > index 000000000000..0baaf3f02476 > --- /dev/null > +++ b/include/linux/static_call_types.h > @@ -0,0 +1,13 @@ > +/* SPDX-License-Identifier: GPL-2.0 */ > +#ifndef _STATIC_CALL_TYPES_H > +#define _STATIC_CALL_TYPES_H > + > +#include <linux/stringify.h> > + > +#define STATIC_CALL_TRAMP_PREFIX ____static_call_tramp_ > +#define STATIC_CALL_TRAMP_PREFIX_STR __stringify(STATIC_CALL_TRAMP_PREFIX) > + > +#define STATIC_CALL_TRAMP(key) __PASTE(STATIC_CALL_TRAMP_PREFIX, key) > +#define STATIC_CALL_TRAMP_STR(key) __stringify(STATIC_CALL_TRAMP(key)) > + > +#endif /* _STATIC_CALL_TYPES_H */
