With CONFIG_DEBUG_INFO_BTF=m the vmlinux BTF is loaded on first use. For that to save anything, nothing may pull it in at boot. The verifier no longer does since the previous patches; two things still do:
- register_btf_kfunc_id_set(), register_btf_id_dtor_kfuncs() and register_bpf_struct_ops() for vmlinux run from initcalls and need the parsed BTF. Queue them instead (btf_defer_reg()) and apply them in btf_parse_vmlinux(), before the BTF is published, so that no program can see a vmlinux BTF without its kfuncs and struct_ops. Applying a struct_ops runs its ->init(), which registers the kfunc sets of its hook; those land back on the queue, so it is drained in a loop until a pass adds nothing, and only then are new registrations applied directly. The dtor arrays are copied: every caller in the tree builds them on the stack of its initcall. - Module BTF is split BTF against the vmlinux BTF and used to be parsed in the module notifier. The notifier cannot load btf_vmlinux (that would nest a module load in a module load), so a module loaded before the vmlinux BTF keeps a copy of its .BTF and .BTF.base, is exposed in /sys/kernel/btf right away (the raw bytes need no parsing) and gets a list entry with btf == NULL. Its kfunc, dtor kfunc and struct_ops registrations wait on that entry. When the vmlinux BTF arrives, btf_parse_deferred_modules() parses the kept copies, gives them ids and applies the waiting registrations; the copy is the one btf_parse_module() makes anyway, so the sysfs file keeps pointing at valid data. Registrations are applied after dropping btf_module_mutex because they walk the module list (btf_check_kfunc_name()). A module whose BTF turns out to mismatch at that point is already running and keeps running without BTF, with a warning. With =y such a module would have been refused at load time unless CONFIG_MODULE_ALLOW_BTF_MISMATCH; that check only applies to modules loaded after the vmlinux BTF. Walkers of the module BTF list skip entries whose BTF is not parsed yet. With =y the BTF is present from boot, so the queues never fill and the notifier takes the existing path. Still nothing is reachable until the Kconfig symbol becomes a tristate. Signed-off-by: Jay Wang <[email protected]> --- include/linux/btf.h | 5 + kernel/bpf/btf.c | 426 +++++++++++++++++++++++++++++++++++++++++- kernel/bpf/verifier.c | 9 +- 3 files changed, 433 insertions(+), 7 deletions(-) diff --git a/include/linux/btf.h b/include/linux/btf.h index 0bf10811fe53..3b99d6386dec 100644 --- a/include/linux/btf.h +++ b/include/linux/btf.h @@ -583,6 +583,11 @@ const char *btf_str_by_offset(const struct btf *btf, u32 offset); struct btf *btf_parse_vmlinux(void); void *btf_vmlinux_data(u32 *size, bool load); u32 btf_vmlinux_size(void); +#if IS_MODULE(CONFIG_DEBUG_INFO_BTF) +void btf_parse_deferred_modules(void); +#else +static inline void btf_parse_deferred_modules(void) {} +#endif struct btf *bpf_prog_get_target_btf(const struct bpf_prog *prog); u32 *btf_kfunc_flags(const struct btf *btf, u32 kfunc_btf_id, const struct bpf_prog *prog); int btf_kfunc_check_flag(const struct btf *btf, u32 kfunc_btf_id, u32 flag); diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 50eb7a95fd82..cbba20a908e9 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -6551,6 +6551,8 @@ static struct btf *btf_parse_base(struct btf_verifier_env *env, const char *name return ERR_PTR(err); } +static void btf_apply_deferred_vmlinux_regs(struct btf *btf); + struct btf *btf_parse_vmlinux(void) { struct btf_verifier_env *env = NULL; @@ -6581,7 +6583,15 @@ struct btf *btf_parse_vmlinux(void) if (err) { btf_free(btf); btf = ERR_PTR(err); + goto err_out; } + + /* + * With CONFIG_DEBUG_INFO_BTF=m, kfunc, dtor kfunc and struct_ops + * registrations for vmlinux made before the BTF was available were + * queued; apply them now, before the BTF becomes visible to anyone. + */ + btf_apply_deferred_vmlinux_regs(btf); err_out: btf_verifier_env_free(env); return btf; @@ -8697,13 +8707,62 @@ enum { #define BTF_MODULE_NOTIFIER 1 #endif +/* + * CONFIG_DEBUG_INFO_BTF=m: a kfunc, dtor kfunc or struct_ops registration + * made while the BTF it applies to is not available yet. Kept until the BTF + * arrives, see btf_defer_reg() and btf_apply_deferred_regs(). + */ +enum btf_deferred_reg_kind { + BTF_DEFERRED_KFUNC_SET, + BTF_DEFERRED_DTOR_KFUNCS, + BTF_DEFERRED_STRUCT_OPS, +}; + +struct btf_deferred_reg { + struct list_head list; + enum btf_deferred_reg_kind kind; + /* set while the registration is queued for a module BTF */ + struct btf *btf; + struct module *module; + union { + struct { + enum btf_kfunc_hook hook; + const struct btf_kfunc_id_set *kset; + } kfunc; + struct { + const struct btf_id_dtor_kfunc *dtors; + u32 cnt; + } dtor; + struct bpf_struct_ops *st_ops; + }; +}; + #ifdef BTF_MODULE_NOTIFIER +static void btf_free_deferred_regs(struct list_head *regs); +#if IS_MODULE(CONFIG_DEBUG_INFO_BTF) +static void btf_free_deferred_reg(struct btf_deferred_reg *reg); +static void btf_apply_deferred_regs(struct list_head *regs); +#endif + struct btf_module { struct list_head list; struct module *module; struct btf *btf; struct bin_attribute *sysfs_attr; int flags; + /* + * CONFIG_DEBUG_INFO_BTF=m: a module loaded before the vmlinux BTF is + * available cannot have its BTF parsed yet. Its .BTF and .BTF.base + * sections are copied here and parsed once the vmlinux BTF arrives + * (btf_parse_deferred_modules()); @btf is NULL until then. + * Registrations of the module's kfuncs, dtor kfuncs and struct_ops + * wait in @deferred_regs. + */ + void *data; + void *base_data; + u32 data_size; + u32 base_data_size; + struct list_head deferred_regs; }; static LIST_HEAD(btf_modules); @@ -8747,8 +8806,14 @@ static void btf_module_free(struct btf_module *btf_mod) { if (btf_mod->sysfs_attr) sysfs_remove_bin_file(btf_kobj, btf_mod->sysfs_attr); - purge_cand_cache(btf_mod->btf); - btf_put(btf_mod->btf); + if (btf_mod->btf) { + purge_cand_cache(btf_mod->btf); + btf_put(btf_mod->btf); + } else { + kvfree(btf_mod->data); + kvfree(btf_mod->base_data); + } + btf_free_deferred_regs(&btf_mod->deferred_regs); kfree(btf_mod->sysfs_attr); kfree(btf_mod); } @@ -8792,11 +8857,55 @@ static int btf_vmlinux_module_coming(struct module *mod) smp_store_release(&btf_vmlinux_raw, data); return 0; } + +/* + * The vmlinux BTF is not available yet and must not be loaded from the + * module notifier (that would nest a module load into a module load). Keep + * the module's BTF for btf_parse_deferred_modules(). The .BTF data can be + * exposed in sysfs right away, it needs no parsing. + */ +static int btf_module_defer(struct btf_module *btf_mod, struct module *mod) +{ + int err; + + btf_mod->data = kvmemdup(mod->btf_data, mod->btf_data_size, + GFP_KERNEL | __GFP_NOWARN); + if (!btf_mod->data) + return -ENOMEM; + btf_mod->data_size = mod->btf_data_size; + + if (mod->btf_base_data) { + btf_mod->base_data = kvmemdup(mod->btf_base_data, + mod->btf_base_data_size, + GFP_KERNEL | __GFP_NOWARN); + if (!btf_mod->base_data) { + kvfree(btf_mod->data); + return -ENOMEM; + } + btf_mod->base_data_size = mod->btf_base_data_size; + } + + err = btf_module_sysfs_add(btf_mod, mod->name, btf_mod->data, + btf_mod->data_size); + if (err) { + kvfree(btf_mod->data); + kvfree(btf_mod->base_data); + return err; + } + + list_add(&btf_mod->list, &btf_modules); + return 0; +} #else static int btf_vmlinux_module_coming(struct module *mod) { return 0; } + +static int btf_module_defer(struct btf_module *btf_mod, struct module *mod) +{ + return 0; +} #endif static int btf_module_notify(struct notifier_block *nb, unsigned long op, @@ -8828,6 +8937,24 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, goto out; } btf_mod->module = module; + INIT_LIST_HEAD(&btf_mod->deferred_regs); + + if (IS_MODULE(CONFIG_DEBUG_INFO_BTF)) { + mutex_lock(&btf_module_mutex); + /* Pairs with the publication in bpf_get_btf_vmlinux() */ + if (!smp_load_acquire(&btf_vmlinux)) { + err = btf_module_defer(btf_mod, mod); + mutex_unlock(&btf_module_mutex); + if (err) { + pr_warn("failed to keep module [%s] BTF: %d\n", + mod->name, err); + kfree(btf_mod); + err = 0; + } + goto out; + } + mutex_unlock(&btf_module_mutex); + } btf = btf_parse_module(mod->name, bpf_get_btf_vmlinux(), mod->btf_data, mod->btf_data_size, false, @@ -8882,7 +9009,8 @@ static int btf_module_notify(struct notifier_block *nb, unsigned long op, * btf_try_get_module() on such BTFs will fail. This may * be called again on btf_put(), but it's ok to do so. */ - btf_free_id(btf_mod->btf); + if (btf_mod->btf) + btf_free_id(btf_mod->btf); list_del(&btf_mod->list); btf_module_free(btf_mod); break; @@ -8905,6 +9033,87 @@ static int __init btf_module_init(void) } fs_initcall(btf_module_init); + +#if IS_MODULE(CONFIG_DEBUG_INFO_BTF) +/* + * CONFIG_DEBUG_INFO_BTF=m: the vmlinux BTF has just become available. Parse + * the BTF of the modules that were loaded before it, and apply the + * registrations that waited for them. Called from bpf_get_btf_vmlinux() + * once btf_vmlinux is published, with no locks held. + */ +void btf_parse_deferred_modules(void) +{ + /* Pairs with the publication in bpf_get_btf_vmlinux() */ + struct btf *vmlinux_btf = smp_load_acquire(&btf_vmlinux); + struct btf_deferred_reg *reg, *rtmp; + struct btf_module *btf_mod, *tmp; + bool parsed = false; + LIST_HEAD(regs); + struct btf *btf; + int err; + + if (IS_ERR_OR_NULL(vmlinux_btf)) + return; + + mutex_lock(&btf_module_mutex); + list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { + if (btf_mod->btf) + continue; + + btf = btf_parse_module(btf_mod->module->name, vmlinux_btf, + btf_mod->data, btf_mod->data_size, true, + btf_mod->base_data, btf_mod->base_data_size); + err = PTR_ERR_OR_ZERO(btf); + if (!err) { + err = btf_alloc_id(btf); + if (err) { + /* btf owns the data now, btf_free() drops it */ + btf_mod->data = NULL; + btf_free(btf); + } + } + if (err) { + /* + * The module is loaded and stays. Unlike at load time + * there is no way to reject it, so drop its BTF. + */ + pr_warn("failed to validate module [%s] BTF: %d\n", + btf_mod->module->name, err); + list_del(&btf_mod->list); + btf_module_free(btf_mod); + continue; + } + + /* btf->data is btf_mod->data now, the sysfs file keeps working */ + btf_mod->data = NULL; + kvfree(btf_mod->base_data); + btf_mod->base_data = NULL; + btf_mod->btf = btf; + parsed = true; + + /* + * Registrations are applied after dropping the mutex (they + * walk btf_modules); pin what they need until then. + */ + list_for_each_entry_safe(reg, rtmp, &btf_mod->deferred_regs, list) { + list_del(®->list); + if (!try_module_get(btf_mod->module)) { + btf_free_deferred_reg(reg); + continue; + } + btf_get(btf); + reg->btf = btf; + reg->module = btf_mod->module; + list_add_tail(®->list, ®s); + } + } + mutex_unlock(&btf_module_mutex); + + if (parsed) + purge_cand_cache(NULL); + btf_apply_deferred_regs(®s); +} +#endif /* IS_MODULE(CONFIG_DEBUG_INFO_BTF) */ #endif /* BTF_MODULE_NOTIFIER */ struct module *btf_try_get_module(const struct btf *btf) @@ -8957,8 +9166,11 @@ struct btf *btf_get_module_btf(const struct module *module) if (btf_mod->module != module) continue; - btf_get(btf_mod->btf); - btf = btf_mod->btf; + /* NULL while waiting for the vmlinux BTF (CONFIG_DEBUG_INFO_BTF=m) */ + if (btf_mod->btf) { + btf_get(btf_mod->btf); + btf = btf_mod->btf; + } break; } mutex_unlock(&btf_module_mutex); @@ -9135,7 +9347,8 @@ static int btf_check_kfunc_name(struct btf *btf, const char *func_name, u32 kind #ifdef CONFIG_DEBUG_INFO_BTF_MODULES guard(mutex)(&btf_module_mutex); list_for_each_entry_safe(btf_mod, tmp, &btf_modules, list) { - if (btf_mod->btf == btf) + /* skip ourselves and, with CONFIG_DEBUG_INFO_BTF=m, unparsed BTF */ + if (btf_mod->btf == btf || !btf_mod->btf) continue; id = btf_find_by_name_kind(btf_mod->btf, func_name, kind); if (id >= 0) { @@ -9475,12 +9688,22 @@ static int btf_kfunc_id_set_add(struct btf *btf, enum btf_kfunc_hook hook, return btf_populate_kfunc_set(btf, hook, kset); } +static int btf_defer_reg(struct module *owner, const struct btf_deferred_reg *tmpl); + static int __register_btf_kfunc_id_set(enum btf_kfunc_hook hook, const struct btf_kfunc_id_set *kset) { + struct btf_deferred_reg tmpl = { + .kind = BTF_DEFERRED_KFUNC_SET, + .kfunc = { .hook = hook, .kset = kset }, + }; struct btf *btf; int ret; + ret = btf_defer_reg(kset->owner, &tmpl); + if (ret) + return ret > 0 ? 0 : ret; + btf = btf_get_module_btf(kset->owner); if (!btf) return check_btf_kconfigs(kset->owner, "kfunc"); @@ -9649,9 +9872,17 @@ static int btf_dtor_kfuncs_add(struct btf *btf, const struct btf_id_dtor_kfunc * int register_btf_id_dtor_kfuncs(const struct btf_id_dtor_kfunc *dtors, u32 add_cnt, struct module *owner) { + struct btf_deferred_reg tmpl = { + .kind = BTF_DEFERRED_DTOR_KFUNCS, + .dtor = { .dtors = dtors, .cnt = add_cnt }, + }; struct btf *btf; int ret; + ret = btf_defer_reg(owner, &tmpl); + if (ret) + return ret > 0 ? 0 : ret; + btf = btf_get_module_btf(owner); if (!btf) return check_btf_kconfigs(owner, "dtor kfuncs"); @@ -10321,9 +10552,17 @@ static int btf_struct_ops_add(struct btf *btf, struct bpf_struct_ops *st_ops) int __register_bpf_struct_ops(struct bpf_struct_ops *st_ops) { + struct btf_deferred_reg tmpl = { + .kind = BTF_DEFERRED_STRUCT_OPS, + .st_ops = st_ops, + }; struct btf *btf; int err; + err = btf_defer_reg(st_ops->owner, &tmpl); + if (err) + return err > 0 ? 0 : err; + btf = btf_get_module_btf(st_ops->owner); if (!btf) return check_btf_kconfigs(st_ops->owner, "struct_ops"); @@ -10335,8 +10574,183 @@ int __register_bpf_struct_ops(struct bpf_struct_ops *st_ops) return err; } EXPORT_SYMBOL_GPL(__register_bpf_struct_ops); +#else +static int btf_struct_ops_add(struct btf *btf, struct bpf_struct_ops *st_ops) +{ + return -EOPNOTSUPP; +} +#endif + +/* + * CONFIG_DEBUG_INFO_BTF=m: registrations made before the BTF they apply to + * is available. Registrations for vmlinux wait in btf_vmlinux_deferred_regs + * until btf_parse_vmlinux() applies them; registrations for a module wait in + * its struct btf_module until btf_parse_deferred_modules() does. Both lists + * are protected by btf_module_mutex. + */ +#ifdef BTF_MODULE_NOTIFIER +static LIST_HEAD(btf_vmlinux_deferred_regs); +/* Set when the vmlinux BTF is parsed; new registrations apply directly */ +static bool btf_vmlinux_regs_closed; + +/* + * Queue @tmpl if the BTF for @owner is not available yet. Returns 1 if the + * registration was queued and is to be considered done, 0 if the caller has + * to apply it, or -ENOMEM. + */ +static int btf_defer_reg(struct module *owner, const struct btf_deferred_reg *tmpl) +{ + struct list_head *head = NULL; + struct btf_deferred_reg *reg; + struct btf_module *btf_mod; + + if (!IS_MODULE(CONFIG_DEBUG_INFO_BTF)) + return 0; + + guard(mutex)(&btf_module_mutex); + if (!owner) { + if (!btf_vmlinux_regs_closed) + head = &btf_vmlinux_deferred_regs; + } else { + list_for_each_entry(btf_mod, &btf_modules, list) { + if (btf_mod->module != owner) + continue; + if (!btf_mod->btf) + head = &btf_mod->deferred_regs; + break; + } + } + if (!head) + return 0; + + reg = kmemdup(tmpl, sizeof(*reg), GFP_KERNEL); + if (!reg) + return -ENOMEM; + /* + * kfunc id sets and struct_ops are static data of their owner, but + * the dtor arrays are commonly built on the stack of the initcall. + */ + if (reg->kind == BTF_DEFERRED_DTOR_KFUNCS) { + reg->dtor.dtors = kmemdup_array(tmpl->dtor.dtors, tmpl->dtor.cnt, + sizeof(*tmpl->dtor.dtors), GFP_KERNEL); + if (!reg->dtor.dtors) { + kfree(reg); + return -ENOMEM; + } + } + list_add_tail(®->list, head); + return 1; +} + +static void btf_free_deferred_reg(struct btf_deferred_reg *reg) +{ + if (reg->kind == BTF_DEFERRED_DTOR_KFUNCS) + kfree(reg->dtor.dtors); + kfree(reg); +} + +static const char *btf_deferred_reg_name(const struct btf_deferred_reg *reg) +{ + switch (reg->kind) { + case BTF_DEFERRED_KFUNC_SET: return "kfunc set"; + case BTF_DEFERRED_DTOR_KFUNCS: return "dtor kfuncs"; + case BTF_DEFERRED_STRUCT_OPS: return "struct_ops"; + } + return "?"; +} + +static int btf_apply_deferred_reg(struct btf *btf, const struct btf_deferred_reg *reg) +{ + switch (reg->kind) { + case BTF_DEFERRED_KFUNC_SET: + return btf_kfunc_id_set_add(btf, reg->kfunc.hook, reg->kfunc.kset); + case BTF_DEFERRED_DTOR_KFUNCS: + return btf_dtor_kfuncs_add(btf, reg->dtor.dtors, reg->dtor.cnt); + case BTF_DEFERRED_STRUCT_OPS: + return btf_struct_ops_add(btf, reg->st_ops); + } + return -EINVAL; +} + +#if IS_MODULE(CONFIG_DEBUG_INFO_BTF) +/* Apply and free the registrations in @regs; each is pinned to its BTF and module. */ +static void btf_apply_deferred_regs(struct list_head *regs) +{ + struct btf_deferred_reg *reg, *tmp; + int err; + + list_for_each_entry_safe(reg, tmp, regs, list) { + err = btf_apply_deferred_reg(reg->btf, reg); + if (err) + pr_warn("failed to register deferred %s for module [%s] BTF: %d\n", + btf_deferred_reg_name(reg), reg->btf->name, err); + btf_put(reg->btf); + module_put(reg->module); + list_del(®->list); + btf_free_deferred_reg(reg); + } +} #endif +static void btf_free_deferred_regs(struct list_head *regs) +{ + struct btf_deferred_reg *reg, *tmp; + + list_for_each_entry_safe(reg, tmp, regs, list) { + list_del(®->list); + btf_free_deferred_reg(reg); + } +} + +/* + * The vmlinux BTF has just been parsed; apply the registrations that waited + * for it. Runs under btf_vmlinux_lock, before @btf is published, so nothing + * can observe a vmlinux BTF without its kfuncs and struct_ops. + * + * Applying a registration can queue further ones: a struct_ops ->init() + * registers the kfuncs of its hook. Those must not go through + * bpf_get_btf_vmlinux() (we hold its lock), so the queue stays open until + * a pass applies nothing new, and only then are registrations applied directly. + */ +static void btf_apply_deferred_vmlinux_regs(struct btf *btf) +{ + struct btf_deferred_reg *reg, *tmp; + LIST_HEAD(regs); + int err; + + if (!IS_MODULE(CONFIG_DEBUG_INFO_BTF)) + return; + + mutex_lock(&btf_module_mutex); + while (!list_empty(&btf_vmlinux_deferred_regs)) { + list_splice_init(&btf_vmlinux_deferred_regs, ®s); + mutex_unlock(&btf_module_mutex); + + list_for_each_entry_safe(reg, tmp, ®s, list) { + err = btf_apply_deferred_reg(btf, reg); + if (err) + pr_warn("failed to register deferred %s for vmlinux BTF: %d\n", + btf_deferred_reg_name(reg), err); + list_del(®->list); + btf_free_deferred_reg(reg); + } + + mutex_lock(&btf_module_mutex); + } + btf_vmlinux_regs_closed = true; + mutex_unlock(&btf_module_mutex); +} +#else +static int btf_defer_reg(struct module *owner, const struct btf_deferred_reg *tmpl) +{ + return 0; +} + +static void btf_apply_deferred_vmlinux_regs(struct btf *btf) +{ +} +#endif /* BTF_MODULE_NOTIFIER */ + bool btf_param_match_suffix(const struct btf *btf, const struct btf_param *arg, const char *suffix) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index a7b73bc146a8..b6f094d5306a 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -21160,12 +21160,14 @@ int bpf_check_attach_btf_id_multi(struct btf *btf, struct bpf_prog *prog, u32 bt /* * Returns the parsed vmlinux BTF, NULL if the kernel has none, or an ERR_PTR * if it is malformed. With CONFIG_DEBUG_INFO_BTF=m the BTF lives in the - * btf_vmlinux module; the first caller loads it and parses it. May sleep. + * btf_vmlinux module; the first caller loads it, parses it and then registers + * the BTF of the modules that were loaded before it. May sleep. */ struct btf *bpf_get_btf_vmlinux(void) { /* Pairs with the smp_store_release() on the parse path below. */ struct btf *btf = smp_load_acquire(&btf_vmlinux); + bool parsed = false; u32 size; if (btf || !IS_ENABLED(CONFIG_DEBUG_INFO_BTF)) @@ -21190,8 +21192,13 @@ struct btf *bpf_get_btf_vmlinux(void) * on the lockless fast path above. */ smp_store_release(&btf_vmlinux, btf); + parsed = true; } mutex_unlock(&btf_vmlinux_lock); + + if (parsed && IS_MODULE(CONFIG_DEBUG_INFO_BTF)) + btf_parse_deferred_modules(); + return btf; } -- 2.47.3
