From: Kaitao Cheng <[email protected]> Add a new kfunc bpf_list_add_impl(head, new, prev, meta, off) that inserts 'new' after 'prev' in the BPF linked list. Both must be in the same list; 'prev' must already be in the list. The new node must be an owning reference (e.g. from bpf_obj_new); the kfunc consumes that reference and the node becomes non-owning once inserted.
We have added an additional parameter bpf_list_head *head to bpf_list_add_impl, as the verifier requires the head parameter to check whether the lock is being held. Returns 0 on success, -EINVAL if 'prev' is not in a list or 'new' is already in a list (or duplicate insertion). On failure, the kernel drops the passed-in node. Signed-off-by: Kaitao Cheng <[email protected]> --- kernel/bpf/helpers.c | 14 ++++++++++++++ kernel/bpf/verifier.c | 12 ++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/kernel/bpf/helpers.c b/kernel/bpf/helpers.c index 669e380746a6..68c83a009275 100644 --- a/kernel/bpf/helpers.c +++ b/kernel/bpf/helpers.c @@ -2447,6 +2447,19 @@ __bpf_kfunc int bpf_list_push_back_impl(struct bpf_list_head *head, return __bpf_list_add(n, head, &h->prev, meta ? meta->record : NULL, off); } +__bpf_kfunc int bpf_list_add_impl(struct bpf_list_head *head, + struct bpf_list_node *new, + struct bpf_list_node *prev, + void *meta__ign, u64 off) +{ + struct bpf_list_node_kern *n = (void *)new, *p = (void *)prev; + struct btf_struct_meta *meta = meta__ign; + struct list_head *prev_ptr = &p->list_head; + + return __bpf_list_add(n, head, &prev_ptr, + meta ? meta->record : NULL, off); +} + static struct bpf_list_node *__bpf_list_del(struct bpf_list_head *head, struct list_head *n) { @@ -4585,6 +4598,7 @@ BTF_ID_FLAGS(func, bpf_list_pop_back, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_del, KF_ACQUIRE | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_front, KF_RET_NULL) BTF_ID_FLAGS(func, bpf_list_back, KF_RET_NULL) +BTF_ID_FLAGS(func, bpf_list_add_impl) BTF_ID_FLAGS(func, bpf_task_acquire, KF_ACQUIRE | KF_RCU | KF_RET_NULL) BTF_ID_FLAGS(func, bpf_task_release, KF_RELEASE) BTF_ID_FLAGS(func, bpf_rbtree_remove, KF_ACQUIRE | KF_RET_NULL) diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index ae8d1f2e32de..69dcf0105973 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -12478,6 +12478,7 @@ enum special_kfunc_type { KF_bpf_refcount_acquire_impl, KF_bpf_list_push_front_impl, KF_bpf_list_push_back_impl, + KF_bpf_list_add_impl, KF_bpf_list_pop_front, KF_bpf_list_pop_back, KF_bpf_list_del, @@ -12539,6 +12540,7 @@ BTF_ID(func, bpf_obj_drop_impl) BTF_ID(func, bpf_refcount_acquire_impl) BTF_ID(func, bpf_list_push_front_impl) BTF_ID(func, bpf_list_push_back_impl) +BTF_ID(func, bpf_list_add_impl) BTF_ID(func, bpf_list_pop_front) BTF_ID(func, bpf_list_pop_back) BTF_ID(func, bpf_list_del) @@ -12617,6 +12619,7 @@ BTF_ID(func, bpf_stream_print_stack) static const enum special_kfunc_type bpf_list_api_kfuncs[] = { KF_bpf_list_push_front_impl, KF_bpf_list_push_back_impl, + KF_bpf_list_add_impl, KF_bpf_list_pop_front, KF_bpf_list_pop_back, KF_bpf_list_del, @@ -12628,6 +12631,7 @@ static const enum special_kfunc_type bpf_list_api_kfuncs[] = { static const enum special_kfunc_type bpf_list_node_api_kfuncs[] = { KF_bpf_list_push_front_impl, KF_bpf_list_push_back_impl, + KF_bpf_list_add_impl, KF_bpf_list_del, }; @@ -12681,6 +12685,7 @@ static const enum special_kfunc_type bpf_stream_api_kfuncs[] = { static const enum special_kfunc_type bpf_collection_insert_kfuncs[] = { KF_bpf_list_push_front_impl, KF_bpf_list_push_back_impl, + KF_bpf_list_add_impl, KF_bpf_rbtree_add_impl, }; @@ -23354,8 +23359,11 @@ static int fixup_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, int struct_meta_reg = BPF_REG_3; int node_offset_reg = BPF_REG_4; - /* rbtree_add has extra 'less' arg, so args-to-fixup are in diff regs */ - if (desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { + /* list/rbtree_add_impl have an extra arg (prev/less), + * so args-to-fixup are in different regs. + */ + if (desc->func_id == special_kfunc_list[KF_bpf_list_add_impl] || + desc->func_id == special_kfunc_list[KF_bpf_rbtree_add_impl]) { struct_meta_reg = BPF_REG_4; node_offset_reg = BPF_REG_5; } -- 2.50.1 (Apple Git-155)
