BPF programs designated as dynamically loaded can be loaded and
attached independently after the initial bpf_object loading and
attaching.
These programs can also be reloaded and reattached multiple times,
enabling more flexible management of a resident BPF program set.
A key motivation for this feature is to reduce load times for
utilities that include hundreds of BPF programs. When the selection
of a resident BPF program set cannot be determined at the time of
bpf_object loading and attaching, all BPF programs would otherwise
need to be marked as autoload, leading to unnecessary overhead.
This patch addresses that inefficiency.
Signed-off-by: Slava Imameev <slava.imam...@crowdstrike.com>
---
tools/lib/bpf/libbpf.c | 144 +++++++++++++++--
tools/lib/bpf/libbpf.h | 5 +-
tools/lib/bpf/libbpf.map | 2 +
.../selftests/bpf/prog_tests/dynamicload.c | 145 ++++++++++++++++++
.../selftests/bpf/prog_tests/load_type.c | 61 ++++++++
.../selftests/bpf/progs/test_dynamicload.c | 31 ++++
.../selftests/bpf/progs/test_load_type.c | 8 +
7 files changed, 385 insertions(+), 11 deletions(-)
create mode 100644 tools/testing/selftests/bpf/prog_tests/dynamicload.c
create mode 100644 tools/testing/selftests/bpf/progs/test_dynamicload.c
diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c
index 9af5c0b08b8b..731a4a09f865 100644
--- a/tools/lib/bpf/libbpf.c
+++ b/tools/lib/bpf/libbpf.c
@@ -689,6 +689,7 @@ struct bpf_object {
bool loaded;
bool has_subcalls;
bool has_rodata;
+ bool has_dynload_progs;
struct bpf_gen *gen_loader;
@@ -7551,13 +7552,15 @@ static int bpf_object_load_prog(struct bpf_object *obj,
struct bpf_program *prog
* custom log_buf is specified; if the program load fails, then we'll
* bump log_level to 1 and use either custom log_buf or we'll allocate
* our own and retry the load to get details on what failed
+ * A shared buffer cannot be used for dynamically loaded programs as
they
+ * can be loaded concurrently.
*/
if (log_level) {
if (prog->log_buf) {
log_buf = prog->log_buf;
log_buf_size = prog->log_size;
own_log_buf = false;
- } else if (obj->log_buf) {
+ } else if (obj->log_buf && prog->load_type !=
BPF_PROG_LOAD_TYPE_DYNAMIC) {
log_buf = obj->log_buf;
log_buf_size = obj->log_size;
own_log_buf = false;
@@ -7911,6 +7914,7 @@ bpf_object__load_progs(struct bpf_object *obj, int
log_level)
pr_debug("prog '%s': skipped auto-loading\n",
prog->name);
continue;
}
+
prog->log_level |= log_level;
if (obj->gen_loader)
@@ -8588,8 +8592,11 @@ static int bpf_object_load(struct bpf_object *obj, int
extra_log_level, const ch
err = bpf_gen__finish(obj->gen_loader,
obj->nr_programs, obj->nr_maps);
}
- /* clean up fd_array */
- zfree(&obj->fd_array);
+ /* The fd array is needed for dynamically loaded programs,
+ * so defer freeing it in that case to the end of the object lifetime.
+ */
+ if (!obj->has_dynload_progs || !obj->fd_array_cnt)
+ zfree(&obj->fd_array);
/* clean up module BTFs */
for (i = 0; i < obj->btf_module_cnt; i++) {
@@ -8597,11 +8604,17 @@ static int bpf_object_load(struct bpf_object *obj, int
extra_log_level, const ch
btf__free(obj->btf_modules[i].btf);
free(obj->btf_modules[i].name);
}
- free(obj->btf_modules);
+ obj->btf_module_cnt = 0;
+ zfree(&obj->btf_modules);
- /* clean up vmlinux BTF */
- btf__free(obj->btf_vmlinux);
- obj->btf_vmlinux = NULL;
+ /* The btf_vmlinux data is needed for dynamically loaded programs,
+ * so defer freeing it in that case to the end of the object lifetime.
+ */
+ if (!obj->has_dynload_progs) {
+ /* clean up vmlinux BTF */
+ btf__free(obj->btf_vmlinux);
+ obj->btf_vmlinux = NULL;
+ }
obj->loaded = true; /* doesn't matter if successfully or not */
@@ -9103,6 +9116,8 @@ void bpf_object__close(struct bpf_object *obj)
zfree(&obj->arena_data);
+ zfree(&obj->fd_array);
+
free(obj);
}
@@ -9230,8 +9245,16 @@ bool bpf_program__autoload(const struct bpf_program
*prog)
int bpf_program__set_autoload(struct bpf_program *prog, bool autoload)
{
- return bpf_program__set_load_type(prog,
- autoload ? BPF_PROG_LOAD_TYPE_AUTO :
BPF_PROG_LOAD_TYPE_DISABLED);
+ enum bpf_prog_load_type type = prog->load_type;
+
+ if (autoload)
+ type = BPF_PROG_LOAD_TYPE_AUTO;
+ else if (prog->load_type == BPF_PROG_LOAD_TYPE_AUTO)
+ type = BPF_PROG_LOAD_TYPE_DISABLED;
+ else
+ return 0; /* Otherwise, keep the current load type. */
+
+ return bpf_program__set_load_type(prog, type);
}
bool bpf_program__autoattach(const struct bpf_program *prog)
@@ -14086,12 +14109,67 @@ void bpf_object__destroy_skeleton(struct
bpf_object_skeleton *s)
free(s);
}
+static int bpf_program__set_dynamicload(struct bpf_program *prog)
+{
+ struct bpf_object *obj;
+ const char *attach_name;
+
+ obj = prog->obj;
+ if (!obj)
+ return libbpf_err(-EINVAL);
+
+ /* Dynamically loaded programs are not supported for gen_loader.
+ * This limitation exists because bpf_object_load_prog is not invoked
+ * for dynamically loaded programs, making them invisible to gen_loader.
+ * To ensure compatibility, bpf_program__set_dynamicload should not be
+ * called when gen_loader is used to generate a BPF object loader.
+ * The gen_loader implementation handles autoloaded programs and follows
+ * its own model for loading BPF programs. To pass a BPF program to
+ * gen_loader, set the program's load type to BPF_PROG_LOAD_TYPE_AUTO.
+ */
+ if (obj->gen_loader)
+ return libbpf_err(-ENOTSUP);
+
+ if (prog_is_subprog(obj, prog))
+ return libbpf_err(-EINVAL);
+
+ attach_name = strchr(prog->sec_name, '/');
+ if (!attach_name || strchr(attach_name, ':')) {
+ /* Dynamic loading is not supported if module's BTF
+ * data is required for a bpf program.
+ * The module's BTF data is required in the folowing cases:
+ * - If a BPF program is annotated with just SEC("fentry")
+ * (or similar) without declaratively specifying
+ * target, then it is expected that target will be
+ * specified with bpf_program__set_attach_target() at
+ * runtime before BPF object load step. The module's
+ * BTF data will be required by libbpf_prepare_prog_load and
+ * libbpf_find_attach_btf_id.
+ * - The attach name is prepended with a module name.
+ */
+ return libbpf_err(-EINVAL);
+ }
+
+ obj->has_dynload_progs = true;
+ prog->load_type = BPF_PROG_LOAD_TYPE_DYNAMIC;
+ prog->autoattach = false;
+
+ return 0;
+}
+
int bpf_program__set_load_type(struct bpf_program *prog, enum
bpf_prog_load_type type)
{
if (prog->obj->loaded)
return libbpf_err(-EINVAL);
- prog->load_type = type;
+ switch (type) {
+ case BPF_PROG_LOAD_TYPE_DYNAMIC:
+ return bpf_program__set_dynamicload(prog);
+ default:
+ prog->load_type = type;
+ break;
+ }
+
return 0;
}
@@ -14099,3 +14177,49 @@ enum bpf_prog_load_type bpf_program__load_type(const
struct bpf_program *prog)
{
return prog->load_type;
}
+
+/*
+ * This function must be called after bpf_object__load_progs.
+ * Dynamically-loaded program data is initialized on object load.
+ * Post-load initialization is not supported.
+ */
+int
+bpf_program__load_dynamically(struct bpf_program *prog, int extra_log_level)
+{
+ int err;
+ struct bpf_object *obj;
+
+ obj = prog->obj;
+ if (!obj || !obj->loaded)
+ return libbpf_err(-EINVAL);
+
+ if (prog_is_subprog(obj, prog) || prog->load_type !=
BPF_PROG_LOAD_TYPE_DYNAMIC)
+ return libbpf_err(-EINVAL);
+
+ prog->log_level |= extra_log_level;
+
+ err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt,
+ obj->license, obj->kern_version,
&prog->fd);
+ if (err) {
+ pr_warn("prog '%s': failed to dynamically load: %d\n",
prog->name, err);
+ prog->log_level &= ~extra_log_level;
+ return err;
+ }
+
+ prog->log_level &= ~extra_log_level;
+ return 0;
+}
+
+int bpf_program__unload_dynamically(struct bpf_program *prog)
+{
+ int err;
+
+ if (!prog || prog->load_type != BPF_PROG_LOAD_TYPE_DYNAMIC)
+ return libbpf_err(-EINVAL);
+
+ /* Close the file descriptor but retain the program's data to
+ * support reloading the program if it is required again.
+ */
+ err = zclose(prog->fd);
+ return err ? libbpf_err(-errno) : 0;
+}
diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h
index 21e3d1f51cb3..531f30491f0b 100644
--- a/tools/lib/bpf/libbpf.h
+++ b/tools/lib/bpf/libbpf.h
@@ -1923,16 +1923,19 @@ LIBBPF_API int libbpf_unregister_prog_handler(int
handler_id);
*
* - BPF_PROG_LOAD_TYPE_DISABLED: the program is not loaded.
* - BPF_PROG_LOAD_TYPE_AUTO: the program is autoloaded when the bpf_object is
loaded.
+ * - BPF_PROG_LOAD_TYPE_DYNAMIC: the program is loaded and attached
dynamically.
*/
enum bpf_prog_load_type {
BPF_PROG_LOAD_TYPE_DISABLED = 0,
BPF_PROG_LOAD_TYPE_AUTO,
+ BPF_PROG_LOAD_TYPE_DYNAMIC,
};
LIBBPF_API int bpf_program__set_load_type(struct bpf_program *prog,
enum bpf_prog_load_type loadtype);
LIBBPF_API enum bpf_prog_load_type bpf_program__load_type(const struct
bpf_program *prog);
-
+LIBBPF_API int bpf_program__load_dynamically(struct bpf_program *prog, int
extra_log_level);
+LIBBPF_API int bpf_program__unload_dynamically(struct bpf_program *prog);
#ifdef __cplusplus
} /* extern "C" */
diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map
index 08323e7930fd..4d84e4794685 100644
--- a/tools/lib/bpf/libbpf.map
+++ b/tools/lib/bpf/libbpf.map
@@ -438,4 +438,6 @@ LIBBPF_1.6.0 {
bpf_linker__new_fd;
bpf_program__load_type;
bpf_program__set_load_type;
+ bpf_program__load_dynamically;
+ bpf_program__unload_dynamically;
} LIBBPF_1.5.0;
diff --git a/tools/testing/selftests/bpf/prog_tests/dynamicload.c
b/tools/testing/selftests/bpf/prog_tests/dynamicload.c
new file mode 100644
index 000000000000..9cde7dd45608
--- /dev/null
+++ b/tools/testing/selftests/bpf/prog_tests/dynamicload.c
@@ -0,0 +1,145 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include <test_progs.h>
+#include <time.h>
+#include "test_dynamicload.skel.h"
+
+void test_dynamicload(void)
+{
+ int duration = 0, err;
+ struct bpf_link *link;
+ struct test_dynamicload *skel;
+
+ skel = test_dynamicload__open();
+ if (CHECK(!skel, "skel_open", "failed to open skeleton\n"))
+ goto cleanup;
+
+ /* don't load prog1 */
+ bpf_program__set_load_type(skel->progs.prog1,
BPF_PROG_LOAD_TYPE_DISABLED);
+
+ /* prog2 is autoload */
+ bpf_program__set_load_type(skel->progs.prog2, BPF_PROG_LOAD_TYPE_AUTO);
+
+ /* prog3 is dynamically loaded */
+ bpf_program__set_load_type(skel->progs.prog3,
BPF_PROG_LOAD_TYPE_DYNAMIC);
+
+ err = test_dynamicload__load(skel);
+ if (CHECK(err, "skel_load", "failed to load skeleton: %d\n", err))
+ goto cleanup;
+
+ err = test_dynamicload__attach(skel);
+ if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
+ goto cleanup;
+
+ /* trigger the BPF programs */
+ usleep(1);
+
+ CHECK(skel->bss->prog1_called, "prog1", "called?!\n");
+ CHECK(!skel->bss->prog2_called, "prog2", "not called\n");
+ CHECK(skel->bss->prog3_called, "prog3", "called?!\n");
+
+ /* prog1 is disabled for load */
+ err = bpf_program__load_dynamically(skel->progs.prog1, 0);
+ if (CHECK(!err, "load_dynamically", "disabled program loaded?!\n"))
+ goto cleanup;
+
+ /* prog1 is disabled for load */
+ err = bpf_program__unload_dynamically(skel->progs.prog1);
+ if (CHECK(!err, "load_dynamically", "disabled program unloaded?!\n"))
+ goto cleanup;
+
+ /* prog2 is autoload */
+ err = bpf_program__load_dynamically(skel->progs.prog1, 0);
+ if (CHECK(!err, "load_dynamically", "autoload loaded dynamically?!\n"))
+ goto cleanup;
+
+ /* prog2 is autoload */
+ err = bpf_program__unload_dynamically(skel->progs.prog1);
+ if (CHECK(!err, "load_dynamically", "autoload unloaded
dynamically?!\n"))
+ goto cleanup;
+
+ /* reset the call flags */
+ skel->bss->prog2_called = false;
+ skel->bss->prog3_called = false;
+
+ usleep(1);
+
+ CHECK(skel->bss->prog1_called, "prog1", "called?!\n");
+ CHECK(!skel->bss->prog2_called, "prog2", "not called\n");
+ CHECK(skel->bss->prog3_called, "prog3", "called?!\n");
+
+ /* load prog3 */
+ err = bpf_program__load_dynamically(skel->progs.prog3, 0);
+ if (CHECK(err, "load_dynamically", "dynamic loading failed: %d\n", err))
+ goto cleanup;
+
+ /* attach prog3 */
+ link = bpf_program__attach(skel->progs.prog3);
+ if (CHECK(libbpf_get_error(link), "attach", "attaching failed: %ld\n",
+ libbpf_get_error(link)))
+ goto cleanup;
+
+ usleep(1);
+
+ CHECK(!skel->bss->prog3_called, "prog3", "not called\n");
+
+ /* detach prog3 as test_dynamicload__destroy doesn't detach dynamically
loaded programs */
+ err = bpf_link__destroy(link);
+ if (CHECK(err, "link__destroy", "link destroy failed: %d\n", err))
+ goto cleanup;
+
+ /* reset the call flags after detach */
+ skel->bss->prog2_called = false;
+ skel->bss->prog3_called = false;
+
+ usleep(1);
+
+ CHECK(!skel->bss->prog2_called, "prog2", "not called\n");
+ CHECK(skel->bss->prog3_called, "prog3", "called?!\n");
+
+ /* unload prog3 */
+ err = bpf_program__unload_dynamically(skel->progs.prog3);
+ if (CHECK(err, "unload_dynamically", "unload dynamically failed: %d\n",
err))
+ goto cleanup;
+
+ /* reload prog3 */
+ err = bpf_program__load_dynamically(skel->progs.prog3, 0);
+ if (CHECK(err, "load_dynamically", "dynamic reloading failed: %d\n",
err))
+ goto cleanup;
+
+ /* reattach prog3 */
+ link = bpf_program__attach(skel->progs.prog3);
+ if (CHECK(libbpf_get_error(link), "attach", "reattaching failed: %d\n",
err))
+ goto cleanup;
+
+ usleep(1);
+
+ CHECK(!skel->bss->prog3_called, "prog3", "not called\n");
+
+ /* detach prog3 as test_dynamicload__destroy doesn't detach dynamically
loaded programs */
+ err = bpf_link__destroy(link);
+ if (CHECK(err, "link__destroy", "link destroy failed: %d\n", err))
+ goto cleanup;
+
+ /* verify regular unload for dynamically loaded program,
+ * unload prog3 as a regular program
+ */
+ bpf_program__unload(skel->progs.prog3);
+
+ /* reset the call flags after unload */
+ skel->bss->prog2_called = false;
+ skel->bss->prog3_called = false;
+
+ usleep(1);
+
+ CHECK(!skel->bss->prog2_called, "prog2", "not called\n");
+ CHECK(skel->bss->prog3_called, "prog3", "called?!\n");
+
+ /* reloading prog3 must fail as it was unloaded as a regular program */
+ err = bpf_program__load_dynamically(skel->progs.prog3, 0);
+ if (CHECK(!err, "load_dynamically", "dynamic reloading succeeded?!
%d\n", err))
+ goto cleanup;
+
+cleanup:
+ test_dynamicload__destroy(skel);
+}
diff --git a/tools/testing/selftests/bpf/prog_tests/load_type.c
b/tools/testing/selftests/bpf/prog_tests/load_type.c
index 7c8d55173b2b..8bd082b3bc9d 100644
--- a/tools/testing/selftests/bpf/prog_tests/load_type.c
+++ b/tools/testing/selftests/bpf/prog_tests/load_type.c
@@ -7,6 +7,7 @@
void test_load_type(void)
{
int duration = 0, err;
+ struct bpf_link *link;
struct test_load_type *skel;
skel = test_load_type__open();
@@ -20,11 +21,47 @@ void test_load_type(void)
bpf_program__set_load_type(skel->progs.prog2, BPF_PROG_LOAD_TYPE_AUTO);
CHECK(!bpf_program__autoload(skel->progs.prog2), "prog2", "not
autoload?!\n");
+ err = bpf_program__set_load_type(skel->progs.prog3,
BPF_PROG_LOAD_TYPE_DYNAMIC);
+ if (CHECK(err, "set_load_type", "set_load_type(DYNAMIC) failed: %d\n",
err))
+ goto cleanup;
+ CHECK(bpf_program__load_type(skel->progs.prog3) !=
BPF_PROG_LOAD_TYPE_DYNAMIC,
+ "prog3", "didn't set type?!\n");
+
+ /* bpf_program__set_autoload(program, false) doesn't have effect if the
program
+ * type is not BPF_PROG_LOAD_TYPE_AUTO
+ */
+ err = bpf_program__set_autoload(skel->progs.prog3, false);
+ if (CHECK(err, "set_autoload", "set_autoload(false) failed: %d\n", err))
+ goto cleanup;
+
+ CHECK(bpf_program__load_type(skel->progs.prog3) !=
BPF_PROG_LOAD_TYPE_DYNAMIC,
+ "prog3", "changed type?!\n");
+
+ err = bpf_program__set_autoload(skel->progs.prog3, true);
+ if (CHECK(err, "set_autoload", "set_autoload(true) failed: %d\n", err))
+ goto cleanup;
+
+ CHECK(bpf_program__load_type(skel->progs.prog3) !=
BPF_PROG_LOAD_TYPE_AUTO,
+ "prog3", "didn't change type to auto?!\n");
+
+ /* change the type back to BPF_PROG_LOAD_TYPE_DYNAMIC */
+ err = bpf_program__set_load_type(skel->progs.prog3,
BPF_PROG_LOAD_TYPE_DYNAMIC);
+ if (CHECK(err, "set_load_type", "changing from AUTO to DYNAMIC failed:
%d\n", err))
+ goto cleanup;
+
+ CHECK(bpf_program__load_type(skel->progs.prog3) !=
BPF_PROG_LOAD_TYPE_DYNAMIC,
+ "prog3", "didn't change type from autoload to dynamic?!\n");
+
err = test_load_type__load(skel);
if (CHECK(err, "skel_load", "failed to load skeleton: %d\n", err))
goto cleanup;
CHECK(!bpf_program__autoattach(skel->progs.prog2), "prog2", "not
autoattach?!\n");
+ CHECK(bpf_program__autoattach(skel->progs.prog3), "prog3",
"autoattach?!\n");
+
+ /* loaded program type cannot be changed */
+ err = bpf_program__set_load_type(skel->progs.prog3,
BPF_PROG_LOAD_TYPE_DISABLED);
+ CHECK(!err, "prog3", "changed type after load?!\n");
err = test_load_type__attach(skel);
if (CHECK(err, "skel_attach", "skeleton attach failed: %d\n", err))
@@ -34,6 +71,30 @@ void test_load_type(void)
CHECK(skel->bss->prog1_called, "prog1", "called?!\n");
CHECK(!skel->bss->prog2_called, "prog2", "not called\n");
+ CHECK(skel->bss->prog3_called, "prog3", "called?!\n");
+
+ err = bpf_program__load_dynamically(skel->progs.prog3, 0);
+ if (CHECK(err, "load_dynamically", "load dynamically failed: %d\n",
err))
+ goto cleanup;
+
+ err = bpf_program__load_dynamically(skel->progs.prog3, 0);
+ if (CHECK(err, "load_dynamically", "load dynamically failed: %d\n",
err))
+ goto cleanup;
+
+ /* attach prog3 */
+ link = bpf_program__attach(skel->progs.prog3);
+ if (CHECK(libbpf_get_error(link), "attach", "attaching failed: %ld\n",
+ libbpf_get_error(link)))
+ goto cleanup;
+
+ usleep(1);
+
+ CHECK(!skel->bss->prog3_called, "prog3", "not called?!\n");
+
+ /* detach prog3 as test_load_type__destroy doesn't detach dynamically
loaded programs */
+ err = bpf_link__destroy(link);
+ if (CHECK(err, "link__destroy", "link destroy failed: %d\n", err))
+ goto cleanup;
cleanup:
test_load_type__destroy(skel);
diff --git a/tools/testing/selftests/bpf/progs/test_dynamicload.c
b/tools/testing/selftests/bpf/progs/test_dynamicload.c
new file mode 100644
index 000000000000..3d9b81691d7a
--- /dev/null
+++ b/tools/testing/selftests/bpf/progs/test_dynamicload.c
@@ -0,0 +1,31 @@
+// SPDX-License-Identifier: GPL-2.0
+
+#include "vmlinux.h"
+#include <bpf/bpf_helpers.h>
+
+bool prog1_called = false;
+bool prog2_called = false;
+bool prog3_called = false;
+
+SEC("raw_tp/sys_enter")
+int prog1(const void *ctx)
+{
+ prog1_called = true;
+ return 0;
+}
+
+SEC("raw_tp/sys_enter")
+int prog2(const void *ctx)
+{
+ prog2_called = true;
+ return 0;
+}
+
+SEC("raw_tp/sys_enter")
+int prog3(const void *ctx)
+{
+ prog3_called = true;
+ return 0;
+}
+
+char _license[] SEC("license") = "GPL";
diff --git a/tools/testing/selftests/bpf/progs/test_load_type.c
b/tools/testing/selftests/bpf/progs/test_load_type.c
index a0d39757c5b9..3d9b81691d7a 100644
--- a/tools/testing/selftests/bpf/progs/test_load_type.c
+++ b/tools/testing/selftests/bpf/progs/test_load_type.c
@@ -5,6 +5,7 @@
bool prog1_called = false;
bool prog2_called = false;
+bool prog3_called = false;
SEC("raw_tp/sys_enter")
int prog1(const void *ctx)
@@ -20,4 +21,11 @@ int prog2(const void *ctx)
return 0;
}
+SEC("raw_tp/sys_enter")
+int prog3(const void *ctx)
+{
+ prog3_called = true;
+ return 0;
+}
+
char _license[] SEC("license") = "GPL";
--
2.39.5 (Apple Git-154)
