On Mon, Jul 19, 2021, 14:37 Alex Bennée <alex.ben...@linaro.org> wrote:
> In user-mode emulation there is a small race between preexit_cleanup
> and exit_group() which means we may end up calling instrumented
> instructions before the kernel reaps child threads. To solve this we
> implement a new helper which ensures the callbacks are flushed along
> with any translations before we let the host do it's a thing.
>
> While we are at it make the documentation of
> qemu_plugin_register_atexit_cb clearer as to what the user can expect.
>
> Signed-off-by: Alex Bennée <alex.ben...@linaro.org>
>
Looks great to me and I cannot reproduce the race with it installed in.
With calling `qemu_plugin_disable_mem_helpers`:
Reviewed-by: Mahmoud Mandour <ma.mando...@gmail.com>
---
> include/qemu/plugin.h | 12 ++++++++++++
> include/qemu/qemu-plugin.h | 13 +++++++++++++
> bsd-user/syscall.c | 6 +++---
> linux-user/exit.c | 2 +-
> plugins/core.c | 33 +++++++++++++++++++++++++++++++++
> 5 files changed, 62 insertions(+), 4 deletions(-)
>
> diff --git a/include/qemu/plugin.h b/include/qemu/plugin.h
> index 0fefbc6084..9a8438f683 100644
> --- a/include/qemu/plugin.h
> +++ b/include/qemu/plugin.h
> @@ -190,6 +190,16 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr);
>
> void qemu_plugin_disable_mem_helpers(CPUState *cpu);
>
> +/**
> + * qemu_plugin_user_exit(): clean-up callbacks before calling exit
> callbacks
> + *
> + * This is a user-mode only helper that ensure we have fully cleared
> + * callbacks from all threads before calling the exit callbacks. This
> + * is so the plugins themselves don't have to jump through hoops to
> + * guard against race conditions.
> + */
> +void qemu_plugin_user_exit(void);
> +
> #else /* !CONFIG_PLUGIN */
>
> static inline void qemu_plugin_add_opts(void)
> @@ -250,6 +260,8 @@ void qemu_plugin_add_dyn_cb_arr(GArray *arr)
> static inline void qemu_plugin_disable_mem_helpers(CPUState *cpu)
> { }
>
> +static inline void qemu_plugin_user_exit(void)
> +{ }
> #endif /* !CONFIG_PLUGIN */
>
> #endif /* QEMU_PLUGIN_H */
> diff --git a/include/qemu/qemu-plugin.h b/include/qemu/qemu-plugin.h
> index dc3496f36c..e6e815abc5 100644
> --- a/include/qemu/qemu-plugin.h
> +++ b/include/qemu/qemu-plugin.h
> @@ -549,6 +549,19 @@ void qemu_plugin_vcpu_for_each(qemu_plugin_id_t id,
> void qemu_plugin_register_flush_cb(qemu_plugin_id_t id,
> qemu_plugin_simple_cb_t cb);
>
> +/**
> + * qemu_plugin_register_atexit_cb() - register exit callback
> + * @id: plugin ID
> + * @cb: callback
> + * @userdata: user data for callback
> + *
> + * The @cb function is called once execution has finished. Plugins
> + * should be able to free all their resources at this point much like
> + * after a reset/uninstall callback is called.
> + *
> + * In user-mode it is possible a few un-instrumented instructions from
> + * child threads may run before the host kernel reaps the threads.
> + */
> void qemu_plugin_register_atexit_cb(qemu_plugin_id_t id,
> qemu_plugin_udata_cb_t cb, void
> *userdata);
>
> diff --git a/bsd-user/syscall.c b/bsd-user/syscall.c
> index 7d986e9700..3f44311396 100644
> --- a/bsd-user/syscall.c
> +++ b/bsd-user/syscall.c
> @@ -335,7 +335,7 @@ abi_long do_freebsd_syscall(void *cpu_env, int num,
> abi_long arg1,
> _mcleanup();
> #endif
> gdb_exit(arg1);
> - qemu_plugin_atexit_cb();
> + qemu_plugin_user_exit();
> /* XXX: should free thread stack and CPU env */
> _exit(arg1);
> ret = 0; /* avoid warning */
> @@ -437,7 +437,7 @@ abi_long do_netbsd_syscall(void *cpu_env, int num,
> abi_long arg1,
> _mcleanup();
> #endif
> gdb_exit(arg1);
> - qemu_plugin_atexit_cb();
> + qemu_plugin_user_exit();
> /* XXX: should free thread stack and CPU env */
> _exit(arg1);
> ret = 0; /* avoid warning */
> @@ -516,7 +516,7 @@ abi_long do_openbsd_syscall(void *cpu_env, int num,
> abi_long arg1,
> _mcleanup();
> #endif
> gdb_exit(arg1);
> - qemu_plugin_atexit_cb();
> + qemu_plugin_user_exit();
> /* XXX: should free thread stack and CPU env */
> _exit(arg1);
> ret = 0; /* avoid warning */
> diff --git a/linux-user/exit.c b/linux-user/exit.c
> index 70b344048c..527e29cbc1 100644
> --- a/linux-user/exit.c
> +++ b/linux-user/exit.c
> @@ -35,5 +35,5 @@ void preexit_cleanup(CPUArchState *env, int code)
> __gcov_dump();
> #endif
> gdb_exit(code);
> - qemu_plugin_atexit_cb();
> + qemu_plugin_user_exit();
> }
> diff --git a/plugins/core.c b/plugins/core.c
> index e1bcdb570d..c573b81a96 100644
> --- a/plugins/core.c
> +++ b/plugins/core.c
> @@ -487,6 +487,39 @@ void qemu_plugin_register_atexit_cb(qemu_plugin_id_t
> id,
> plugin_register_cb_udata(id, QEMU_PLUGIN_EV_ATEXIT, cb, udata);
> }
>
> +/*
> + * Handle exit from linux-user. Unlike the normal atexit() mechanism
> + * we need to handle the clean-up manually as it's possible threads
> + * are still running. We need to remove all callbacks from code
> + * generation, flush the current translations and then we can safely
> + * trigger the exit callbacks.
> + */
> +
> +void qemu_plugin_user_exit(void)
> +{
> + enum qemu_plugin_event ev;
> +
> + QEMU_LOCK_GUARD(&plugin.lock);
> +
> + start_exclusive();
> +
> + /* un-register all callbacks except the final AT_EXIT one */
> + for (ev = 0; ev < QEMU_PLUGIN_EV_MAX; ev++) {
> + if (ev != QEMU_PLUGIN_EV_ATEXIT) {
> + struct qemu_plugin_ctx *ctx;
> + QTAILQ_FOREACH(ctx, &plugin.ctxs, entry) {
> + plugin_unregister_cb__locked(ctx, ev);
> + }
> + }
> + }
> +
> + tb_flush(current_cpu);
> + end_exclusive();
> +
> + /* now it's safe to handle the exit case */
> + qemu_plugin_atexit_cb();
> +}
> +
> /*
> * Call this function after longjmp'ing to the main loop. It's possible
> that the
> * last instruction of a TB might have used helpers, and therefore the
> --
> 2.32.0.264.g75ae10bc75
>
>
