The branch main has been updated by br:
URL:
https://cgit.FreeBSD.org/src/commit/?id=df114daef4c48548c3c2b86717612761185ae18f
commit df114daef4c48548c3c2b86717612761185ae18f
Author: Ruslan Bukin <b...@freebsd.org>
AuthorDate: 2025-07-03 13:10:45 +0000
Commit: Ruslan Bukin <b...@freebsd.org>
CommitDate: 2025-07-03 14:48:34 +0000
Import the Hardware Trace (HWT) framework.
The HWT framework provides infrastructure for hardware-assisted tracing. It
collects detailed information about software execution and records it as
"events" in highly compressed format into DRAM. The events cover information
about control flow changes of a program, whether branches taken or not,
exceptions taken, timing information, cycles elapsed and more. This allows
to reconstruct entire program flow of a given application.
This comes with separate machine-dependent tracing backends for trace
collection, trace decoder libraries and an instrumentation tool.
Reviewed by: kib (sys/kern bits)
Sponsored by: UKRI
Differential Revision: https://reviews.freebsd.org/D40466
---
sys/conf/files | 13 ++
sys/conf/options | 3 +
sys/dev/hwt/hwt.c | 242 ++++++++++++++++++++
sys/dev/hwt/hwt_backend.c | 289 ++++++++++++++++++++++++
sys/dev/hwt/hwt_backend.h | 87 ++++++++
sys/dev/hwt/hwt_config.c | 108 +++++++++
sys/dev/hwt/hwt_config.h | 36 +++
sys/dev/hwt/hwt_context.c | 201 +++++++++++++++++
sys/dev/hwt/hwt_context.h | 86 ++++++++
sys/dev/hwt/hwt_contexthash.c | 133 +++++++++++
sys/dev/hwt/hwt_contexthash.h | 42 ++++
sys/dev/hwt/hwt_cpu.c | 115 ++++++++++
sys/dev/hwt/hwt_cpu.h | 45 ++++
sys/dev/hwt/hwt_hook.c | 323 +++++++++++++++++++++++++++
sys/dev/hwt/hwt_hook.h | 56 +++++
sys/dev/hwt/hwt_intr.h | 33 +++
sys/dev/hwt/hwt_ioctl.c | 443 +++++++++++++++++++++++++++++++++++++
sys/dev/hwt/hwt_ioctl.h | 35 +++
sys/dev/hwt/hwt_owner.c | 157 +++++++++++++
sys/dev/hwt/hwt_owner.h | 45 ++++
sys/dev/hwt/hwt_ownerhash.c | 141 ++++++++++++
sys/dev/hwt/hwt_ownerhash.h | 42 ++++
sys/dev/hwt/hwt_record.c | 302 +++++++++++++++++++++++++
sys/dev/hwt/hwt_record.h | 47 ++++
sys/dev/hwt/hwt_thread.c | 162 ++++++++++++++
sys/dev/hwt/hwt_thread.h | 64 ++++++
sys/dev/hwt/hwt_vm.c | 501 ++++++++++++++++++++++++++++++++++++++++++
sys/dev/hwt/hwt_vm.h | 47 ++++
sys/kern/kern_exec.c | 19 ++
sys/kern/kern_linker.c | 5 +-
sys/kern/kern_pmc.c | 4 +
sys/kern/kern_thr.c | 12 +-
sys/kern/kern_thread.c | 9 +
sys/kern/sched_4bsd.c | 22 +-
sys/kern/sched_ule.c | 19 ++
sys/kern/vfs_vnops.c | 23 ++
sys/modules/Makefile | 5 +
sys/modules/hwt/Makefile | 21 ++
sys/sys/hwt.h | 129 +++++++++++
sys/sys/hwt_record.h | 70 ++++++
sys/sys/proc.h | 2 +
sys/vm/vm_mmap.c | 16 ++
42 files changed, 4150 insertions(+), 4 deletions(-)
diff --git a/sys/conf/files b/sys/conf/files
index 75ee10be5896..f6d473b1431b 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -1776,6 +1776,19 @@ dev/hwpmc/hwpmc_soft.c optional hwpmc
dev/hwreset/hwreset.c optional hwreset
dev/hwreset/hwreset_array.c optional hwreset
dev/hwreset/hwreset_if.m optional hwreset
+dev/hwt/hwt.c optional hwt
+dev/hwt/hwt_backend.c optional hwt
+dev/hwt/hwt_config.c optional hwt
+dev/hwt/hwt_context.c optional hwt
+dev/hwt/hwt_contexthash.c optional hwt
+dev/hwt/hwt_cpu.c optional hwt
+dev/hwt/hwt_hook.c optional hwt
+dev/hwt/hwt_ioctl.c optional hwt
+dev/hwt/hwt_owner.c optional hwt
+dev/hwt/hwt_ownerhash.c optional hwt
+dev/hwt/hwt_record.c optional hwt
+dev/hwt/hwt_thread.c optional hwt
+dev/hwt/hwt_vm.c optional hwt
dev/ichiic/ig4_acpi.c optional ig4 acpi iicbus
dev/ichiic/ig4_iic.c optional ig4 iicbus
dev/ichiic/ig4_pci.c optional ig4 pci iicbus
diff --git a/sys/conf/options b/sys/conf/options
index 03e8964e965d..a637b0b74a77 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -885,6 +885,9 @@ DCONS_FORCE_GDB opt_dcons.h
HWPMC_DEBUG opt_global.h
HWPMC_HOOKS
+# Hardware Trace (HWT) framework options
+HWT_HOOKS
+
# 802.11 support layer
IEEE80211_DEBUG opt_wlan.h
IEEE80211_DEBUG_REFCNT opt_wlan.h
diff --git a/sys/dev/hwt/hwt.c b/sys/dev/hwt/hwt.c
new file mode 100644
index 000000000000..c476e6031ba8
--- /dev/null
+++ b/sys/dev/hwt/hwt.c
@@ -0,0 +1,242 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2025 Ruslan Bukin <b...@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/*
+ * Hardware Tracing framework.
+ *
+ * The framework manages hardware tracing units that collect information
+ * about software execution and store it as events in highly compressed format
+ * into DRAM. The events cover information about control flow changes of a
+ * program, whether branches taken or not, exceptions taken, timing
information,
+ * cycles elapsed and more. That allows us to restore entire program flow of a
+ * given application without performance impact.
+ *
+ * Design overview.
+ *
+ * The framework provides character devices for mmap(2) and ioctl(2) system
+ * calls to allow user to manage CPU (hardware) tracing units.
+ *
+ * /dev/hwt:
+ * .ioctl:
+ * hwt_ioctl():
+ * a) HWT_IOC_ALLOC
+ * Allocates kernel tracing context CTX based on requested
mode
+ * of operation. Verifies the information that comes with the
+ * request (pid, cpus), allocates unique ID for the context.
+ * Creates a new character device for CTX management.
+ *
+ * /dev/hwt_%d[_%d], ident[, thread_id]
+ * .mmap
+ * Maps tracing buffers of the corresponding thread to userspace.
+ * .ioctl
+ * hwt_thread_ioctl():
+ * a) HWT_IOC_START
+ * Enables tracing unit for a given context.
+ * b) HWT_IOC_RECORD_GET
+ * Transfers (small) record entries collected during program
+ * execution for a given context to userspace, such as mmaping
+ * tables of executable and dynamic libraries, interpreter,
+ * kernel mappings, tid of threads created, etc.
+ * c) HWT_IOC_SET_CONFIG
+ * Allows to specify backend-specific configuration of the
+ * trace unit.
+ * d) HWT_IOC_WAKEUP
+ * Wakes up a thread that is currently sleeping.
+ * e) HWT_IOC_BUFPTR_GET
+ * Transfers current hardware pointer in the filling buffer
+ * to the userspace.
+ * f) HWT_IOC_SVC_BUF
+ * To avoid data loss, userspace may notify kernel it has
+ * copied out the given buffer, so kernel is ok to overwrite
+ *
+ * HWT context lifecycle in THREAD mode of operation:
+ * 1. User invokes HWT_IOC_ALLOC ioctl with information about pid to trace and
+ * size of the buffers for the trace data to allocate.
+ * Some architectures may have different tracing units supported, so user
+ * also provides backend name to use for this context, e.g. "coresight".
+ * 2. Kernel allocates context, lookups the proc for the given pid. Then it
+ * creates first hwt_thread in the context and allocates trace buffers for
+ * it. Immediately, kernel initializes tracing backend.
+ * Kernel creates character device and returns unique identificator of
+ * trace context to the user.
+ * 3. To manage the new context, user opens the character device created.
+ * User invokes HWT_IOC_START ioctl, kernel marks context as RUNNING.
+ * At this point any HWT hook invocation by scheduler enables/disables
+ * tracing for threads associated with the context (threads of the proc).
+ * Any new threads creation (of the target proc) procedures will be invoking
+ * corresponding hooks in HWT framework, so that new hwt_thread and buffers
+ * allocated, character device for mmap(2) created on the fly.
+ * 4. User issues HWT_IOC_RECORD_GET ioctl to fetch information about mmaping
+ * tables and threads created during application startup.
+ * 5. User mmaps tracing buffers of each thread to userspace (using
+ * /dev/hwt_%d_%d % (ident, thread_id) character devices).
+ * 6. User can repeat 4 if expected thread is not yet created during target
+ * application execution.
+ * 7. User issues HWT_IOC_BUFPTR_GET ioctl to get current filling level of the
+ * hardware buffer of a given thread.
+ * 8. User invokes trace decoder library to process available data and see the
+ * results in human readable form.
+ * 9. User repeats 7 if needed.
+ *
+ * HWT context lifecycle in CPU mode of operation:
+ * 1. User invokes HWT_IOC_ALLOC ioctl providing a set of CPU to trace within
+ * single CTX.
+ * 2. Kernel verifies the set of CPU and allocates tracing context, creates
+ * a buffer for each CPU.
+ * Kernel creates a character device for every CPU provided in the request.
+ * Kernel initialized tracing backend.
+ * 3. User opens character devices of interest to map the buffers to userspace.
+ * User can start tracing by invoking HWT_IOC_START on any of character
+ * device within the context, entire context will be marked as RUNNING.
+ * 4. The rest is similar to the THREAD mode.
+ *
+ */
+
+#include <sys/param.h>
+#include <sys/conf.h>
+#include <sys/eventhandler.h>
+#include <sys/kernel.h>
+#include <sys/module.h>
+
+#include <dev/hwt/hwt_context.h>
+#include <dev/hwt/hwt_contexthash.h>
+#include <dev/hwt/hwt_thread.h>
+#include <dev/hwt/hwt_owner.h>
+#include <dev/hwt/hwt_ownerhash.h>
+#include <dev/hwt/hwt_backend.h>
+#include <dev/hwt/hwt_record.h>
+#include <dev/hwt/hwt_ioctl.h>
+#include <dev/hwt/hwt_hook.h>
+
+#define HWT_DEBUG
+#undef HWT_DEBUG
+
+#ifdef HWT_DEBUG
+#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
+#else
+#define dprintf(fmt, ...)
+#endif
+
+static eventhandler_tag hwt_exit_tag;
+static struct cdev *hwt_cdev;
+static struct cdevsw hwt_cdevsw = {
+ .d_version = D_VERSION,
+ .d_name = "hwt",
+ .d_mmap_single = NULL,
+ .d_ioctl = hwt_ioctl
+};
+
+static void
+hwt_process_exit(void *arg __unused, struct proc *p)
+{
+ struct hwt_owner *ho;
+
+ /* Stop HWTs associated with exiting owner, if any. */
+ ho = hwt_ownerhash_lookup(p);
+ if (ho)
+ hwt_owner_shutdown(ho);
+}
+
+static int
+hwt_load(void)
+{
+ struct make_dev_args args;
+ int error;
+
+ make_dev_args_init(&args);
+ args.mda_devsw = &hwt_cdevsw;
+ args.mda_flags = MAKEDEV_CHECKNAME | MAKEDEV_WAITOK;
+ args.mda_uid = UID_ROOT;
+ args.mda_gid = GID_WHEEL;
+ args.mda_mode = 0660;
+ args.mda_si_drv1 = NULL;
+
+ hwt_backend_load();
+ hwt_ctx_load();
+ hwt_contexthash_load();
+ hwt_ownerhash_load();
+ hwt_record_load();
+
+ error = make_dev_s(&args, &hwt_cdev, "hwt");
+ if (error != 0)
+ return (error);
+
+ hwt_exit_tag = EVENTHANDLER_REGISTER(process_exit, hwt_process_exit,
+ NULL, EVENTHANDLER_PRI_ANY);
+
+ hwt_hook_load();
+
+ return (0);
+}
+
+static int
+hwt_unload(void)
+{
+
+ hwt_hook_unload();
+ EVENTHANDLER_DEREGISTER(process_exit, hwt_exit_tag);
+ destroy_dev(hwt_cdev);
+ hwt_record_unload();
+ hwt_ownerhash_unload();
+ hwt_contexthash_unload();
+ hwt_ctx_unload();
+ hwt_backend_unload();
+
+ return (0);
+}
+
+static int
+hwt_modevent(module_t mod, int type, void *data)
+{
+ int error;
+
+ switch (type) {
+ case MOD_LOAD:
+ error = hwt_load();
+ break;
+ case MOD_UNLOAD:
+ error = hwt_unload();
+ break;
+ default:
+ error = 0;
+ break;
+ }
+
+ return (error);
+}
+
+static moduledata_t hwt_mod = {
+ "hwt",
+ hwt_modevent,
+ NULL
+};
+
+DECLARE_MODULE(hwt, hwt_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
+MODULE_VERSION(hwt, 1);
diff --git a/sys/dev/hwt/hwt_backend.c b/sys/dev/hwt/hwt_backend.c
new file mode 100644
index 000000000000..1ba5db0d3d09
--- /dev/null
+++ b/sys/dev/hwt/hwt_backend.c
@@ -0,0 +1,289 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2025 Ruslan Bukin <b...@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+/* Hardware Trace (HWT) framework. */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/hwt.h>
+
+#include <dev/hwt/hwt_hook.h>
+#include <dev/hwt/hwt_context.h>
+#include <dev/hwt/hwt_config.h>
+#include <dev/hwt/hwt_thread.h>
+#include <dev/hwt/hwt_backend.h>
+
+#define HWT_BACKEND_DEBUG
+#undef HWT_BACKEND_DEBUG
+
+#ifdef HWT_BACKEND_DEBUG
+#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
+#else
+#define dprintf(fmt, ...)
+#endif
+
+static struct mtx hwt_backend_mtx;
+
+struct hwt_backend_entry {
+ struct hwt_backend *backend;
+ LIST_ENTRY(hwt_backend_entry) next;
+};
+
+static LIST_HEAD(, hwt_backend_entry) hwt_backends;
+
+static MALLOC_DEFINE(M_HWT_BACKEND, "hwt_backend", "HWT backend");
+
+int
+hwt_backend_init(struct hwt_context *ctx)
+{
+ int error;
+
+ dprintf("%s\n", __func__);
+
+ error = ctx->hwt_backend->ops->hwt_backend_init(ctx);
+
+ return (error);
+}
+
+void
+hwt_backend_deinit(struct hwt_context *ctx)
+{
+
+ dprintf("%s\n", __func__);
+
+ ctx->hwt_backend->ops->hwt_backend_deinit(ctx);
+}
+
+int
+hwt_backend_configure(struct hwt_context *ctx, int cpu_id, int thread_id)
+{
+ int error;
+
+ dprintf("%s\n", __func__);
+
+ error = ctx->hwt_backend->ops->hwt_backend_configure(ctx, cpu_id,
+ thread_id);
+
+ return (error);
+}
+
+void
+hwt_backend_enable(struct hwt_context *ctx, int cpu_id)
+{
+
+ dprintf("%s\n", __func__);
+
+ ctx->hwt_backend->ops->hwt_backend_enable(ctx, cpu_id);
+}
+
+void
+hwt_backend_disable(struct hwt_context *ctx, int cpu_id)
+{
+
+ dprintf("%s\n", __func__);
+
+ ctx->hwt_backend->ops->hwt_backend_disable(ctx, cpu_id);
+}
+
+void
+hwt_backend_enable_smp(struct hwt_context *ctx)
+{
+
+ dprintf("%s\n", __func__);
+
+ ctx->hwt_backend->ops->hwt_backend_enable_smp(ctx);
+}
+
+void
+hwt_backend_disable_smp(struct hwt_context *ctx)
+{
+
+ dprintf("%s\n", __func__);
+
+ ctx->hwt_backend->ops->hwt_backend_disable_smp(ctx);
+}
+
+void __unused
+hwt_backend_dump(struct hwt_context *ctx, int cpu_id)
+{
+
+ dprintf("%s\n", __func__);
+
+ ctx->hwt_backend->ops->hwt_backend_dump(cpu_id);
+}
+
+int
+hwt_backend_read(struct hwt_context *ctx, struct hwt_vm *vm, int *ident,
+ vm_offset_t *offset, uint64_t *data)
+{
+ int error;
+
+ dprintf("%s\n", __func__);
+
+ error = ctx->hwt_backend->ops->hwt_backend_read(vm, ident,
+ offset, data);
+
+ return (error);
+}
+
+struct hwt_backend *
+hwt_backend_lookup(const char *name)
+{
+ struct hwt_backend_entry *entry;
+ struct hwt_backend *backend;
+
+ HWT_BACKEND_LOCK();
+ LIST_FOREACH(entry, &hwt_backends, next) {
+ backend = entry->backend;
+ if (strcmp(backend->name, name) == 0) {
+ HWT_BACKEND_UNLOCK();
+ return (backend);
+ }
+ }
+ HWT_BACKEND_UNLOCK();
+
+ return (NULL);
+}
+
+int
+hwt_backend_register(struct hwt_backend *backend)
+{
+ struct hwt_backend_entry *entry;
+
+ if (backend == NULL ||
+ backend->name == NULL ||
+ backend->ops == NULL)
+ return (EINVAL);
+
+ entry = malloc(sizeof(struct hwt_backend_entry), M_HWT_BACKEND,
+ M_WAITOK | M_ZERO);
+ entry->backend = backend;
+
+ HWT_BACKEND_LOCK();
+ LIST_INSERT_HEAD(&hwt_backends, entry, next);
+ HWT_BACKEND_UNLOCK();
+
+ return (0);
+}
+
+int
+hwt_backend_unregister(struct hwt_backend *backend)
+{
+ struct hwt_backend_entry *entry, *tmp;
+
+ if (backend == NULL)
+ return (EINVAL);
+
+ /* TODO: check if not in use */
+
+ HWT_BACKEND_LOCK();
+ LIST_FOREACH_SAFE(entry, &hwt_backends, next, tmp) {
+ if (entry->backend == backend) {
+ LIST_REMOVE(entry, next);
+ HWT_BACKEND_UNLOCK();
+ free(entry, M_HWT_BACKEND);
+ return (0);
+ }
+ }
+ HWT_BACKEND_UNLOCK();
+
+ return (ENOENT);
+}
+
+void
+hwt_backend_load(void)
+{
+
+ mtx_init(&hwt_backend_mtx, "hwt backend", NULL, MTX_DEF);
+ LIST_INIT(&hwt_backends);
+}
+
+void
+hwt_backend_unload(void)
+{
+
+ /* TODO: ensure all unregistered */
+
+ mtx_destroy(&hwt_backend_mtx);
+}
+
+void
+hwt_backend_stop(struct hwt_context *ctx)
+{
+ dprintf("%s\n", __func__);
+
+ ctx->hwt_backend->ops->hwt_backend_stop(ctx);
+}
+
+int
+hwt_backend_svc_buf(struct hwt_context *ctx, void *data, size_t data_size,
+ int data_version)
+{
+ int error;
+
+ dprintf("%s\n", __func__);
+
+ error = ctx->hwt_backend->ops->hwt_backend_svc_buf(ctx, data, data_size,
+ data_version);
+
+ return (error);
+}
+
+int
+hwt_backend_thread_alloc(struct hwt_context *ctx, struct hwt_thread *thr)
+{
+ int error;
+
+ dprintf("%s\n", __func__);
+
+ if (ctx->hwt_backend->ops->hwt_backend_thread_alloc == NULL)
+ return (0);
+ KASSERT(thr->private == NULL,
+ ("%s: thread private data is not NULL\n", __func__));
+ error = ctx->hwt_backend->ops->hwt_backend_thread_alloc(thr);
+
+ return (error);
+}
+
+void
+hwt_backend_thread_free(struct hwt_thread *thr)
+{
+ dprintf("%s\n", __func__);
+
+ if (thr->backend->ops->hwt_backend_thread_free == NULL)
+ return;
+ KASSERT(thr->private != NULL,
+ ("%s: thread private data is NULL\n", __func__));
+ thr->backend->ops->hwt_backend_thread_free(thr);
+
+ return;
+}
diff --git a/sys/dev/hwt/hwt_backend.h b/sys/dev/hwt/hwt_backend.h
new file mode 100644
index 000000000000..3b6c9442a7a6
--- /dev/null
+++ b/sys/dev/hwt/hwt_backend.h
@@ -0,0 +1,87 @@
+/*-
+ * Copyright (c) 2023-2025 Ruslan Bukin <b...@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _DEV_HWT_HWT_BACKEND_H_
+#define _DEV_HWT_HWT_BACKEND_H_
+
+struct hwt_backend_ops {
+ int (*hwt_backend_init)(struct hwt_context *);
+ int (*hwt_backend_deinit)(struct hwt_context *);
+ int (*hwt_backend_configure)(struct hwt_context *, int cpu_id,
+ int thread_id);
+ int (*hwt_backend_svc_buf)(struct hwt_context *, void *data,
+ size_t data_size, int data_version);
+ void (*hwt_backend_enable)(struct hwt_context *, int cpu_id);
+ void (*hwt_backend_disable)(struct hwt_context *, int cpu_id);
+ int (*hwt_backend_read)(struct hwt_vm *, int *ident,
+ vm_offset_t *offset, uint64_t *data);
+ void (*hwt_backend_stop)(struct hwt_context *);
+ /* For backends that are tied to local CPU registers */
+ int (*hwt_backend_enable_smp)(struct hwt_context *);
+ int (*hwt_backend_disable_smp)(struct hwt_context *);
+ /* Allocation and initialization of backend-specific thread data. */
+ int (*hwt_backend_thread_alloc)(struct hwt_thread *);
+ void (*hwt_backend_thread_free)(struct hwt_thread *);
+ /* Debugging only. */
+ void (*hwt_backend_dump)(int cpu_id);
+};
+
+struct hwt_backend {
+ const char *name;
+ struct hwt_backend_ops *ops;
+ /* buffers require kernel virtual addresses */
+ bool kva_req;
+};
+
+int hwt_backend_init(struct hwt_context *ctx);
+void hwt_backend_deinit(struct hwt_context *ctx);
+int hwt_backend_configure(struct hwt_context *ctx, int cpu_id, int thread_id);
+void hwt_backend_enable(struct hwt_context *ctx, int cpu_id);
+void hwt_backend_disable(struct hwt_context *ctx, int cpu_id);
+void hwt_backend_enable_smp(struct hwt_context *ctx);
+void hwt_backend_disable_smp(struct hwt_context *ctx);
+void hwt_backend_dump(struct hwt_context *ctx, int cpu_id);
+int hwt_backend_read(struct hwt_context *ctx, struct hwt_vm *vm, int *ident,
+ vm_offset_t *offset, uint64_t *data);
+int hwt_backend_register(struct hwt_backend *);
+int hwt_backend_unregister(struct hwt_backend *);
+void hwt_backend_stop(struct hwt_context *);
+int hwt_backend_svc_buf(struct hwt_context *ctx, void *data, size_t data_size,
+ int data_version);
+struct hwt_backend * hwt_backend_lookup(const char *name);
+int hwt_backend_thread_alloc(struct hwt_context *ctx, struct hwt_thread *);
+void hwt_backend_thread_free(struct hwt_thread *);
+
+void hwt_backend_load(void);
+void hwt_backend_unload(void);
+
+#define HWT_BACKEND_LOCK() mtx_lock(&hwt_backend_mtx)
+#define HWT_BACKEND_UNLOCK() mtx_unlock(&hwt_backend_mtx)
+
+#endif /* !_DEV_HWT_HWT_BACKEND_H_ */
+
diff --git a/sys/dev/hwt/hwt_config.c b/sys/dev/hwt/hwt_config.c
new file mode 100644
index 000000000000..30688e7fc76b
--- /dev/null
+++ b/sys/dev/hwt/hwt_config.c
@@ -0,0 +1,108 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2025 Ruslan Bukin <b...@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/proc.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/lock.h>
+#include <sys/hwt.h>
+
+#include <vm/vm.h>
+
+#include <dev/hwt/hwt_hook.h>
+#include <dev/hwt/hwt_context.h>
+#include <dev/hwt/hwt_contexthash.h>
+#include <dev/hwt/hwt_config.h>
+#include <dev/hwt/hwt_thread.h>
+#include <dev/hwt/hwt_record.h>
+
+#define HWT_MAXCONFIGSIZE PAGE_SIZE
+
+#define HWT_CONFIG_DEBUG
+#undef HWT_CONFIG_DEBUG
+
+#ifdef HWT_CONFIG_DEBUG
+#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
+#else
+#define dprintf(fmt, ...)
+#endif
+
+static MALLOC_DEFINE(M_HWT_CONFIG, "hwt_config", "HWT config");
+
+int
+hwt_config_set(struct thread *td, struct hwt_context *ctx,
+ struct hwt_set_config *sconf)
+{
+ size_t config_size;
+ void *old_config;
+ void *config;
+ int error;
+
+ config_size = sconf->config_size;
+ if (config_size == 0)
+ return (0);
+
+ if (config_size > HWT_MAXCONFIGSIZE)
+ return (EFBIG);
+
+ config = malloc(config_size, M_HWT_CONFIG, M_WAITOK | M_ZERO);
+
+ error = copyin(sconf->config, config, config_size);
+ if (error) {
+ free(config, M_HWT_CONFIG);
+ return (error);
+ }
+
+ HWT_CTX_LOCK(ctx);
+ old_config = ctx->config;
+ ctx->config = config;
+ ctx->config_size = sconf->config_size;
+ ctx->config_version = sconf->config_version;
+ HWT_CTX_UNLOCK(ctx);
+
+ if (old_config != NULL)
+ free(old_config, M_HWT_CONFIG);
+
+ return (error);
+}
+
+void
+hwt_config_free(struct hwt_context *ctx)
+{
+
+ if (ctx->config == NULL)
+ return;
+
+ free(ctx->config, M_HWT_CONFIG);
+
+ ctx->config = NULL;
+}
diff --git a/sys/dev/hwt/hwt_config.h b/sys/dev/hwt/hwt_config.h
new file mode 100644
index 000000000000..47485583063c
--- /dev/null
+++ b/sys/dev/hwt/hwt_config.h
@@ -0,0 +1,36 @@
+/*-
+ * Copyright (c) 2023-2025 Ruslan Bukin <b...@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#ifndef _DEV_HWT_HWT_CONFIG_H_
+#define _DEV_HWT_HWT_CONFIG_H_
+
+int hwt_config_set(struct thread *td, struct hwt_context *ctx,
+ struct hwt_set_config *sconf);
+void hwt_config_free(struct hwt_context *ctx);
+
+#endif /* !_DEV_HWT_HWT_CONFIG_H_ */
diff --git a/sys/dev/hwt/hwt_context.c b/sys/dev/hwt/hwt_context.c
new file mode 100644
index 000000000000..9af76cffc928
--- /dev/null
+++ b/sys/dev/hwt/hwt_context.c
@@ -0,0 +1,201 @@
+/*-
+ * SPDX-License-Identifier: BSD-2-Clause
+ *
+ * Copyright (c) 2023-2025 Ruslan Bukin <b...@bsdpad.com>
+ *
+ * This work was supported by Innovate UK project 105694, "Digital Security
+ * by Design (DSbD) Technology Platform Prototype".
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/param.h>
+#include <sys/bitstring.h>
+#include <sys/conf.h>
+#include <sys/proc.h>
+#include <sys/kernel.h>
+#include <sys/malloc.h>
+#include <sys/mman.h>
+#include <sys/mutex.h>
+#include <sys/refcount.h>
+#include <sys/rwlock.h>
+#include <sys/hwt.h>
+
+#include <dev/hwt/hwt_hook.h>
+#include <dev/hwt/hwt_context.h>
+#include <dev/hwt/hwt_config.h>
+#include <dev/hwt/hwt_thread.h>
+#include <dev/hwt/hwt_owner.h>
+#include <dev/hwt/hwt_vm.h>
+#include <dev/hwt/hwt_cpu.h>
+
+#define HWT_DEBUG
+#undef HWT_DEBUG
+
+#ifdef HWT_DEBUG
+#define dprintf(fmt, ...) printf(fmt, ##__VA_ARGS__)
+#else
+#define dprintf(fmt, ...)
+#endif
+
+static MALLOC_DEFINE(M_HWT_CTX, "hwt_ctx", "Hardware Trace");
+
+static bitstr_t *ident_set;
+static int ident_set_size;
+static struct mtx ident_set_mutex;
+
+static int
+hwt_ctx_ident_alloc(int *new_ident)
+{
+
+ mtx_lock(&ident_set_mutex);
+ bit_ffc(ident_set, ident_set_size, new_ident);
+ if (*new_ident == -1) {
+ mtx_unlock(&ident_set_mutex);
+ return (ENOMEM);
+ }
+ bit_set(ident_set, *new_ident);
+ mtx_unlock(&ident_set_mutex);
+
+ return (0);
+}
*** 3705 LINES SKIPPED ***
