The branch main has been updated by imp: URL: https://cgit.FreeBSD.org/src/commit/?id=e51ef8ae490fc9f73191f33e7ad388c2511c454a
commit e51ef8ae490fc9f73191f33e7ad388c2511c454a Author: Ali Mashtizadeh <[email protected]> AuthorDate: 2026-01-30 09:12:17 +0000 Commit: Warner Losh <[email protected]> CommitDate: 2026-02-27 21:22:16 +0000 hwpmc: Initial support for AMD IBS This patch adds support for AMD IBS. It adds a new class of performance counter that cotains two events: ibs-fetch and ibs-op events. Unlike most existing sampled events, IBS events provide a number of values containing extra information regarding the sample. To support this we use the existing callchain event, and introduce a new flag for multipart payloads. The first 8 bytes of the pc_sample contains a header that defines up to four payloads. Sponsored by: Netflix Reviewed by: imp,mhorne Pull Request: https://github.com/freebsd/freebsd-src/pull/2022 --- lib/libpmc/libpmc.c | 64 ++++- sys/amd64/include/pmc_mdep.h | 5 + sys/conf/files.x86 | 1 + sys/dev/hwpmc/hwpmc_amd.c | 25 +- sys/dev/hwpmc/hwpmc_ibs.c | 614 +++++++++++++++++++++++++++++++++++++++++ sys/dev/hwpmc/hwpmc_ibs.h | 176 ++++++++++++ sys/dev/hwpmc/hwpmc_mod.c | 96 ++++++- sys/dev/hwpmc/pmc_events.h | 13 +- sys/i386/include/pmc_mdep.h | 5 + sys/modules/hwpmc/Makefile | 4 +- sys/sys/pmc.h | 19 +- sys/sys/pmclog.h | 14 + sys/x86/x86/local_apic.c | 27 +- usr.sbin/pmcstat/pmcstat_log.c | 99 ++++++- 14 files changed, 1130 insertions(+), 32 deletions(-) diff --git a/lib/libpmc/libpmc.c b/lib/libpmc/libpmc.c index 10e357f55935..155da7cf6a7b 100644 --- a/lib/libpmc/libpmc.c +++ b/lib/libpmc/libpmc.c @@ -50,8 +50,8 @@ #if defined(__amd64__) || defined(__i386__) static int k8_allocate_pmc(enum pmc_event _pe, char *_ctrspec, struct pmc_op_pmcallocate *_pmc_config); -#endif -#if defined(__amd64__) || defined(__i386__) +static int ibs_allocate_pmc(enum pmc_event _pe, char *_ctrspec, + struct pmc_op_pmcallocate *_pmc_config); static int tsc_allocate_pmc(enum pmc_event _pe, char *_ctrspec, struct pmc_op_pmcallocate *_pmc_config); #endif @@ -132,6 +132,7 @@ struct pmc_class_descr { PMC_CLASSDEP_TABLE(iaf, IAF); PMC_CLASSDEP_TABLE(k8, K8); +PMC_CLASSDEP_TABLE(ibs, IBS); PMC_CLASSDEP_TABLE(armv7, ARMV7); PMC_CLASSDEP_TABLE(armv8, ARMV8); PMC_CLASSDEP_TABLE(cmn600_pmu, CMN600_PMU); @@ -201,8 +202,7 @@ static const struct pmc_class_descr NAME##_class_table_descr = \ #if defined(__i386__) || defined(__amd64__) PMC_CLASS_TABLE_DESC(k8, K8, k8, k8); -#endif -#if defined(__i386__) || defined(__amd64__) +PMC_CLASS_TABLE_DESC(ibs, IBS, ibs, ibs); PMC_CLASS_TABLE_DESC(tsc, TSC, tsc, tsc); #endif #if defined(__arm__) @@ -691,9 +691,49 @@ k8_allocate_pmc(enum pmc_event pe, char *ctrspec, return (0); } -#endif +static int +ibs_allocate_pmc(enum pmc_event pe, char *ctrspec, + struct pmc_op_pmcallocate *pmc_config) +{ + char *e, *p, *q; + uint64_t ctl; + + pmc_config->pm_caps |= + (PMC_CAP_SYSTEM | PMC_CAP_EDGE | PMC_CAP_PRECISE); + pmc_config->pm_md.pm_ibs.ibs_ctl = 0; + + /* setup parsing tables */ + switch (pe) { + case PMC_EV_IBS_FETCH: + pmc_config->pm_md.pm_ibs.ibs_type = IBS_PMC_FETCH; + break; + case PMC_EV_IBS_OP: + pmc_config->pm_md.pm_ibs.ibs_type = IBS_PMC_OP; + break; + default: + return (-1); + } + + /* parse parameters */ + while ((p = strsep(&ctrspec, ",")) != NULL) { + if (KWPREFIXMATCH(p, "ctl=")) { + q = strchr(p, '='); + if (*++q == '\0') /* skip '=' */ + return (-1); + + ctl = strtoull(q, &e, 0); + if (e == q || *e != '\0') + return (-1); + + pmc_config->pm_md.pm_ibs.ibs_ctl |= ctl; + } else { + return (-1); + } + } + + return (0); +} -#if defined(__i386__) || defined(__amd64__) static int tsc_allocate_pmc(enum pmc_event pe, char *ctrspec, struct pmc_op_pmcallocate *pmc_config) @@ -1268,6 +1308,10 @@ pmc_event_names_of_class(enum pmc_class cl, const char ***eventnames, ev = k8_event_table; count = PMC_EVENT_TABLE_SIZE(k8); break; + case PMC_CLASS_IBS: + ev = ibs_event_table; + count = PMC_EVENT_TABLE_SIZE(ibs); + break; case PMC_CLASS_ARMV7: switch (cpu_info.pm_cputype) { default: @@ -1471,6 +1515,10 @@ pmc_init(void) case PMC_CLASS_K8: pmc_class_table[n++] = &k8_class_table_descr; break; + + case PMC_CLASS_IBS: + pmc_class_table[n++] = &ibs_class_table_descr; + break; #endif case PMC_CLASS_SOFT: @@ -1676,7 +1724,9 @@ _pmc_name_of_event(enum pmc_event pe, enum pmc_cputype cpu) if (pe >= PMC_EV_K8_FIRST && pe <= PMC_EV_K8_LAST) { ev = k8_event_table; evfence = k8_event_table + PMC_EVENT_TABLE_SIZE(k8); - + } else if (pe >= PMC_EV_IBS_FIRST && pe <= PMC_EV_IBS_LAST) { + ev = ibs_event_table; + evfence = ibs_event_table + PMC_EVENT_TABLE_SIZE(ibs); } else if (pe >= PMC_EV_ARMV7_FIRST && pe <= PMC_EV_ARMV7_LAST) { switch (cpu) { case PMC_CPU_ARMV7_CORTEX_A8: diff --git a/sys/amd64/include/pmc_mdep.h b/sys/amd64/include/pmc_mdep.h index 5c20d8473855..24b785312a16 100644 --- a/sys/amd64/include/pmc_mdep.h +++ b/sys/amd64/include/pmc_mdep.h @@ -41,6 +41,7 @@ struct pmc_mdep; #include <dev/hwpmc/hwpmc_amd.h> #include <dev/hwpmc/hwpmc_core.h> +#include <dev/hwpmc/hwpmc_ibs.h> #include <dev/hwpmc/hwpmc_tsc.h> #include <dev/hwpmc/hwpmc_uncore.h> @@ -51,6 +52,7 @@ struct pmc_mdep; */ #define PMC_MDEP_CLASS_INDEX_TSC 1 #define PMC_MDEP_CLASS_INDEX_K8 2 +#define PMC_MDEP_CLASS_INDEX_IBS 3 #define PMC_MDEP_CLASS_INDEX_P4 2 #define PMC_MDEP_CLASS_INDEX_IAP 2 #define PMC_MDEP_CLASS_INDEX_IAF 3 @@ -62,6 +64,7 @@ struct pmc_mdep; * * TSC The timestamp counter * K8 AMD Athlon64 and Opteron PMCs in 64 bit mode. + * IBS AMD IBS * PIV Intel P4/HTT and P4/EMT64 * IAP Intel Core/Core2/Atom CPUs in 64 bits mode. * IAF Intel fixed-function PMCs in Core2 and later CPUs. @@ -71,6 +74,7 @@ struct pmc_mdep; union pmc_md_op_pmcallocate { struct pmc_md_amd_op_pmcallocate pm_amd; + struct pmc_md_ibs_op_pmcallocate pm_ibs; struct pmc_md_iap_op_pmcallocate pm_iap; struct pmc_md_ucf_op_pmcallocate pm_ucf; struct pmc_md_ucp_op_pmcallocate pm_ucp; @@ -85,6 +89,7 @@ union pmc_md_op_pmcallocate { union pmc_md_pmc { struct pmc_md_amd_pmc pm_amd; + struct pmc_md_ibs_pmc pm_ibs; struct pmc_md_iaf_pmc pm_iaf; struct pmc_md_iap_pmc pm_iap; struct pmc_md_ucf_pmc pm_ucf; diff --git a/sys/conf/files.x86 b/sys/conf/files.x86 index fabcd5d9ebe5..8a7e0b78feb4 100644 --- a/sys/conf/files.x86 +++ b/sys/conf/files.x86 @@ -114,6 +114,7 @@ dev/hptrr/hptrr_osm_bsd.c optional hptrr dev/hptrr/hptrr_config.c optional hptrr dev/hptrr/$M-elf.hptrr_lib.o optional hptrr dev/hwpmc/hwpmc_amd.c optional hwpmc +dev/hwpmc/hwpmc_ibs.c optional hwpmc dev/hwpmc/hwpmc_intel.c optional hwpmc dev/hwpmc/hwpmc_core.c optional hwpmc dev/hwpmc/hwpmc_uncore.c optional hwpmc diff --git a/sys/dev/hwpmc/hwpmc_amd.c b/sys/dev/hwpmc/hwpmc_amd.c index b34cbffcffa8..cf44f9362a72 100644 --- a/sys/dev/hwpmc/hwpmc_amd.c +++ b/sys/dev/hwpmc/hwpmc_amd.c @@ -543,6 +543,10 @@ amd_intr(struct trapframe *tf) pac = amd_pcpu[cpu]; + retval = pmc_ibs_intr(tf); + if (retval) + goto done; + /* * look for all PMCs that have interrupted: * - look for a running, sampling PMC which has overflowed @@ -613,6 +617,7 @@ amd_intr(struct trapframe *tf) } } +done: if (retval) counter_u64_add(pmc_stats.pm_intr_processed, 1); else @@ -760,7 +765,7 @@ pmc_amd_initialize(void) struct pmc_classdep *pcd; struct pmc_mdep *pmc_mdep; enum pmc_cputype cputype; - int error, i, ncpus; + int error, i, ncpus, nclasses; int family, model, stepping; int amd_core_npmcs, amd_l3_npmcs, amd_df_npmcs; struct amd_descr *d; @@ -884,10 +889,16 @@ pmc_amd_initialize(void) M_WAITOK | M_ZERO); /* - * These processors have two classes of PMCs: the TSC and - * programmable PMCs. + * These processors have two or three classes of PMCs: the TSC, + * programmable PMCs, and AMD IBS. */ - pmc_mdep = pmc_mdep_alloc(2); + if ((amd_feature2 & AMDID2_IBS) != 0) { + nclasses = 3; + } else { + nclasses = 2; + } + + pmc_mdep = pmc_mdep_alloc(nclasses); ncpus = pmc_cpu_max(); @@ -927,6 +938,12 @@ pmc_amd_initialize(void) PMCDBG0(MDP, INI, 0, "amd-initialize"); + if (nclasses >= 3) { + error = pmc_ibs_initialize(pmc_mdep, ncpus); + if (error != 0) + goto error; + } + return (pmc_mdep); error: diff --git a/sys/dev/hwpmc/hwpmc_ibs.c b/sys/dev/hwpmc/hwpmc_ibs.c new file mode 100644 index 000000000000..66d3260cf040 --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_ibs.c @@ -0,0 +1,614 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026, Ali Jose Mashtizadeh + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* Support for the AMD IBS */ + +#include <sys/param.h> +#include <sys/lock.h> +#include <sys/malloc.h> +#include <sys/mutex.h> +#include <sys/pcpu.h> +#include <sys/pmc.h> +#include <sys/pmckern.h> +#include <sys/pmclog.h> +#include <sys/smp.h> +#include <sys/systm.h> + +#include <machine/cpu.h> +#include <machine/cpufunc.h> +#include <machine/md_var.h> +#include <machine/specialreg.h> + +#define IBS_STOP_ITER 50 /* Stopping iterations */ + +/* AMD IBS PMCs */ +struct ibs_descr { + struct pmc_descr pm_descr; /* "base class" */ +}; + +/* + * Globals + */ +static uint64_t ibs_features; + +/* + * Per-processor information + */ +#define IBS_CPU_RUNNING 1 +#define IBS_CPU_STOPPING 2 +#define IBS_CPU_STOPPED 3 + +struct ibs_cpu { + int pc_status; + struct pmc_hw pc_ibspmcs[IBS_NPMCS]; +}; +static struct ibs_cpu **ibs_pcpu; + +/* + * Read a PMC value from the MSR. + */ +static int +ibs_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < IBS_NPMCS, + ("[ibs,%d] illegal row-index %d", __LINE__, ri)); + KASSERT(ibs_pcpu[cpu], + ("[ibs,%d] null per-cpu, cpu %d", __LINE__, cpu)); + + /* read the IBS ctl */ + switch (ri) { + case IBS_PMC_FETCH: + *v = rdmsr(IBS_FETCH_CTL); + break; + case IBS_PMC_OP: + *v = rdmsr(IBS_OP_CTL); + break; + } + + PMCDBG2(MDP, REA, 2, "ibs-read id=%d -> %jd", ri, *v); + + return (0); +} + +/* + * Write a PMC MSR. + */ +static int +ibs_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v) +{ + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < IBS_NPMCS, + ("[ibs,%d] illegal row-index %d", __LINE__, ri)); + + PMCDBG3(MDP, WRI, 1, "ibs-write cpu=%d ri=%d v=%jx", cpu, ri, v); + + return (0); +} + +/* + * Configure hardware PMC according to the configuration recorded in 'pm'. + */ +static int +ibs_config_pmc(int cpu, int ri, struct pmc *pm) +{ + struct pmc_hw *phw; + + PMCDBG3(MDP, CFG, 1, "cpu=%d ri=%d pm=%p", cpu, ri, pm); + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < IBS_NPMCS, + ("[ibs,%d] illegal row-index %d", __LINE__, ri)); + + phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; + + KASSERT(pm == NULL || phw->phw_pmc == NULL, + ("[ibs,%d] pm=%p phw->pm=%p hwpmc not unconfigured", + __LINE__, pm, phw->phw_pmc)); + + phw->phw_pmc = pm; + + return (0); +} + +/* + * Retrieve a configured PMC pointer from hardware state. + */ +static int +ibs_get_config(int cpu, int ri, struct pmc **ppm) +{ + + *ppm = ibs_pcpu[cpu]->pc_ibspmcs[ri].phw_pmc; + + return (0); +} + +/* + * Check if a given PMC allocation is feasible. + */ +static int +ibs_allocate_pmc(int cpu __unused, int ri, struct pmc *pm, + const struct pmc_op_pmcallocate *a) +{ + uint64_t caps, config; + + KASSERT(ri >= 0 && ri < IBS_NPMCS, + ("[ibs,%d] illegal row index %d", __LINE__, ri)); + + /* check class match */ + if (a->pm_class != PMC_CLASS_IBS) + return (EINVAL); + if (a->pm_md.pm_ibs.ibs_type != ri) + return (EINVAL); + + caps = pm->pm_caps; + + PMCDBG2(MDP, ALL, 1, "ibs-allocate ri=%d caps=0x%x", ri, caps); + + if ((caps & PMC_CAP_SYSTEM) == 0) + return (EINVAL); + + config = a->pm_md.pm_ibs.ibs_ctl; + pm->pm_md.pm_ibs.ibs_ctl = config; + + PMCDBG2(MDP, ALL, 2, "ibs-allocate ri=%d -> config=0x%x", ri, config); + + return (0); +} + +/* + * Release machine dependent state associated with a PMC. This is a + * no-op on this architecture. + */ +static int +ibs_release_pmc(int cpu, int ri, struct pmc *pmc __unused) +{ + struct pmc_hw *phw __diagused; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < IBS_NPMCS, + ("[ibs,%d] illegal row-index %d", __LINE__, ri)); + + PMCDBG1(MDP, ALL, 1, "ibs-release ri=%d", ri); + + phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; + + KASSERT(phw->phw_pmc == NULL, + ("[ibs,%d] PHW pmc %p non-NULL", __LINE__, phw->phw_pmc)); + + return (0); +} + +/* + * Start a PMC. + */ +static int +ibs_start_pmc(int cpu __diagused, int ri, struct pmc *pm) +{ + uint64_t config; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < IBS_NPMCS, + ("[ibs,%d] illegal row-index %d", __LINE__, ri)); + + PMCDBG2(MDP, STA, 1, "ibs-start cpu=%d ri=%d", cpu, ri); + + /* + * This is used to handle spurious NMIs. All that matters is that it + * is not in the stopping state. + */ + atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_RUNNING); + + /* + * Turn on the ENABLE bit. Zeroing out the control register eliminates + * stale valid bits from spurious NMIs and it resets the counter. + */ + switch (ri) { + case IBS_PMC_FETCH: + wrmsr(IBS_FETCH_CTL, 0); + config = pm->pm_md.pm_ibs.ibs_ctl | IBS_FETCH_CTL_ENABLE; + wrmsr(IBS_FETCH_CTL, config); + break; + case IBS_PMC_OP: + wrmsr(IBS_OP_CTL, 0); + config = pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE; + wrmsr(IBS_OP_CTL, config); + break; + } + + return (0); +} + +/* + * Stop a PMC. + */ +static int +ibs_stop_pmc(int cpu __diagused, int ri, struct pmc *pm) +{ + int i; + uint64_t config; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] illegal CPU value %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < IBS_NPMCS, + ("[ibs,%d] illegal row-index %d", __LINE__, ri)); + + PMCDBG1(MDP, STO, 1, "ibs-stop ri=%d", ri); + + /* + * Turn off the ENABLE bit, but unfortunately there are a few quirks + * that generate excess NMIs. Workaround #420 in the Revision Guide + * for AMD Family 10h Processors 41322 Rev. 3.92 March 2012. requires + * that we clear the count before clearing enable. + * + * Even after clearing the counter spurious NMIs are still possible so + * we use a per-CPU atomic variable to notify the interrupt handler we + * are stopping and discard spurious NMIs. We then retry clearing the + * control register for 50us. This gives us enough time and ensures + * that the valid bit is not accidently stuck after a spurious NMI. + */ + config = pm->pm_md.pm_ibs.ibs_ctl; + + atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPING); + + switch (ri) { + case IBS_PMC_FETCH: + wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_MAXCNTMASK); + DELAY(1); + config &= ~IBS_FETCH_CTL_ENABLE; + wrmsr(IBS_FETCH_CTL, config); + break; + case IBS_PMC_OP: + wrmsr(IBS_FETCH_CTL, config & ~IBS_FETCH_CTL_MAXCNTMASK); + DELAY(1); + config &= ~IBS_OP_CTL_ENABLE; + wrmsr(IBS_OP_CTL, config); + break; + } + + for (i = 0; i < IBS_STOP_ITER; i++) { + DELAY(1); + + switch (ri) { + case IBS_PMC_FETCH: + wrmsr(IBS_FETCH_CTL, 0); + break; + case IBS_PMC_OP: + wrmsr(IBS_OP_CTL, 0); + break; + } + } + + atomic_store_int(&ibs_pcpu[cpu]->pc_status, IBS_CPU_STOPPED); + + return (0); +} + +static void +pmc_ibs_process_fetch(struct pmc *pm, struct trapframe *tf, uint64_t config) +{ + struct pmc_multipart mpd; + + if (pm == NULL) + return; + + if (pm->pm_state != PMC_STATE_RUNNING) + return; + + memset(&mpd, 0, sizeof(mpd)); + + mpd.pl_type = PMC_CC_MULTIPART_IBS_FETCH; + mpd.pl_length = 4; + mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config; + if (ibs_features) { + mpd.pl_mpdata[PMC_MPIDX_FETCH_EXTCTL] = rdmsr(IBS_FETCH_EXTCTL); + } + mpd.pl_mpdata[PMC_MPIDX_FETCH_CTL] = config; + mpd.pl_mpdata[PMC_MPIDX_FETCH_LINADDR] = rdmsr(IBS_FETCH_LINADDR); + if ((config & IBS_FETCH_CTL_PHYSADDRVALID) != 0) { + mpd.pl_mpdata[PMC_MPIDX_FETCH_PHYSADDR] = + rdmsr(IBS_FETCH_PHYSADDR); + } + + pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd); +} + +static void +pmc_ibs_process_op(struct pmc *pm, struct trapframe *tf, uint64_t config) +{ + struct pmc_multipart mpd; + + if (pm == NULL) + return; + + if (pm->pm_state != PMC_STATE_RUNNING) + return; + + memset(&mpd, 0, sizeof(mpd)); + + mpd.pl_type = PMC_CC_MULTIPART_IBS_OP; + mpd.pl_length = 8; + mpd.pl_mpdata[PMC_MPIDX_OP_CTL] = config; + mpd.pl_mpdata[PMC_MPIDX_OP_RIP] = rdmsr(IBS_OP_RIP); + mpd.pl_mpdata[PMC_MPIDX_OP_DATA] = rdmsr(IBS_OP_DATA); + mpd.pl_mpdata[PMC_MPIDX_OP_DATA2] = rdmsr(IBS_OP_DATA2); + mpd.pl_mpdata[PMC_MPIDX_OP_DATA3] = rdmsr(IBS_OP_DATA3); + mpd.pl_mpdata[PMC_MPIDX_OP_DC_LINADDR] = rdmsr(IBS_OP_DC_LINADDR); + mpd.pl_mpdata[PMC_MPIDX_OP_DC_PHYSADDR] = rdmsr(IBS_OP_DC_PHYSADDR); + + pmc_process_interrupt_mp(PMC_HR, pm, tf, &mpd); + + wrmsr(IBS_OP_CTL, pm->pm_md.pm_ibs.ibs_ctl | IBS_OP_CTL_ENABLE); +} + +/* + * Interrupt handler. This function needs to return '1' if the + * interrupt was this CPU's PMCs or '0' otherwise. It is not allowed + * to sleep or do anything a 'fast' interrupt handler is not allowed + * to do. + */ +int +pmc_ibs_intr(struct trapframe *tf) +{ + struct ibs_cpu *pac; + struct pmc *pm; + int retval, cpu; + uint64_t config; + + cpu = curcpu; + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] out of range CPU %d", __LINE__, cpu)); + + PMCDBG3(MDP, INT, 1, "cpu=%d tf=%p um=%d", cpu, tf, TRAPF_USERMODE(tf)); + + retval = 0; + + pac = ibs_pcpu[cpu]; + + config = rdmsr(IBS_FETCH_CTL); + if ((config & IBS_FETCH_CTL_VALID) != 0) { + pm = pac->pc_ibspmcs[IBS_PMC_FETCH].phw_pmc; + + retval = 1; + + pmc_ibs_process_fetch(pm, tf, config); + } + + config = rdmsr(IBS_OP_CTL); + if ((retval == 0) && ((config & IBS_OP_CTL_VALID) != 0)) { + pm = pac->pc_ibspmcs[IBS_PMC_OP].phw_pmc; + + retval = 1; + + pmc_ibs_process_op(pm, tf, config); + } + + if (retval == 0) { + // Lets check for a stray NMI when stopping + if (atomic_load_int(&pac->pc_status) == IBS_CPU_STOPPING) { + return (1); + } + } + + + if (retval) + counter_u64_add(pmc_stats.pm_intr_processed, 1); + else + counter_u64_add(pmc_stats.pm_intr_ignored, 1); + + PMCDBG1(MDP, INT, 2, "retval=%d", retval); + + return (retval); +} + +/* + * Describe a PMC. + */ +static int +ibs_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc) +{ + struct pmc_hw *phw; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] illegal CPU %d", __LINE__, cpu)); + KASSERT(ri >= 0 && ri < IBS_NPMCS, + ("[ibs,%d] row-index %d out of range", __LINE__, ri)); + + phw = &ibs_pcpu[cpu]->pc_ibspmcs[ri]; + + if (ri == IBS_PMC_FETCH) { + strlcpy(pi->pm_name, "IBS-FETCH", sizeof(pi->pm_name)); + pi->pm_class = PMC_CLASS_IBS; + pi->pm_enabled = true; + *ppmc = phw->phw_pmc; + } else { + strlcpy(pi->pm_name, "IBS-OP", sizeof(pi->pm_name)); + pi->pm_class = PMC_CLASS_IBS; + pi->pm_enabled = true; + *ppmc = phw->phw_pmc; + } + + return (0); +} + +/* + * Processor-dependent initialization. + */ +static int +ibs_pcpu_init(struct pmc_mdep *md, int cpu) +{ + struct ibs_cpu *pac; + struct pmc_cpu *pc; + struct pmc_hw *phw; + int first_ri, n; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] insane cpu number %d", __LINE__, cpu)); + + PMCDBG1(MDP, INI, 1, "ibs-init cpu=%d", cpu); + + ibs_pcpu[cpu] = pac = malloc(sizeof(struct ibs_cpu), M_PMC, + M_WAITOK | M_ZERO); + + /* + * Set the content of the hardware descriptors to a known + * state and initialize pointers in the MI per-cpu descriptor. + */ + pc = pmc_pcpu[cpu]; + first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri; + + KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu pointer", __LINE__)); + + for (n = 0, phw = pac->pc_ibspmcs; n < IBS_NPMCS; n++, phw++) { + phw->phw_state = PMC_PHW_FLAG_IS_ENABLED | + PMC_PHW_CPU_TO_STATE(cpu) | PMC_PHW_INDEX_TO_STATE(n); + phw->phw_pmc = NULL; + pc->pc_hwpmcs[n + first_ri] = phw; + } + + return (0); +} + +/* + * Processor-dependent cleanup prior to the KLD being unloaded. + */ +static int +ibs_pcpu_fini(struct pmc_mdep *md, int cpu) +{ + struct ibs_cpu *pac; + struct pmc_cpu *pc; + int first_ri, i; + + KASSERT(cpu >= 0 && cpu < pmc_cpu_max(), + ("[ibs,%d] insane cpu number (%d)", __LINE__, cpu)); + + PMCDBG1(MDP, INI, 1, "ibs-cleanup cpu=%d", cpu); + + /* + * Turn off IBS. + */ + wrmsr(IBS_FETCH_CTL, 0); + wrmsr(IBS_OP_CTL, 0); + + /* + * Free up allocated space. + */ + if ((pac = ibs_pcpu[cpu]) == NULL) + return (0); + + ibs_pcpu[cpu] = NULL; + + pc = pmc_pcpu[cpu]; + KASSERT(pc != NULL, ("[ibs,%d] NULL per-cpu state", __LINE__)); + + first_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS].pcd_ri; + + /* + * Reset pointers in the MI 'per-cpu' state. + */ + for (i = 0; i < IBS_NPMCS; i++) + pc->pc_hwpmcs[i + first_ri] = NULL; + + free(pac, M_PMC); + + return (0); +} + +/* + * Initialize ourselves. + */ +int +pmc_ibs_initialize(struct pmc_mdep *pmc_mdep, int ncpus) +{ + u_int regs[4]; + struct pmc_classdep *pcd; + + /* + * Allocate space for pointers to PMC HW descriptors and for + * the MDEP structure used by MI code. + */ + ibs_pcpu = malloc(sizeof(struct ibs_cpu *) * pmc_cpu_max(), M_PMC, + M_WAITOK | M_ZERO); + + /* Initialize AMD IBS handling. */ + pcd = &pmc_mdep->pmd_classdep[PMC_MDEP_CLASS_INDEX_IBS]; + + pcd->pcd_caps = IBS_PMC_CAPS; + pcd->pcd_class = PMC_CLASS_IBS; + pcd->pcd_num = IBS_NPMCS; + pcd->pcd_ri = pmc_mdep->pmd_npmc; + pcd->pcd_width = 0; + + pcd->pcd_allocate_pmc = ibs_allocate_pmc; + pcd->pcd_config_pmc = ibs_config_pmc; + pcd->pcd_describe = ibs_describe; + pcd->pcd_get_config = ibs_get_config; + pcd->pcd_pcpu_fini = ibs_pcpu_fini; + pcd->pcd_pcpu_init = ibs_pcpu_init; + pcd->pcd_release_pmc = ibs_release_pmc; + pcd->pcd_start_pmc = ibs_start_pmc; + pcd->pcd_stop_pmc = ibs_stop_pmc; + pcd->pcd_read_pmc = ibs_read_pmc; + pcd->pcd_write_pmc = ibs_write_pmc; + + pmc_mdep->pmd_npmc += IBS_NPMCS; + + if (cpu_exthigh >= CPUID_IBSID) { + do_cpuid(CPUID_IBSID, regs); + ibs_features = regs[0]; + } else { + ibs_features = 0; + } + + PMCDBG0(MDP, INI, 0, "ibs-initialize"); + + return (0); +} + +/* + * Finalization code for AMD CPUs. + */ +void +pmc_ibs_finalize(struct pmc_mdep *md) +{ + PMCDBG0(MDP, INI, 1, "ibs-finalize"); + + for (int i = 0; i < pmc_cpu_max(); i++) + KASSERT(ibs_pcpu[i] == NULL, + ("[ibs,%d] non-null pcpu cpu %d", __LINE__, i)); + + free(ibs_pcpu, M_PMC); + ibs_pcpu = NULL; +} diff --git a/sys/dev/hwpmc/hwpmc_ibs.h b/sys/dev/hwpmc/hwpmc_ibs.h new file mode 100644 index 000000000000..4449b44c8368 --- /dev/null +++ b/sys/dev/hwpmc/hwpmc_ibs.h @@ -0,0 +1,176 @@ +/*- + * SPDX-License-Identifier: BSD-2-Clause + * + * Copyright (c) 2026, Ali Jose Mashtizadeh + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#ifndef _DEV_HWPMC_IBS_H_ +#define _DEV_HWPMC_IBS_H_ 1 + +#define IBS_NPMCS 2 +#define IBS_PMC_FETCH 0 +#define IBS_PMC_OP 1 + +/* + * All of the CPUID definitions come from AMD PPR Vol 1 for AMD Family 1Ah + * Model 02h C1 (57238) 2024-09-29 Revision 0.24. + */ +#define CPUID_IBSID 0x8000001B +#define CPUID_IBSID_IBSFFV 0x00000001 /* IBS Feature Flags Valid */ +#define CPUID_IBSID_FETCHSAM 0x00000002 /* IBS Fetch Sampling */ +#define CPUID_IBSID_OPSAM 0x00000004 /* IBS Execution Sampling */ +#define CPUID_IBSID_RDWROPCNT 0x00000008 /* RdWr Operationg Counter */ +#define CPUID_IBSID_OPCNT 0x00000010 /* Operation Counter */ +#define CPUID_IBSID_BRNTRGT 0x00000020 /* Branch Target Address */ +#define CPUID_IBSID_OPCNTEXT 0x00000040 /* Extend Counter */ +#define CPUID_IBSID_RIPINVALIDCHK 0x00000080 /* Invalid RIP Indication */ +#define CPUID_IBSID_OPFUSE 0x00000010 /* Fused Branch Operation */ +#define CPUID_IBSID_IBSFETCHCTLEXTD 0x00000020 /* IBS Fetch Control Ext */ +#define CPUID_IBSID_IBSOPDATA4 0x00000040 /* IBS OP DATA4 */ +#define CPUID_IBSID_ZEN4IBSEXTENSIONS 0x00000080 /* IBS Zen 4 Extensions */ +#define CPUID_IBSID_IBSLOADLATENCYFILT 0x00000100 /* Load Latency Filtering */ +#define CPUID_IBSID_IBSUPDTDDTLBSTATS 0x00080000 /* Simplified DTLB Stats */ + +/* + * All of these definitions here come from AMD64 Architecture Programmer's + * Manual Volume 2: System Programming (24593) 2025-07-02 Version 3.43. with + * the following exceptions: + * + * OpData4 and fields come from the BKDG for AMD Family 15h Model 70-7Fh + * (55072) 2018-06-20 Revision 3.09. + */ + +/* IBS MSRs */ +#define IBS_CTL 0xC001103A /* IBS Control */ +#define IBS_CTL_LVTOFFSETVALID (1ULL << 8) +#define IBS_CTL_LVTOFFSETMASK 0x0000000F + +/* IBS Fetch Control */ +#define IBS_FETCH_CTL 0xC0011030 /* IBS Fetch Control */ +#define IBS_FETCH_CTL_L3MISS (1ULL << 61) /* L3 Cache Miss */ +#define IBS_FETCH_CTL_OPCACHEMISS (1ULL << 60) /* Op Cache Miss */ +#define IBS_FETCH_CTL_L3MISSONLY (1ULL << 59) /* L3 Miss Filtering */ +#define IBS_FETCH_CTL_RANDOMIZE (1ULL << 57) /* Randomized Tagging */ +#define IBS_FETCH_CTL_L1TLBMISS (1ULL << 55) /* L1 TLB Miss */ +// Page size 54:53 +#define IBS_FETCH_CTL_PHYSADDRVALID (1ULL << 52) /* PHYSADDR Valid */ +#define IBS_FETCH_CTL_ICMISS (1ULL << 51) /* Inst. Cache Miss */ +#define IBS_FETCH_CTL_COMPLETE (1ULL << 50) /* Complete */ *** 650 LINES SKIPPED ***
