Arm MPAM systems running resctrl expose memory bandwidth differently
from x86:
- MB control/monitor domains are keyed by NUMA node id, which need
not match the L3 cache id (on a Grace/Vera system L3 ids are {1,2}
while MB domains are {0,1,2,10,...}).
- Monitoring lives in a separate MB_MON resource that provides only
mbm_total_bytes; there is no local/total split and no L3_MON
bandwidth event.
- In "mbm_event" (ABMC) assignment mode a hardware counter must be
explicitly assigned to a group+domain+event before the bandwidth
file returns a value (it reads "Unassigned" otherwise), and
removing a group does not release its counter, so it must be
unassigned explicitly or it leaks.
Teach the test framework this layout:
- get_domain_id() resolves MB domains via the CPU's NUMA node when
the MB_MON resource exists. x86 (no MB_MON) is unaffected.
- initialize_mem_bw_resctrl() falls back to
mon_MB_<domain>/mbm_total_bytes when mbm_local_bytes is absent,
and assigns an ABMC counter when the group's mbm_MB_assignments
file is present. Note the assignment parser requires a trailing
newline; without it the write fails with EINVAL.
- mem_bw_ref_cleanup() unassigns the counter again.
- The MBA feature check accepts MB_MON's mbm_total_bytes as the
monitoring source.
Comparing a reference PMU total (read+write) with mbm_total_bytes is
a like-for-like comparison; the iMC path keeps comparing read
bandwidth with mbm_local_bytes as before.
No effect yet on any vendor: the MBA test is still gated to Intel.
Signed-off-by: Richard Cheng <[email protected]>
---
tools/testing/selftests/resctrl/mba_test.c | 3 +-
tools/testing/selftests/resctrl/resctrl_val.c | 69 +++++++++++++++++--
tools/testing/selftests/resctrl/resctrlfs.c | 45 +++++++++++-
3 files changed, 111 insertions(+), 6 deletions(-)
diff --git a/tools/testing/selftests/resctrl/mba_test.c
b/tools/testing/selftests/resctrl/mba_test.c
index 8b891d3e6650..6c85107c32ee 100644
--- a/tools/testing/selftests/resctrl/mba_test.c
+++ b/tools/testing/selftests/resctrl/mba_test.c
@@ -211,7 +211,8 @@ static int mba_run_test(const struct resctrl_test *test,
const struct user_param
static bool mba_feature_check(const struct resctrl_test *test)
{
return test_resource_feature_check(test) &&
- resctrl_mon_feature_exists("L3_MON", "mbm_local_bytes") &&
+ (resctrl_mon_feature_exists("L3_MON", "mbm_local_bytes") ||
+ resctrl_mon_feature_exists("MB_MON", "mbm_total_bytes")) &&
mem_bw_ref_available();
}
diff --git a/tools/testing/selftests/resctrl/resctrl_val.c
b/tools/testing/selftests/resctrl/resctrl_val.c
index a2b919bd4839..0c3079d31340 100644
--- a/tools/testing/selftests/resctrl/resctrl_val.c
+++ b/tools/testing/selftests/resctrl/resctrl_val.c
@@ -20,6 +20,21 @@
#define CON_MBM_LOCAL_BYTES_PATH \
"%s/%s/mon_data/mon_L3_%02d/mbm_local_bytes"
+/*
+ * MPAM exposes only total (read+write) bandwidth, under MB_MON, and per the
+ * NUMA-node-keyed MB domain rather than the L3 cache id.
+ */
+#define CON_MBM_TOTAL_BYTES_PATH \
+ "%s/%s/mon_data/mon_MB_%02d/mbm_total_bytes"
+
+/*
+ * In MPAM "mbm_event" (ABMC) counter-assignment mode a hardware counter must
+ * be explicitly assigned to a group+domain+event before the bandwidth file
+ * reads a value; otherwise it returns "Unassigned".
+ */
+#define MBM_MB_ASSIGN_PATH \
+ "%s/%s/mbm_MB_assignments"
+
struct membw_read_format {
__u64 value; /* The value of the event */
__u64 time_enabled; /* if PERF_FORMAT_TOTAL_TIME_ENABLED */
@@ -58,6 +73,10 @@ static struct mem_bw_counter bw_counters[MAX_BW_COUNTERS];
static const struct resctrl_test *current_test;
static const struct mem_bw_backend *mem_bw_backend;
+/* MPAM ABMC counter assignment state, see initialize_mem_bw_resctrl(). */
+static char mbm_assign_path[1024];
+static int mbm_assign_domain = -1;
+
static void read_mem_bw_initialize_perf_event_attr(int i)
{
memset(&bw_counters[i].pe, 0,
@@ -452,21 +471,63 @@ static int get_mem_bw_ref(float *bw_ref)
* initialize_mem_bw_resctrl: Appropriately populate "mbm_total_path"
* @param: Parameters passed to resctrl_val()
* @domain_id: Domain ID (cache ID; for MB, L3 cache ID)
+ *
+ * x86 resctrl reports per-RMID local bandwidth under L3_MON. MPAM exposes only
+ * total (read+write) bandwidth under MB_MON, indexed by the NUMA-node-keyed MB
+ * domain, and in "mbm_event" (ABMC) mode a hardware counter must be assigned
to
+ * the group+domain before the file reads a value.
*/
void initialize_mem_bw_resctrl(const struct resctrl_val_param *param,
int domain_id)
{
- sprintf(mbm_total_path, CON_MBM_LOCAL_BYTES_PATH, RESCTRL_PATH,
- param->ctrlgrp, domain_id);
+ const char *grp = param->ctrlgrp ? param->ctrlgrp : "";
+
+ if (resctrl_mon_feature_exists("L3_MON", "mbm_local_bytes")) {
+ sprintf(mbm_total_path, CON_MBM_LOCAL_BYTES_PATH, RESCTRL_PATH,
+ grp, domain_id);
+ return;
+ }
+
+ /* MPAM: total bandwidth under MB_MON. */
+ sprintf(mbm_total_path, CON_MBM_TOTAL_BYTES_PATH, RESCTRL_PATH,
+ grp, domain_id);
+
+ /* Assign an ABMC counter for this group+domain if in mbm_event mode. */
+ snprintf(mbm_assign_path, sizeof(mbm_assign_path), MBM_MB_ASSIGN_PATH,
+ RESCTRL_PATH, grp);
+ if (access(mbm_assign_path, W_OK) == 0) {
+ FILE *fp = fopen(mbm_assign_path, "w");
+
+ if (fp) {
+ /* The assignment parser requires a trailing newline. */
+ fprintf(fp, "mbm_total_bytes:%d=e\n", domain_id);
+ fclose(fp);
+ mbm_assign_domain = domain_id;
+ }
+ } else {
+ /* Not in counter-assignment mode (or no MB_MON); nothing to
do. */
+ mbm_assign_path[0] = '\0';
+ }
}
/*
- * mem_bw_ref_cleanup - Reset the selected reference-bandwidth backend
+ * mem_bw_ref_cleanup - Release an assigned ABMC counter and reset backend
state
*
- * Safe to call when no backend was selected.
+ * Group removal does not reclaim assigned MPAM counters, so the test must
+ * unassign explicitly. Safe to call when nothing was assigned.
*/
void mem_bw_ref_cleanup(void)
{
+ if (mbm_assign_domain >= 0 && mbm_assign_path[0]) {
+ FILE *fp = fopen(mbm_assign_path, "w");
+
+ if (fp) {
+ fprintf(fp, "mbm_total_bytes:%d=_\n",
mbm_assign_domain);
+ fclose(fp);
+ }
+ }
+ mbm_assign_domain = -1;
+ mbm_assign_path[0] = '\0';
mem_bw_backend = NULL;
}
diff --git a/tools/testing/selftests/resctrl/resctrlfs.c
b/tools/testing/selftests/resctrl/resctrlfs.c
index b9c1bfb6cc02..893f6748c68c 100644
--- a/tools/testing/selftests/resctrl/resctrlfs.c
+++ b/tools/testing/selftests/resctrl/resctrlfs.c
@@ -121,11 +121,40 @@ static int get_resource_cache_level(const char *resource)
return get_cache_level(resource);
}
+/*
+ * cpu_to_numa_node - NUMA node a logical CPU belongs to
+ * @cpu_no: CPU number
+ *
+ * Reads the "node<N>" symlink in the CPU's sysfs directory.
+ *
+ * Return: NUMA node id (>= 0) on success, < 0 on failure.
+ */
+static int cpu_to_numa_node(int cpu_no)
+{
+ char cpu_dir[1024];
+ struct dirent *ep;
+ int node = -1;
+ DIR *dp;
+
+ snprintf(cpu_dir, sizeof(cpu_dir), "%s%d", PHYS_ID_PATH, cpu_no);
+ dp = opendir(cpu_dir);
+ if (!dp)
+ return -1;
+ while ((ep = readdir(dp))) {
+ if (!strncmp(ep->d_name, "node", 4) && isdigit(ep->d_name[4])) {
+ node = atoi(ep->d_name + 4);
+ break;
+ }
+ }
+ closedir(dp);
+ return node;
+}
+
/*
* get_domain_id - Get resctrl domain ID for a specified CPU
* @resource: resource name
* @cpu_no: CPU number
- * @domain_id: domain ID (cache ID; for MB, L3 cache ID)
+ * @domain_id: domain ID (cache ID; for MB, L3 cache ID, or NUMA node on MPAM)
*
* Return: >= 0 on success, < 0 on failure.
*/
@@ -135,6 +164,20 @@ int get_domain_id(const char *resource, int cpu_no, int
*domain_id)
int cache_num;
FILE *fp;
+ /*
+ * On MPAM the MB monitoring/allocation domains are keyed by NUMA node
+ * (surfaced via a separate MB_MON group) and need not match the L3
cache
+ * id used on x86, so resolve the MB domain from the CPU's NUMA node.
+ */
+ if (!strcmp(resource, "MB") && resctrl_resource_exists("MB_MON")) {
+ int node = cpu_to_numa_node(cpu_no);
+
+ if (node >= 0) {
+ *domain_id = node;
+ return 0;
+ }
+ }
+
cache_num = get_resource_cache_level(resource);
if (cache_num < 0)
return cache_num;
--
2.43.0
