The branch main has been updated by bnovkov:
URL:
https://cgit.FreeBSD.org/src/commit/?id=f1d705d4f431a5553aa5e64310bdc703ba27eab4
commit f1d705d4f431a5553aa5e64310bdc703ba27eab4
Author: Bojan Novković <bnov...@freebsd.org>
AuthorDate: 2024-09-08 16:06:57 +0000
Commit: Bojan Novković <bnov...@freebsd.org>
CommitDate: 2025-07-27 16:31:48 +0000
bhyve: Add support for specifying guest NUMA configuration
This patch adds basic support for adding NUMA domains to a bhyve VM.
The user can define a NUMA domain using the -n flag, which expects
a domain id, a CPU set, and memory size for each NUMA domain.
After parsing the node configurations, we use the interfaces added in
previous patches to set the NUMA configuration for the virtual machine.
Afterwards, we use the configuration to build the ACPI
'Static Resource Affinity Table' (SRAT) which is used to pass NUMA
information to the guest.
Users can optionally configure domainset(9) allocation policies
for each domain. Since each NUMA domain is essentially a separate
system memory segment, we can parse user-provided domainset(9) policies
and install them into its backing 'vm_object'.
Differential Revision: https://reviews.freebsd.org/D44567
Reviewed by: markj
Relnotes: yes
---
usr.sbin/bhyve/acpi.c | 124 +++++++++++++++++++++++
usr.sbin/bhyve/acpi.h | 1 +
usr.sbin/bhyve/amd64/bhyverun_machdep.c | 14 ++-
usr.sbin/bhyve/bhyve.8 | 80 ++++++++++++++-
usr.sbin/bhyve/bhyverun.c | 174 +++++++++++++++++++++++++++++++-
usr.sbin/bhyve/bhyverun.h | 1 +
usr.sbin/bhyve/bootrom.c | 1 +
usr.sbin/bhyve/pci_emul.c | 1 +
usr.sbin/bhyve/pci_fbuf.c | 1 +
usr.sbin/bhyve/pci_passthru.c | 1 +
10 files changed, 393 insertions(+), 5 deletions(-)
diff --git a/usr.sbin/bhyve/acpi.c b/usr.sbin/bhyve/acpi.c
index 85864da57af2..6ff8dd8e273b 100644
--- a/usr.sbin/bhyve/acpi.c
+++ b/usr.sbin/bhyve/acpi.c
@@ -37,9 +37,12 @@
*/
#include <sys/param.h>
+#include <sys/cpuset.h>
+#include <sys/domainset.h>
#include <sys/endian.h>
#include <sys/errno.h>
#include <sys/stat.h>
+#include <sys/tree.h>
#include <err.h>
#include <paths.h>
@@ -50,7 +53,9 @@
#include <string.h>
#include <unistd.h>
+#include <dev/vmm/vmm_mem.h>
#include <machine/vmm.h>
+#include <machine/vmm_dev.h>
#include <vmmapi.h>
#include "bhyverun.h"
@@ -78,6 +83,22 @@ static int basl_ncpu;
static char basl_template[MAXPATHLEN];
static char basl_stemplate[MAXPATHLEN];
+/*
+ * SRAT vCPU affinity info.
+ */
+struct acpi_vcpu_affinity_entry {
+ RB_ENTRY(acpi_vcpu_affinity_entry) entry;
+ int vcpuid;
+ int domain;
+};
+
+static int vcpu_affinity_cmp(struct acpi_vcpu_affinity_entry *const a1,
+ struct acpi_vcpu_affinity_entry *const a2);
+static RB_HEAD(vcpu_affinities,
+ acpi_vcpu_affinity_entry) aff_head = RB_INITIALIZER(&aff_head);
+RB_GENERATE_STATIC(vcpu_affinities, acpi_vcpu_affinity_entry, entry,
+ vcpu_affinity_cmp);
+
/*
* State for dsdt_line(), dsdt_indent(), and dsdt_unindent().
*/
@@ -121,6 +142,31 @@ acpi_tables_add_device(const struct acpi_device *const dev)
return (0);
}
+static int
+vcpu_affinity_cmp(struct acpi_vcpu_affinity_entry *a1,
+ struct acpi_vcpu_affinity_entry *a2)
+{
+ return (a1->vcpuid < a2->vcpuid ? -1 : a1->vcpuid > a2->vcpuid);
+}
+
+int
+acpi_add_vcpu_affinity(int vcpuid, int domain)
+{
+ struct acpi_vcpu_affinity_entry *entry = calloc(1, sizeof(*entry));
+ if (entry == NULL) {
+ return (ENOMEM);
+ }
+
+ entry->vcpuid = vcpuid;
+ entry->domain = domain;
+ if (RB_INSERT(vcpu_affinities, &aff_head, entry) != NULL) {
+ free(entry);
+ return (EEXIST);
+ }
+
+ return (0);
+}
+
/*
* Helper routines for writing to the DSDT from other modules.
*/
@@ -726,6 +772,83 @@ build_spcr(struct vmctx *const ctx)
return (0);
}
+static int
+build_srat(struct vmctx *const ctx)
+{
+ ACPI_TABLE_SRAT srat;
+ ACPI_SRAT_MEM_AFFINITY srat_mem_affinity;
+ ACPI_SRAT_CPU_AFFINITY srat_cpu_affinity;
+
+ struct acpi_vcpu_affinity_entry *ep;
+ struct basl_table *table;
+ int segid, domain;
+ int _flags, _prot;
+ vm_ooffset_t _off;
+ size_t maplen;
+ uint64_t gpa;
+ int ret;
+
+ if (RB_EMPTY(&aff_head))
+ return (0);
+
+ memset(&srat, 0, sizeof(srat));
+ BASL_EXEC(basl_table_create(&table, ctx, ACPI_SIG_SRAT,
+ BASL_TABLE_ALIGNMENT));
+ BASL_EXEC(basl_table_append_header(table, ACPI_SIG_SRAT, 1, 1));
+ srat.TableRevision = 1;
+ BASL_EXEC(basl_table_append_content(table, &srat, sizeof(srat)));
+
+ /*
+ * Iterate over the VM's memory maps and add
+ * a 'Memory Affinity Structure' for each mapping.
+ */
+ gpa = 0;
+ while (1) {
+ ret = vm_mmap_getnext(ctx, &gpa, &segid, &_off, &maplen, &_prot,
+ &_flags);
+ if (ret) {
+ break;
+ }
+
+ if (segid >= VM_SYSMEM && segid < VM_BOOTROM) {
+ domain = segid - VM_SYSMEM;
+ } else {
+ /* Treat devmem segs as domain 0. */
+ domain = 0;
+ }
+ memset(&srat_mem_affinity, 0, sizeof(srat_mem_affinity));
+ srat_mem_affinity.Header.Type = ACPI_SRAT_TYPE_MEMORY_AFFINITY;
+ srat_mem_affinity.Header.Length = sizeof(srat_mem_affinity);
+ srat_mem_affinity.Flags |= ACPI_SRAT_MEM_ENABLED;
+ srat_mem_affinity.ProximityDomain = htole32(domain);
+ srat_mem_affinity.BaseAddress = htole64(gpa);
+ srat_mem_affinity.Length = htole64(maplen);
+ srat_mem_affinity.Flags = htole32(ACPI_SRAT_MEM_ENABLED);
+ BASL_EXEC(basl_table_append_bytes(table, &srat_mem_affinity,
+ sizeof(srat_mem_affinity)));
+ gpa += maplen;
+ }
+
+ /*
+ * Iterate over each "vCPUid to domain id" mapping and emit a
+ * 'Processor Local APIC/SAPIC Affinity Structure' for each entry.
+ */
+ RB_FOREACH(ep, vcpu_affinities, &aff_head) {
+ memset(&srat_cpu_affinity, 0, sizeof(srat_cpu_affinity));
+ srat_cpu_affinity.Header.Type = ACPI_SRAT_TYPE_CPU_AFFINITY;
+ srat_cpu_affinity.Header.Length = sizeof(srat_cpu_affinity);
+ srat_cpu_affinity.ProximityDomainLo = (uint8_t)ep->domain;
+ srat_cpu_affinity.ApicId = (uint8_t)ep->vcpuid;
+ srat_cpu_affinity.Flags = htole32(ACPI_SRAT_CPU_USE_AFFINITY);
+ BASL_EXEC(basl_table_append_bytes(table, &srat_cpu_affinity,
+ sizeof(srat_cpu_affinity)));
+ }
+
+ BASL_EXEC(basl_table_register_to_rsdt(table));
+
+ return (0);
+}
+
int
acpi_build(struct vmctx *ctx, int ncpu)
{
@@ -765,6 +888,7 @@ acpi_build(struct vmctx *ctx, int ncpu)
BASL_EXEC(build_mcfg(ctx));
BASL_EXEC(build_facs(ctx));
BASL_EXEC(build_spcr(ctx));
+ BASL_EXEC(build_srat(ctx));
/* Build ACPI device-specific tables such as a TPM2 table. */
const struct acpi_device_list_entry *entry;
diff --git a/usr.sbin/bhyve/acpi.h b/usr.sbin/bhyve/acpi.h
index 4b557993d67f..824dce776306 100644
--- a/usr.sbin/bhyve/acpi.h
+++ b/usr.sbin/bhyve/acpi.h
@@ -56,6 +56,7 @@ struct vmctx;
int acpi_build(struct vmctx *ctx, int ncpu);
void acpi_raise_gpe(struct vmctx *ctx, unsigned bit);
int acpi_tables_add_device(const struct acpi_device *const dev);
+int acpi_add_vcpu_affinity(int vcpuid, int domain);
void dsdt_line(const char *fmt, ...);
void dsdt_fixed_ioport(uint16_t iobase, uint16_t length);
void dsdt_fixed_irq(uint8_t irq);
diff --git a/usr.sbin/bhyve/amd64/bhyverun_machdep.c
b/usr.sbin/bhyve/amd64/bhyverun_machdep.c
index 85af124b5536..dad8f1e52e4e 100644
--- a/usr.sbin/bhyve/amd64/bhyverun_machdep.c
+++ b/usr.sbin/bhyve/amd64/bhyverun_machdep.c
@@ -91,6 +91,7 @@ bhyve_usage(int code)
" -K: PS2 keyboard layout\n"
" -l: LPC device configuration\n"
" -m: memory size\n"
+ " -n: NUMA domain specification\n"
" -o: set config 'var' to 'value'\n"
" -P: vmexit from the guest on pause\n"
" -p: pin 'vcpu' to 'hostcpu'\n"
@@ -117,9 +118,9 @@ bhyve_optparse(int argc, char **argv)
int c;
#ifdef BHYVE_SNAPSHOT
- optstr = "aehuwxACDHIPSWYk:f:o:p:G:c:s:m:l:K:U:r:";
+ optstr = "aehuwxACDHIPSWYk:f:o:p:G:c:s:m:n:l:K:U:r:";
#else
- optstr = "aehuwxACDHIPSWYk:f:o:p:G:c:s:m:l:K:U:";
+ optstr = "aehuwxACDHIPSWYk:f:o:p:G:c:s:m:n:l:K:U:";
#endif
while ((c = getopt(argc, argv, optstr)) != -1) {
switch (c) {
@@ -194,6 +195,15 @@ bhyve_optparse(int argc, char **argv)
case 'm':
set_config_value("memory.size", optarg);
break;
+ case 'n':
+ if (bhyve_numa_parse(optarg) != 0)
+ errx(EX_USAGE,
+ "invalid NUMA configuration "
+ "'%s'",
+ optarg);
+ if (!get_config_bool("acpi_tables"))
+ errx(EX_USAGE, "NUMA emulation requires ACPI");
+ break;
case 'o':
if (!bhyve_parse_config_option(optarg)) {
errx(EX_USAGE,
diff --git a/usr.sbin/bhyve/bhyve.8 b/usr.sbin/bhyve/bhyve.8
index 62e567fd359d..89c0b23961a8 100644
--- a/usr.sbin/bhyve/bhyve.8
+++ b/usr.sbin/bhyve/bhyve.8
@@ -269,8 +269,56 @@ or
(either upper or lower case)
to indicate a multiple of kilobytes, megabytes, gigabytes, or terabytes.
If no suffix is given, the value is assumed to be in megabytes.
-.Pp
The default is 256M.
+.Pp
+.It Fl n Ar id Ns Cm \&, Ns Ar size Ns Cm \&, Ns Ar cpus Ns Op Cm \&, Ns Ar
domain_policy
+Configure guest NUMA domains.
+This option applies only to the amd64 platform.
+.Pp
+The
+.Fl n
+option allows the guest physical address space to be partitioned into domains.
+The layout of each domain is encoded in an ACPI table
+visible to the guest operating system.
+The
+.Fl n
+option also allows the specification of a
+.Xr domainset 9
+memory allocation policy for the host memory backing a given NUMA domain.
+A guest can have up to 8 NUMA domains.
+This feature requires that the guest use a boot ROM, and in
+particular cannot be used if the guest was initialized using
+.Xr bhyveload 8 .
+.Pp
+Each domain is identified by a numerical
+.Em id .
+The domain memory
+.Em size
+is specified using the same format as the
+.Fl m
+flag.
+The sum of all
+.Em size
+parameters overrides the total VM memory size specified by the
+.Fl m
+flag.
+However, if at least one domain memory size parameter is
+missing, the total VM memory size will be equally distributed across
+all emulated domains.
+The
+.Em cpuset
+parameter specifies the set of CPUs that are part of the domain.
+The
+.Em domain_policy
+parameter may be optionally used to configure the
+.Xr domainset 9
+host NUMA memory allocation policy for an emulated
+domain.
+See the
+.Ar -n
+flag in
+.Xr cpuset 1
+for a list of valid NUMA memory allocation policies and their formats.
.It Fl o Ar var Ns Cm = Ns Ar value
Set the configuration variable
.Ar var
@@ -1202,6 +1250,33 @@ using this configuration file, use flag
.Bd -literal -offset indent
/usr/sbin/bhyve -k configfile vm0
.Ed
+.Pp
+Run a UEFI virtual machine with four CPUs and two emulated NUMA domains:
+.Bd -literal -offset indent
+bhyve -c 4 -w -H \\
+ -s 0,hostbridge \\
+ -s 4,ahci-hd,disk.img \\
+ -s 31,lpc -l com1,stdio \\
+ -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \\
+ -n id=0,size=4G,cpus=0-1 \\
+ -n id=1,size=4G,cpus=2-3 \\
+ numavm
+.Ed
+.Pp
+Assuming a host machine with two NUMA domains,
+run a UEFI virtual machine with four CPUs using a
+.Ar prefer
+.Xr domainset 9
+policy to allocate guest memory from the first host NUMA domain only.
+.Bd -literal -offset indent
+bhyve -c 2 -w -H \\
+ -s 0,hostbridge \\
+ -s 4,ahci-hd,disk.img \\
+ -s 31,lpc -l com1,stdio \\
+ -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd \\
+ -n id=0,size=4G,cpus=0-1,domain_policy=prefer:0 \\
+ numavm
+.Ed
.Sh SEE ALSO
.Xr bhyve 4 ,
.Xr netgraph 4 ,
@@ -1211,7 +1286,8 @@ using this configuration file, use flag
.Xr bhyve_config 5 ,
.Xr ethers 5 ,
.Xr bhyvectl 8 ,
-.Xr bhyveload 8
+.Xr bhyveload 8 ,
+.Xr domainset 9
.Pp
.Rs
.%A Intel
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index be9cd1611700..9ead49582a7d 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -30,6 +30,8 @@
#ifndef WITHOUT_CAPSICUM
#include <sys/capsicum.h>
#endif
+#include <sys/cpuset.h>
+#include <sys/domainset.h>
#include <sys/mman.h>
#ifdef BHYVE_SNAPSHOT
#include <sys/socket.h>
@@ -54,6 +56,7 @@
#include <fcntl.h>
#endif
#include <libgen.h>
+#include <libutil.h>
#include <unistd.h>
#include <assert.h>
#include <pthread.h>
@@ -68,6 +71,7 @@
#include <libxo/xo.h>
#endif
+#include <dev/vmm/vmm_mem.h>
#include <vmmapi.h>
#include "acpi.h"
@@ -108,6 +112,9 @@ static const int BSP = 0;
static cpuset_t cpumask;
+static struct vm_mem_domain guest_domains[VM_MAXMEMDOM];
+static int guest_ndomains = 0;
+
static void vm_loop(struct vmctx *ctx, struct vcpu *vcpu);
static struct vcpu_info {
@@ -179,6 +186,118 @@ parse_int_value(const char *key, const char *value, int
minval, int maxval)
return (lval);
}
+int
+bhyve_numa_parse(const char *opt)
+{
+ int id = -1;
+ nvlist_t *nvl;
+ char *cp, *str, *tofree;
+ char pathbuf[64] = { 0 };
+ char *size = NULL, *cpus = NULL, *domain_policy = NULL;
+
+ if (*opt == '\0') {
+ return (-1);
+ }
+
+ tofree = str = strdup(opt);
+ if (str == NULL)
+ errx(4, "Failed to allocate memory");
+
+ while ((cp = strsep(&str, ",")) != NULL) {
+ if (strncmp(cp, "id=", strlen("id=")) == 0)
+ id = parse_int_value("id", cp + strlen("id="), 0,
+ UINT8_MAX);
+ else if (strncmp(cp, "size=", strlen("size=")) == 0)
+ size = cp + strlen("size=");
+ else if (strncmp(cp,
+ "domain_policy=", strlen("domain_policy=")) == 0)
+ domain_policy = cp + strlen("domain_policy=");
+ else if (strncmp(cp, "cpus=", strlen("cpus=")) == 0)
+ cpus = cp + strlen("cpus=");
+ }
+
+ if (id == -1) {
+ EPRINTLN("Missing NUMA domain ID in '%s'", opt);
+ goto out;
+ }
+
+ snprintf(pathbuf, sizeof(pathbuf), "domains.%d", id);
+ nvl = find_config_node(pathbuf);
+ if (nvl == NULL)
+ nvl = create_config_node(pathbuf);
+ if (size != NULL)
+ set_config_value_node(nvl, "size", size);
+ if (domain_policy != NULL)
+ set_config_value_node(nvl, "domain_policy", domain_policy);
+ if (cpus != NULL)
+ set_config_value_node(nvl, "cpus", cpus);
+
+ free(tofree);
+ return (0);
+
+out:
+ free(tofree);
+ return (-1);
+}
+
+static void
+calc_mem_affinity(size_t vm_memsize)
+{
+ int i;
+ nvlist_t *nvl;
+ bool need_recalc;
+ const char *value;
+ struct vm_mem_domain *dom;
+ char pathbuf[64] = { 0 };
+
+ need_recalc = false;
+ for (i = 0; i < VM_MAXMEMDOM; i++) {
+ dom = &guest_domains[i];
+ snprintf(pathbuf, sizeof(pathbuf), "domains.%d", i);
+ nvl = find_config_node(pathbuf);
+ if (nvl == NULL) {
+ break;
+ }
+
+ value = get_config_value_node(nvl, "size");
+ need_recalc |= value == NULL;
+ if (value != NULL && vm_parse_memsize(value, &dom->size)) {
+ errx(EX_USAGE, "invalid memsize for domain %d: '%s'", i,
+ value);
+ }
+
+ dom->ds_mask = calloc(1, sizeof(domainset_t));
+ if (dom->ds_mask == NULL) {
+ errx(EX_OSERR, "Failed to allocate domainset mask");
+ }
+ dom->ds_size = sizeof(domainset_t);
+ value = get_config_value_node(nvl, "domain_policy");
+ if (value == NULL) {
+ dom->ds_policy = DOMAINSET_POLICY_INVALID;
+ DOMAINSET_ZERO(dom->ds_mask);
+ } else if (domainset_parselist(value, dom->ds_mask,
&dom->ds_policy) !=
+ CPUSET_PARSE_OK) {
+ errx(EX_USAGE, "failed to parse domain policy
'%s'", value);
+ }
+ }
+
+ guest_ndomains = i;
+ if (guest_ndomains == 0) {
+ /*
+ * No domains were specified - create domain
+ * 0 holding all CPUs and memory.
+ */
+ guest_ndomains = 1;
+ guest_domains[0].size = vm_memsize;
+ } else if (need_recalc) {
+ warnx("At least one domain memory size was not specified,
distributing"
+ " total VM memory size across all domains");
+ for (i = 0; i < guest_ndomains; i++) {
+ guest_domains[i].size = vm_memsize / guest_ndomains;
+ }
+ }
+}
+
/*
* Set the sockets, cores, threads, and guest_cpus variables based on
* the configured topology.
@@ -340,6 +459,56 @@ build_vcpumaps(void)
}
}
+static void
+set_vcpu_affinities(void)
+{
+ int cpu, error;
+ nvlist_t *nvl = NULL;
+ cpuset_t cpus;
+ const char *value;
+ char pathbuf[64] = { 0 };
+
+ for (int dom = 0; dom < guest_ndomains; dom++) {
+ snprintf(pathbuf, sizeof(pathbuf), "domains.%d", dom);
+ nvl = find_config_node(pathbuf);
+ if (nvl == NULL)
+ break;
+
+ value = get_config_value_node(nvl, "cpus");
+ if (value == NULL) {
+ EPRINTLN("Missing CPU set for domain %d", dom);
+ exit(4);
+ }
+
+ parse_cpuset(dom, value, &cpus);
+ CPU_FOREACH_ISSET(cpu, &cpus) {
+ error = acpi_add_vcpu_affinity(cpu, dom);
+ if (error) {
+ EPRINTLN(
+ "Unable to set vCPU %d affinity for domain
%d: %s",
+ cpu, dom, strerror(errno));
+ exit(4);
+ }
+ }
+ }
+ if (guest_ndomains > 1 || nvl != NULL)
+ return;
+
+ /*
+ * If we're dealing with one domain and no cpuset was provided, create a
+ * default one holding all cpus.
+ */
+ for (cpu = 0; cpu < guest_ncpus; cpu++) {
+ error = acpi_add_vcpu_affinity(cpu, 0);
+ if (error) {
+ EPRINTLN(
+ "Unable to set vCPU %d affinity for domain %d: %s",
+ cpu, 0, strerror(errno));
+ exit(4);
+ }
+ }
+}
+
void *
paddr_guest2host(struct vmctx *ctx, uintptr_t gaddr, size_t len)
{
@@ -713,18 +882,21 @@ main(int argc, char *argv[])
vcpu_info[vcpuid].vcpu = vm_vcpu_open(ctx, vcpuid);
}
+ calc_mem_affinity(memsize);
memflags = 0;
if (get_config_bool_default("memory.wired", false))
memflags |= VM_MEM_F_WIRED;
if (get_config_bool_default("memory.guest_in_core", false))
memflags |= VM_MEM_F_INCORE;
vm_set_memflags(ctx, memflags);
- error = vm_setup_memory(ctx, memsize, VM_MMAP_ALL);
+ error = vm_setup_memory_domains(ctx, VM_MMAP_ALL, guest_domains,
+ guest_ndomains);
if (error) {
fprintf(stderr, "Unable to setup memory (%d)\n", errno);
exit(4);
}
+ set_vcpu_affinities();
init_mem(guest_ncpus);
init_bootrom(ctx);
if (bhyve_init_platform(ctx, bsp) != 0)
diff --git a/usr.sbin/bhyve/bhyverun.h b/usr.sbin/bhyve/bhyverun.h
index 005de6dc5410..0a7bbd72a19c 100644
--- a/usr.sbin/bhyve/bhyverun.h
+++ b/usr.sbin/bhyve/bhyverun.h
@@ -73,6 +73,7 @@ void bhyve_parse_gdb_options(const char *opt);
#endif
int bhyve_pincpu_parse(const char *opt);
int bhyve_topology_parse(const char *opt);
+int bhyve_numa_parse(const char *opt);
void bhyve_init_vcpu(struct vcpu *vcpu);
void bhyve_start_vcpu(struct vcpu *vcpu, bool bsp);
diff --git a/usr.sbin/bhyve/bootrom.c b/usr.sbin/bhyve/bootrom.c
index e4adaca55947..339974cb2017 100644
--- a/usr.sbin/bhyve/bootrom.c
+++ b/usr.sbin/bhyve/bootrom.c
@@ -31,6 +31,7 @@
#include <sys/mman.h>
#include <sys/stat.h>
+#include <dev/vmm/vmm_mem.h>
#include <machine/vmm.h>
#include <err.h>
diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index 2f04a488d9c1..9d6060e3e254 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -42,6 +42,7 @@
#include <stdbool.h>
#include <sysexits.h>
+#include <dev/vmm/vmm_mem.h>
#include <machine/vmm.h>
#include <machine/vmm_snapshot.h>
#include <vmmapi.h>
diff --git a/usr.sbin/bhyve/pci_fbuf.c b/usr.sbin/bhyve/pci_fbuf.c
index 125428e0b772..1e3ec77c15b0 100644
--- a/usr.sbin/bhyve/pci_fbuf.c
+++ b/usr.sbin/bhyve/pci_fbuf.c
@@ -29,6 +29,7 @@
#include <sys/types.h>
#include <sys/mman.h>
+#include <dev/vmm/vmm_mem.h>
#include <machine/vmm.h>
#include <machine/vmm_snapshot.h>
#include <vmmapi.h>
diff --git a/usr.sbin/bhyve/pci_passthru.c b/usr.sbin/bhyve/pci_passthru.c
index 9d38ae9168a1..a82078f6e036 100644
--- a/usr.sbin/bhyve/pci_passthru.c
+++ b/usr.sbin/bhyve/pci_passthru.c
@@ -38,6 +38,7 @@
#include <dev/io/iodev.h>
#include <dev/pci/pcireg.h>
+#include <dev/vmm/vmm_mem.h>
#include <vm/vm.h>
