The branch main has been updated by corvink:
URL:
https://cgit.FreeBSD.org/src/commit/?id=b2221534a7bc16ea879c9fbb1a1fe4b337d2623b
commit b2221534a7bc16ea879c9fbb1a1fe4b337d2623b
Author: Corvin Köhne <corv...@freebsd.org>
AuthorDate: 2023-12-14 14:50:59 +0000
Commit: Corvin Köhne <corv...@freebsd.org>
CommitDate: 2025-06-24 06:11:52 +0000
bhyve: support VBTs which lay outside of the OpRegion
Previously, all of our devices had an OpRegion with a VBT which lays inside
the
OpRegion. Unfortunately, the OpRegion has a fixed size and VBTs get more and
more features/settings. A VBT can be larger than the allocated space of 6 KB
inside the OpRegion. For larger VBTs, Intel added two fields to the
OpRegion to
report the VBT address and the VBT size. Make bhyve aware of those fields
and
put the VBT into the guest memory too.
Unfortunately, Intel forgot to consider virtualization when defining
OpRegion
v2.0. OpRegion v2.0 uses an absolute address for pointing to the VBT. This
address is useless in a guest environment. It's possible to patch that.
However, it requires some more work and we can't test it because we have no
test device with this combination. Therefore, we don't want to do that yet
until we have a use case. Fortunately, Intel noticed the issue and from v2.1
onwards, the VBT address is a relative address.
Reviewed by: markj
MFC after: 1 week
Sponsored by: Beckhoff Automation GmbH & Co. KG
Differential Revision: https://reviews.freebsd.org/D45337
---
usr.sbin/bhyve/amd64/pci_gvt-d.c | 112 +++++++++++++++++++++++++++++++++++++++
usr.sbin/bhyve/pci_passthru.c | 2 +-
2 files changed, 113 insertions(+), 1 deletion(-)
diff --git a/usr.sbin/bhyve/amd64/pci_gvt-d.c b/usr.sbin/bhyve/amd64/pci_gvt-d.c
index 8cd5d21c8e6d..60bf460d71ed 100644
--- a/usr.sbin/bhyve/amd64/pci_gvt-d.c
+++ b/usr.sbin/bhyve/amd64/pci_gvt-d.c
@@ -39,6 +39,7 @@
#define GVT_D_MAP_GSM 0
#define GVT_D_MAP_OPREGION 1
+#define GVT_D_MAP_VBT 2
static int
gvt_d_probe(struct pci_devinst *const pi)
@@ -177,13 +178,78 @@ gvt_d_setup_gsm(struct pci_devinst *const pi)
passthru_cfgwrite_emulate));
}
+static int
+gvt_d_setup_vbt(struct pci_devinst *const pi, int memfd, uint64_t vbt_hpa,
+ uint64_t vbt_len, vm_paddr_t *vbt_gpa)
+{
+ struct passthru_softc *sc;
+ struct passthru_mmio_mapping *vbt;
+
+ sc = pi->pi_arg;
+
+ vbt = passthru_get_mmio(sc, GVT_D_MAP_VBT);
+ if (vbt == NULL) {
+ warnx("%s: Unable to access VBT", __func__);
+ return (-1);
+ }
+
+ vbt->hpa = vbt_hpa;
+ vbt->len = vbt_len;
+
+ vbt->hva = mmap(NULL, vbt->len, PROT_READ, MAP_SHARED, memfd, vbt->hpa);
+ if (vbt->hva == MAP_FAILED) {
+ warn("%s: Unable to map VBT", __func__);
+ return (-1);
+ }
+
+ vbt->gpa = gvt_d_alloc_mmio_memory(vbt->hpa, vbt->len,
+ E820_ALIGNMENT_NONE, E820_TYPE_NVS);
+ if (vbt->gpa == 0) {
+ warnx(
+ "%s: Unable to add VBT to E820 table (hpa 0x%lx len 0x%lx)",
+ __func__, vbt->hpa, vbt->len);
+ munmap(vbt->hva, vbt->len);
+ e820_dump_table();
+ return (-1);
+ }
+ vbt->gva = vm_map_gpa(pi->pi_vmctx, vbt->gpa, vbt->len);
+ if (vbt->gva == NULL) {
+ warnx("%s: Unable to map guest VBT", __func__);
+ munmap(vbt->hva, vbt->len);
+ return (-1);
+ }
+
+ if (vbt->gpa != vbt->hpa) {
+ /*
+ * A 1:1 host to guest mapping is not required but this could
+ * change in the future.
+ */
+ warnx(
+ "Warning: Unable to reuse host address of VBT. GPU
passthrough might not work properly.");
+ }
+
+ memcpy(vbt->gva, vbt->hva, vbt->len);
+
+ /*
+ * Return the guest physical address. It's used to patch the OpRegion
+ * properly.
+ */
+ *vbt_gpa = vbt->gpa;
+
+ return (0);
+}
+
static int
gvt_d_setup_opregion(struct pci_devinst *const pi)
{
struct passthru_softc *sc;
struct passthru_mmio_mapping *opregion;
+ struct igd_opregion *opregion_ptr;
struct igd_opregion_header *header;
+ vm_paddr_t vbt_gpa = 0;
+ vm_paddr_t vbt_hpa;
uint64_t asls;
+ int error = 0;
int memfd;
sc = pi->pi_arg;
@@ -236,6 +302,38 @@ gvt_d_setup_opregion(struct pci_devinst *const pi)
close(memfd);
return (-1);
}
+
+ opregion_ptr = (struct igd_opregion *)opregion->hva;
+ if (opregion_ptr->mbox3.rvda != 0) {
+ /*
+ * OpRegion v2.0 contains a physical address to the VBT. This
+ * address is useless in a guest environment. It's possible to
+ * patch that but we don't support that yet. So, the only thing
+ * we can do is give up.
+ */
+ if (opregion_ptr->header.over == 0x02000000) {
+ warnx(
+ "%s: VBT lays outside OpRegion. That's not yet
supported for a version 2.0 OpRegion",
+ __func__);
+ close(memfd);
+ return (-1);
+ }
+ vbt_hpa = opregion->hpa + opregion_ptr->mbox3.rvda;
+ if (vbt_hpa < opregion->hpa) {
+ warnx(
+ "%s: overflow when calculating VBT address
(OpRegion @ 0x%lx, RVDA = 0x%lx)",
+ __func__, opregion->hpa, opregion_ptr->mbox3.rvda);
+ close(memfd);
+ return (-1);
+ }
+
+ if ((error = gvt_d_setup_vbt(pi, memfd, vbt_hpa,
+ opregion_ptr->mbox3.rvds, &vbt_gpa)) != 0) {
+ close(memfd);
+ return (error);
+ }
+ }
+
close(memfd);
opregion->gpa = gvt_d_alloc_mmio_memory(opregion->hpa, opregion->len,
@@ -263,6 +361,20 @@ gvt_d_setup_opregion(struct pci_devinst *const pi)
memcpy(opregion->gva, opregion->hva, opregion->len);
+ /*
+ * Patch the VBT address to match our guest physical address.
+ */
+ if (vbt_gpa != 0) {
+ if (vbt_gpa < opregion->gpa) {
+ warnx(
+ "%s: invalid guest VBT address 0x%16lx (OpRegion @
0x%16lx)",
+ __func__, vbt_gpa, opregion->gpa);
+ return (-1);
+ }
+
+ ((struct igd_opregion *)opregion->gva)->mbox3.rvda = vbt_gpa -
opregion->gpa;
+ }
+
pci_set_cfgdata32(pi, PCIR_ASLS_CTL, opregion->gpa);
return (set_pcir_handler(sc, PCIR_ASLS_CTL, 4, passthru_cfgread_emulate,
diff --git a/usr.sbin/bhyve/pci_passthru.c b/usr.sbin/bhyve/pci_passthru.c
index 61983010192a..9d38ae9168a1 100644
--- a/usr.sbin/bhyve/pci_passthru.c
+++ b/usr.sbin/bhyve/pci_passthru.c
@@ -72,7 +72,7 @@
#define MSIX_TABLE_COUNT(ctrl) (((ctrl) & PCIM_MSIXCTRL_TABLE_SIZE) + 1)
#define MSIX_CAPLEN 12
-#define PASSTHRU_MMIO_MAX 2
+#define PASSTHRU_MMIO_MAX 3
static int pcifd = -1;
