The branch main has been updated by markj:
URL:
https://cgit.FreeBSD.org/src/commit/?id=55c13f6e7a412cc4bd0ea3fc183cd7c5c2348f01
commit 55c13f6e7a412cc4bd0ea3fc183cd7c5c2348f01
Author: Mark Johnston <ma...@freebsd.org>
AuthorDate: 2023-10-04 16:25:19 +0000
Commit: Mark Johnston <ma...@freebsd.org>
CommitDate: 2023-10-04 16:53:17 +0000
bhyve: Move legacy PCI interrupt handling under amd64/
Specifically, move IO-APIC, LPC and PIRQ routing code under amd64/.
Use ifdefs to conditionally compile related code in other files. In
particular, legacy PCI interrupt handling is now compiled only on amd64.
This is not too invasive, but suggestions for a more modular approach
would be appreciated.
I am not sure why qemu fwcfg handling is tied to LPC, and I suspect it
should be decoupled. In this commit I just apply an ifdef hammer, but
we will eventually want fwcfg on arm64 as well.
No functional change intended.
Reviewed by: corvink, jhb
MFC after: 1 week
Sponsored by: Innovate UK
Differential Revision: https://reviews.freebsd.org/D40739
---
usr.sbin/bhyve/Makefile | 3 ---
usr.sbin/bhyve/amd64/Makefile.inc | 3 +++
usr.sbin/bhyve/{ => amd64}/ioapic.c | 0
usr.sbin/bhyve/{ => amd64}/ioapic.h | 0
usr.sbin/bhyve/{ => amd64}/pci_irq.c | 0
usr.sbin/bhyve/{ => amd64}/pci_irq.h | 0
usr.sbin/bhyve/{ => amd64}/pci_lpc.c | 0
usr.sbin/bhyve/{ => amd64}/pci_lpc.h | 0
usr.sbin/bhyve/bhyverun.c | 16 ++++++++++++----
usr.sbin/bhyve/pci_emul.c | 36 +++++++++++++++++++++++++++++-------
usr.sbin/bhyve/pci_emul.h | 27 +++++++++++++++------------
usr.sbin/bhyve/qemu_fwcfg.c | 18 ++++++++++++++++--
usr.sbin/bhyve/virtio.h | 2 ++
13 files changed, 77 insertions(+), 28 deletions(-)
diff --git a/usr.sbin/bhyve/Makefile b/usr.sbin/bhyve/Makefile
index 0ec6ad73e2a2..5aaf0d4ea2a0 100644
--- a/usr.sbin/bhyve/Makefile
+++ b/usr.sbin/bhyve/Makefile
@@ -31,7 +31,6 @@ SRCS= \
gdb.c \
hda_codec.c \
inout.c \
- ioapic.c \
iov.c \
mem.c \
mevent.c \
@@ -40,8 +39,6 @@ SRCS= \
pci_emul.c \
pci_fbuf.c \
pci_hostbridge.c \
- pci_irq.c \
- pci_lpc.c \
pci_nvme.c \
pci_passthru.c \
pci_virtio_9p.c \
diff --git a/usr.sbin/bhyve/amd64/Makefile.inc
b/usr.sbin/bhyve/amd64/Makefile.inc
index 862aeddb16e0..76813891b0a0 100644
--- a/usr.sbin/bhyve/amd64/Makefile.inc
+++ b/usr.sbin/bhyve/amd64/Makefile.inc
@@ -2,12 +2,15 @@ SRCS+= \
atkbdc.c \
e820.c \
fwctl.c \
+ ioapic.c \
kernemu_dev.c \
mptbl.c \
pci_ahci.c \
pci_e82545.c \
pci_gvt-d.c \
pci_hda.c \
+ pci_irq.c \
+ pci_lpc.c \
pci_uart.c \
pci_xhci.c \
pctestdev.c \
diff --git a/usr.sbin/bhyve/ioapic.c b/usr.sbin/bhyve/amd64/ioapic.c
similarity index 100%
rename from usr.sbin/bhyve/ioapic.c
rename to usr.sbin/bhyve/amd64/ioapic.c
diff --git a/usr.sbin/bhyve/ioapic.h b/usr.sbin/bhyve/amd64/ioapic.h
similarity index 100%
rename from usr.sbin/bhyve/ioapic.h
rename to usr.sbin/bhyve/amd64/ioapic.h
diff --git a/usr.sbin/bhyve/pci_irq.c b/usr.sbin/bhyve/amd64/pci_irq.c
similarity index 100%
rename from usr.sbin/bhyve/pci_irq.c
rename to usr.sbin/bhyve/amd64/pci_irq.c
diff --git a/usr.sbin/bhyve/pci_irq.h b/usr.sbin/bhyve/amd64/pci_irq.h
similarity index 100%
rename from usr.sbin/bhyve/pci_irq.h
rename to usr.sbin/bhyve/amd64/pci_irq.h
diff --git a/usr.sbin/bhyve/pci_lpc.c b/usr.sbin/bhyve/amd64/pci_lpc.c
similarity index 100%
rename from usr.sbin/bhyve/pci_lpc.c
rename to usr.sbin/bhyve/amd64/pci_lpc.c
diff --git a/usr.sbin/bhyve/pci_lpc.h b/usr.sbin/bhyve/amd64/pci_lpc.h
similarity index 100%
rename from usr.sbin/bhyve/pci_lpc.h
rename to usr.sbin/bhyve/amd64/pci_lpc.h
diff --git a/usr.sbin/bhyve/bhyverun.c b/usr.sbin/bhyve/bhyverun.c
index c27cb427f1e2..ba63a6405538 100644
--- a/usr.sbin/bhyve/bhyverun.c
+++ b/usr.sbin/bhyve/bhyverun.c
@@ -85,8 +85,8 @@
#include "amd64/fwctl.h"
#endif
#include "gdb.h"
-#include "ioapic.h"
#ifdef __amd64__
+#include "amd64/ioapic.h"
#include "amd64/kernemu_dev.h"
#endif
#include "mem.h"
@@ -95,8 +95,10 @@
#include "amd64/mptbl.h"
#endif
#include "pci_emul.h"
-#include "pci_irq.h"
-#include "pci_lpc.h"
+#ifdef __amd64__
+#include "amd64/pci_irq.h"
+#include "amd64/pci_lpc.h"
+#endif
#include "qemu_fwcfg.h"
#include "smbiostbl.h"
#ifdef BHYVE_SNAPSHOT
@@ -634,8 +636,10 @@ do_open(const char *vmname)
reinit = romboot = false;
+#ifdef __amd64__
if (lpc_bootrom())
romboot = true;
+#endif
error = vm_create(vmname);
if (error) {
@@ -855,6 +859,7 @@ main(int argc, char *argv[])
case 'K':
set_config_value("keyboard.layout", optarg);
break;
+#ifdef __amd64__
case 'l':
if (strncmp(optarg, "help", strlen(optarg)) == 0) {
lpc_print_supported_devices();
@@ -864,6 +869,7 @@ main(int argc, char *argv[])
"configuration '%s'", optarg);
}
break;
+#endif
#ifdef BHYVE_SNAPSHOT
case 'r':
restore_file = optarg;
@@ -1037,9 +1043,9 @@ main(int argc, char *argv[])
init_bootrom(ctx);
#ifdef __amd64__
atkbdc_init(ctx);
-#endif
pci_irq_init(ctx);
ioapic_init(ctx);
+#endif
#ifdef __amd64__
rtc_init(ctx);
@@ -1085,6 +1091,7 @@ main(int argc, char *argv[])
init_gdb(ctx);
+#ifdef __amd64__
if (lpc_bootrom()) {
if (vm_set_capability(bsp, VM_CAP_UNRESTRICTED_GUEST, 1)) {
fprintf(stderr, "ROM boot failed: unrestricted guest "
@@ -1094,6 +1101,7 @@ main(int argc, char *argv[])
error = vcpu_reset(bsp);
assert(error == 0);
}
+#endif
/*
* Add all vCPUs.
diff --git a/usr.sbin/bhyve/pci_emul.c b/usr.sbin/bhyve/pci_emul.c
index 5fb25dbfe9c7..6b2e46ce917d 100644
--- a/usr.sbin/bhyve/pci_emul.c
+++ b/usr.sbin/bhyve/pci_emul.c
@@ -52,11 +52,15 @@
#include "config.h"
#include "debug.h"
#include "inout.h"
-#include "ioapic.h"
+#ifdef __amd64__
+#include "amd64/ioapic.h"
+#endif
#include "mem.h"
#include "pci_emul.h"
-#include "pci_irq.h"
-#include "pci_lpc.h"
+#ifdef __amd64__
+#include "amd64/pci_irq.h"
+#include "amd64/pci_lpc.h"
+#endif
#include "pci_passthru.h"
#include "qemu_fwcfg.h"
@@ -143,9 +147,12 @@ SYSRES_MEM(PCI_EMUL_ECFG_BASE, PCI_EMUL_ECFG_SIZE);
#define PCI_EMUL_MEMLIMIT32 PCI_EMUL_ECFG_BASE
#define PCI_EMUL_MEMSIZE64 (32*GB)
-static struct pci_devemu *pci_emul_finddev(const char *name);
+#ifdef __amd64__
static void pci_lintr_route(struct pci_devinst *pi);
static void pci_lintr_update(struct pci_devinst *pi);
+#endif
+
+static struct pci_devemu *pci_emul_finddev(const char *name);
static void pci_cfgrw(int in, int bus, int slot, int func, int coff,
int bytes, uint32_t *val);
@@ -1061,11 +1068,13 @@ pci_emul_init(struct vmctx *ctx, struct pci_devemu
*pde, int bus, int slot,
pdi->pi_bus = bus;
pdi->pi_slot = slot;
pdi->pi_func = func;
+#ifdef __amd64__
pthread_mutex_init(&pdi->pi_lintr.lock, NULL);
pdi->pi_lintr.pin = 0;
pdi->pi_lintr.state = IDLE;
pdi->pi_lintr.pirq_pin = 0;
pdi->pi_lintr.ioapic_irq = 0;
+#endif
pdi->pi_d = pde;
snprintf(pdi->pi_name, PI_NAMESZ, "%s@pci.%d.%d.%d", pde->pe_emu, bus,
slot, func);
@@ -1203,7 +1212,9 @@ msixcap_cfgwrite(struct pci_devinst *pi, int capoff, int
offset,
pi->pi_msix.enabled = val & PCIM_MSIXCTRL_MSIX_ENABLE;
pi->pi_msix.function_mask = val & PCIM_MSIXCTRL_FUNCTION_MASK;
+#ifdef __amd64__
pci_lintr_update(pi);
+#endif
}
CFGWRITE(pi, offset, val, bytes);
@@ -1245,7 +1256,9 @@ msicap_cfgwrite(struct pci_devinst *pi, int capoff, int
offset,
} else {
pi->pi_msi.maxmsgnum = 0;
}
+#ifdef __amd64__
pci_lintr_update(pi);
+#endif
}
static void
@@ -1538,6 +1551,7 @@ init_pci(struct vmctx *ctx)
bi->memlimit64 = pci_emul_membase64;
}
+#ifdef __amd64__
/*
* PCI backends are initialized before routing INTx interrupts
* so that LPC devices are able to reserve ISA IRQs before
@@ -1558,6 +1572,7 @@ init_pci(struct vmctx *ctx)
}
}
lpc_pirq_routed();
+#endif
if ((error = init_bootorder()) != 0) {
warnx("%s: Unable to init bootorder", __func__);
@@ -1601,6 +1616,7 @@ init_pci(struct vmctx *ctx)
return (0);
}
+#ifdef __amd64__
static void
pci_apic_prt_entry(int bus __unused, int slot, int pin, int pirq_pin __unused,
int ioapic_irq, void *arg __unused)
@@ -1633,6 +1649,7 @@ pci_pirq_prt_entry(int bus __unused, int slot, int pin,
int pirq_pin,
dsdt_line(" },");
free(name);
}
+#endif
/*
* A bhyve virtual machine has a flat PCI hierarchy with a root port
@@ -1644,7 +1661,7 @@ pci_bus_write_dsdt(int bus)
struct businfo *bi;
struct slotinfo *si;
struct pci_devinst *pi;
- int count, func, slot;
+ int func, slot;
/*
* If there are no devices on this 'bus' then just return.
@@ -1747,8 +1764,8 @@ pci_bus_write_dsdt(int bus)
dsdt_line(" ,, , AddressRangeMemory, TypeStatic)");
dsdt_line(" })");
- count = pci_count_lintr(bus);
- if (count != 0) {
+#ifdef __amd64__
+ if (pci_count_lintr(bus) != 0) {
dsdt_indent(2);
dsdt_line("Name (PPRT, Package ()");
dsdt_line("{");
@@ -1771,6 +1788,7 @@ pci_bus_write_dsdt(int bus)
dsdt_line("}");
dsdt_unindent(2);
}
+#endif
dsdt_indent(2);
for (slot = 0; slot < MAXSLOTS; slot++) {
@@ -1866,6 +1884,7 @@ pci_generate_msi(struct pci_devinst *pi, int index)
}
}
+#ifdef __amd64__
static bool
pci_lintr_permitted(struct pci_devinst *pi)
{
@@ -2026,6 +2045,7 @@ pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg)
}
}
}
+#endif /* __amd64__ */
/*
* Return 1 if the emulated device in 'slot' is a multi-function device.
@@ -2130,11 +2150,13 @@ pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t
old)
}
}
+#ifdef __amd64__
/*
* If INTx has been unmasked and is pending, assert the
* interrupt.
*/
pci_lintr_update(pi);
+#endif
}
static void
diff --git a/usr.sbin/bhyve/pci_emul.h b/usr.sbin/bhyve/pci_emul.h
index 0fd438151093..c367169113f8 100644
--- a/usr.sbin/bhyve/pci_emul.h
+++ b/usr.sbin/bhyve/pci_emul.h
@@ -115,12 +115,6 @@ struct msix_table_entry {
#define MAX_MSIX_TABLE_ENTRIES 2048
#define PBA_SIZE(msgnum) (roundup2((msgnum), 64) / 8)
-enum lintr_stat {
- IDLE,
- ASSERTED,
- PENDING
-};
-
struct pci_devinst {
struct pci_devemu *pi_d;
struct vmctx *pi_vmctx;
@@ -130,13 +124,19 @@ struct pci_devinst {
int pi_prevcap;
int pi_capend;
+#ifdef __amd64__
struct {
int8_t pin;
- enum lintr_stat state;
+ enum {
+ IDLE,
+ ASSERTED,
+ PENDING,
+ } state;
int pirq_pin;
int ioapic_irq;
pthread_mutex_t lock;
} pi_lintr;
+#endif
struct {
int enabled;
@@ -221,8 +221,15 @@ struct pciecap {
} __packed;
static_assert(sizeof(struct pciecap) == 60, "compile-time assertion failed");
+#ifdef __amd64__
typedef void (*pci_lintr_cb)(int b, int s, int pin, int pirq_pin,
int ioapic_irq, void *arg);
+void pci_lintr_assert(struct pci_devinst *pi);
+void pci_lintr_deassert(struct pci_devinst *pi);
+void pci_lintr_request(struct pci_devinst *pi);
+int pci_count_lintr(int bus);
+void pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg);
+#endif
int init_pci(struct vmctx *ctx);
void pci_callback(void);
@@ -241,9 +248,6 @@ void pci_emul_capwrite(struct pci_devinst *pi, int
offset, int bytes,
void pci_emul_cmd_changed(struct pci_devinst *pi, uint16_t old);
void pci_generate_msi(struct pci_devinst *pi, int msgnum);
void pci_generate_msix(struct pci_devinst *pi, int msgnum);
-void pci_lintr_assert(struct pci_devinst *pi);
-void pci_lintr_deassert(struct pci_devinst *pi);
-void pci_lintr_request(struct pci_devinst *pi);
int pci_msi_enabled(struct pci_devinst *pi);
int pci_msix_enabled(struct pci_devinst *pi);
int pci_msix_table_bar(struct pci_devinst *pi);
@@ -257,11 +261,10 @@ int pci_emul_add_msixcap(struct pci_devinst *pi,
int msgnum, int barnum);
int pci_emul_msix_twrite(struct pci_devinst *pi, uint64_t offset, int size,
uint64_t value);
uint64_t pci_emul_msix_tread(struct pci_devinst *pi, uint64_t offset, int
size);
-int pci_count_lintr(int bus);
-void pci_walk_lintr(int bus, pci_lintr_cb cb, void *arg);
void pci_write_dsdt(void);
uint64_t pci_ecfg_base(void);
int pci_bus_configured(int bus);
+
#ifdef BHYVE_SNAPSHOT
struct pci_devinst *pci_next(const struct pci_devinst *cursor);
int pci_snapshot(struct vm_snapshot_meta *meta);
diff --git a/usr.sbin/bhyve/qemu_fwcfg.c b/usr.sbin/bhyve/qemu_fwcfg.c
index e845c70950b1..73a401ad8a81 100644
--- a/usr.sbin/bhyve/qemu_fwcfg.c
+++ b/usr.sbin/bhyve/qemu_fwcfg.c
@@ -23,7 +23,9 @@
#include "acpi_device.h"
#include "bhyverun.h"
#include "inout.h"
-#include "pci_lpc.h"
+#ifdef __amd64__
+#include "amd64/pci_lpc.h"
+#endif
#include "qemu_fwcfg.h"
#define QEMU_FWCFG_ACPI_DEVICE_NAME "FWCF"
@@ -423,6 +425,18 @@ int
qemu_fwcfg_init(struct vmctx *const ctx)
{
int error;
+ bool fwcfg_enabled;
+
+ /*
+ * The fwcfg implementation currently only provides an I/O port
+ * interface and thus is amd64-specific for now. An MMIO interface is
+ * required for other platforms.
+ */
+#ifdef __amd64__
+ fwcfg_enabled = strcmp(lpc_fwcfg(), "qemu") == 0;
+#else
+ fwcfg_enabled = false;
+#endif
/*
* Bhyve supports fwctl (bhyve) and fwcfg (qemu) as firmware interfaces.
@@ -430,7 +444,7 @@ qemu_fwcfg_init(struct vmctx *const ctx)
* interfaces at the same time to the guest. Therefore, only create acpi
* tables and register io ports for fwcfg, if it's used.
*/
- if (strcmp(lpc_fwcfg(), "qemu") == 0) {
+ if (fwcfg_enabled) {
error = acpi_device_create(&fwcfg_sc.acpi_dev, &fwcfg_sc, ctx,
&qemu_fwcfg_acpi_device_emul);
if (error) {
diff --git a/usr.sbin/bhyve/virtio.h b/usr.sbin/bhyve/virtio.h
index 2b72b862ab21..4c6c8004b2d1 100644
--- a/usr.sbin/bhyve/virtio.h
+++ b/usr.sbin/bhyve/virtio.h
@@ -358,7 +358,9 @@ vi_interrupt(struct virtio_softc *vs, uint8_t isr, uint16_t
msix_idx)
VS_LOCK(vs);
vs->vs_isr |= isr;
pci_generate_msi(vs->vs_pi, 0);
+#ifdef __amd64__
pci_lintr_assert(vs->vs_pi);
+#endif
VS_UNLOCK(vs);
}
}
