On Sun, Jul 23, 2023 at 7:35 PM Alistair Francis <alistai...@gmail.com> wrote:
>
> On Thu, Jul 20, 2023 at 12:35 PM Tomasz Jeznach <tjezn...@rivosinc.com> wrote:
> >
> > Adding virt machine property 'iommu' to enable/disable IOMMU
> > support, with platform RISC-V IOMMU device implementation.
> >
> > Generate device tree entry for riscv-iommu device, along with
> > mapping all PCI device identifiers to the single IOMMU device
> > instance.
> >
> > Signed-off-by: Tomasz Jeznach <tjezn...@rivosinc.com>
> > ---
> > hw/riscv/Kconfig | 1 +
> > hw/riscv/virt.c | 100 +++++++++++++++++++++++++++++++++++++++-
> > include/hw/riscv/virt.h | 3 ++
> > 3 files changed, 103 insertions(+), 1 deletion(-)
> >
> > diff --git a/hw/riscv/Kconfig b/hw/riscv/Kconfig
> > index 617a509f1b..b1a3a9994f 100644
> > --- a/hw/riscv/Kconfig
> > +++ b/hw/riscv/Kconfig
> > @@ -41,6 +41,7 @@ config RISCV_VIRT
> > select SERIAL
> > select RISCV_ACLINT
> > select RISCV_APLIC
> > + select RISCV_IOMMU
> > select RISCV_IMSIC
> > select SIFIVE_PLIC
> > select SIFIVE_TEST
> > diff --git a/hw/riscv/virt.c b/hw/riscv/virt.c
> > index d90286dc46..49cc7105af 100644
> > --- a/hw/riscv/virt.c
> > +++ b/hw/riscv/virt.c
> > @@ -32,6 +32,7 @@
> > #include "hw/core/sysbus-fdt.h"
> > #include "target/riscv/pmu.h"
> > #include "hw/riscv/riscv_hart.h"
> > +#include "hw/riscv/iommu.h"
> > #include "hw/riscv/virt.h"
> > #include "hw/riscv/boot.h"
> > #include "hw/riscv/numa.h"
> > @@ -88,7 +89,8 @@ static const MemMapEntry virt_memmap[] = {
> > [VIRT_APLIC_M] = { 0xc000000, APLIC_SIZE(VIRT_CPUS_MAX) },
> > [VIRT_APLIC_S] = { 0xd000000, APLIC_SIZE(VIRT_CPUS_MAX) },
> > [VIRT_UART0] = { 0x10000000, 0x100 },
> > - [VIRT_VIRTIO] = { 0x10001000, 0x1000 },
> > + [VIRT_IOMMU] = { 0x10001000, 0x1000 },
> > + [VIRT_VIRTIO] = { 0x10008000, 0x1000 }, /* VIRTIO_COUNT */
>
> We shouldn't change existing addresses
>
OK, I'll find another 4k window for IOMMU.
> Alistair
>
> > [VIRT_FW_CFG] = { 0x10100000, 0x18 },
> > [VIRT_FLASH] = { 0x20000000, 0x4000000 },
> > [VIRT_IMSIC_M] = { 0x24000000, VIRT_IMSIC_MAX_SIZE },
> > @@ -1019,6 +1021,44 @@ static void create_fdt_fw_cfg(RISCVVirtState *s,
> > const MemMapEntry *memmap)
> > g_free(nodename);
> > }
> >
> > +static void create_fdt_iommu(RISCVVirtState *s, const MemMapEntry *memmap,
> > + uint32_t irq_mmio_phandle)
> > +{
> > + MachineState *ms = MACHINE(s);
> > + uint32_t iommu_phandle;
> > + const char *irq_names[] = { "cmdq", "fltq", "pm", "priq" };
> > + char *iommu_node;
> > + char *pci_node;
> > +
> > + pci_node = g_strdup_printf("/soc/pci@%" PRIx64,
> > memmap[VIRT_PCIE_ECAM].base);
> > + iommu_node = g_strdup_printf("/soc/iommu@%" PRIx64,
> > memmap[VIRT_IOMMU].base);
> > +
> > + iommu_phandle = qemu_fdt_alloc_phandle(ms->fdt);
> > + qemu_fdt_add_subnode(ms->fdt, iommu_node);
> > + qemu_fdt_setprop_string(ms->fdt, iommu_node, "compatible",
> > "riscv,iommu");
> > + qemu_fdt_setprop_cell(ms->fdt, iommu_node, "#iommu-cells", 1);
> > + qemu_fdt_setprop_cell(ms->fdt, iommu_node, "phandle", iommu_phandle);
> > + qemu_fdt_setprop_cells(ms->fdt, iommu_node, "reg",
> > + 0x0, memmap[VIRT_IOMMU].base, 0x0, memmap[VIRT_IOMMU].size);
> > + qemu_fdt_setprop_cell(ms->fdt, iommu_node, "interrupt-parent",
> > irq_mmio_phandle);
> > + qemu_fdt_setprop_string_array(ms->fdt, iommu_node, "interrupt-names",
> > + (char **) &irq_names, ARRAY_SIZE(irq_names));
> > + qemu_fdt_setprop_cells(ms->fdt, iommu_node, "interrupts",
> > + IOMMU_IRQ + 0, 0x4,
> > + IOMMU_IRQ + 1, 0x4,
> > + IOMMU_IRQ + 2, 0x4,
> > + IOMMU_IRQ + 3, 0x4);
> > + qemu_fdt_setprop_cells(ms->fdt, pci_node, "iommu-map",
> > + 0x0, iommu_phandle, 0x0, 0xffff);
> > + g_free(iommu_node);
> > + g_free(pci_node);
> > +}
> > +
> > +static bool virt_is_iommu_enabled(RISCVVirtState *s)
> > +{
> > + return s->iommu != ON_OFF_AUTO_OFF;
> > +}
> > +
> > static void create_fdt(RISCVVirtState *s, const MemMapEntry *memmap)
> > {
> > MachineState *ms = MACHINE(s);
> > @@ -1051,6 +1091,10 @@ static void create_fdt(RISCVVirtState *s, const
> > MemMapEntry *memmap)
> >
> > create_fdt_pcie(s, memmap, irq_pcie_phandle, msi_pcie_phandle);
> >
> > + if (virt_is_iommu_enabled(s)) {
> > + create_fdt_iommu(s, memmap, irq_mmio_phandle);
> > + }
> > +
> > create_fdt_reset(s, memmap, &phandle);
> >
> > create_fdt_uart(s, memmap, irq_mmio_phandle);
> > @@ -1210,6 +1254,31 @@ static DeviceState *virt_create_aia(RISCVVirtAIAType
> > aia_type, int aia_guests,
> > return aplic_m;
> > }
> >
> > +static DeviceState *virt_create_iommu(RISCVVirtState *s, DeviceState
> > *irqchip)
> > +{
> > + DeviceState *iommu;
> > + int i;
> > +
> > + iommu = qdev_new(TYPE_RISCV_IOMMU_SYS);
> > +
> > + if (s->aia_type != VIRT_AIA_TYPE_APLIC_IMSIC) {
> > + /* Disable MSI_FLAT [22], MSI_MRIF [23] if IMSIC is not enabled. */
> > + qdev_prop_set_uint64(iommu, "capabilities", ~(BIT_ULL(22) |
> > BIT_ULL(23)));
> > + }
> > +
> > + /* Fixed base register address */
> > + qdev_prop_set_uint64(iommu, "addr", virt_memmap[VIRT_IOMMU].base);
> > +
> > + sysbus_realize_and_unref(SYS_BUS_DEVICE(iommu), &error_fatal);
> > +
> > + for (i = 0; i < 4; i++) {
> > + sysbus_connect_irq(SYS_BUS_DEVICE(iommu), i,
> > + qdev_get_gpio_in(irqchip, IOMMU_IRQ + i));
> > + }
> > +
> > + return iommu;
> > +}
> > +
> > static void create_platform_bus(RISCVVirtState *s, DeviceState *irqchip)
> > {
> > DeviceState *dev;
> > @@ -1506,6 +1575,10 @@ static void virt_machine_init(MachineState *machine)
> >
> > create_platform_bus(s, mmio_irqchip);
> >
> > + if (virt_is_iommu_enabled(s)) {
> > + virt_create_iommu(s, mmio_irqchip);
> > + }
> > +
> > serial_mm_init(system_memory, memmap[VIRT_UART0].base,
> > 0, qdev_get_gpio_in(mmio_irqchip, UART0_IRQ), 399193,
> > serial_hd(0), DEVICE_LITTLE_ENDIAN);
> > @@ -1533,6 +1606,7 @@ static void virt_machine_instance_init(Object *obj)
> > s->oem_id = g_strndup(ACPI_BUILD_APPNAME6, 6);
> > s->oem_table_id = g_strndup(ACPI_BUILD_APPNAME8, 8);
> > s->acpi = ON_OFF_AUTO_AUTO;
> > + s->iommu = ON_OFF_AUTO_AUTO;
> > }
> >
> > static char *virt_get_aia_guests(Object *obj, Error **errp)
> > @@ -1607,6 +1681,23 @@ static void virt_set_aclint(Object *obj, bool value,
> > Error **errp)
> > s->have_aclint = value;
> > }
> >
> > +static void virt_get_iommu(Object *obj, Visitor *v, const char *name,
> > + void *opaque, Error **errp)
> > +{
> > + RISCVVirtState *s = RISCV_VIRT_MACHINE(obj);
> > + OnOffAuto iommu = s->iommu;
> > +
> > + visit_type_OnOffAuto(v, name, &iommu, errp);
> > +}
> > +
> > +static void virt_set_iommu(Object *obj, Visitor *v, const char *name,
> > + void *opaque, Error **errp)
> > +{
> > + RISCVVirtState *s = RISCV_VIRT_MACHINE(obj);
> > +
> > + visit_type_OnOffAuto(v, name, &s->iommu, errp);
> > +}
> > +
> > bool virt_is_acpi_enabled(RISCVVirtState *s)
> > {
> > return s->acpi != ON_OFF_AUTO_OFF;
> > @@ -1683,6 +1774,13 @@ static void virt_machine_class_init(ObjectClass *oc,
> > void *data)
> > machine_class_allow_dynamic_sysbus_dev(mc, TYPE_TPM_TIS_SYSBUS);
> > #endif
> >
> > + machine_class_allow_dynamic_sysbus_dev(mc, TYPE_RISCV_IOMMU_SYS);
> > + object_class_property_add(oc, "iommu", "OnOffAuto",
> > + virt_get_iommu, virt_set_iommu,
> > + NULL, NULL);
> > + object_class_property_set_description(oc, "iommu",
> > + "Set on/off to enable/disable emulating RISC-V IOMMU platform
> > device");
> > +
> > if (tcg_enabled()) {
> > object_class_property_add_bool(oc, "aclint", virt_get_aclint,
> > virt_set_aclint);
> > diff --git a/include/hw/riscv/virt.h b/include/hw/riscv/virt.h
> > index e5c474b26e..47b9a4f103 100644
> > --- a/include/hw/riscv/virt.h
> > +++ b/include/hw/riscv/virt.h
> > @@ -59,6 +59,7 @@ struct RISCVVirtState {
> > char *oem_id;
> > char *oem_table_id;
> > OnOffAuto acpi;
> > + OnOffAuto iommu;
> > const MemMapEntry *memmap;
> > };
> >
> > @@ -73,6 +74,7 @@ enum {
> > VIRT_APLIC_M,
> > VIRT_APLIC_S,
> > VIRT_UART0,
> > + VIRT_IOMMU,
> > VIRT_VIRTIO,
> > VIRT_FW_CFG,
> > VIRT_IMSIC_M,
> > @@ -91,6 +93,7 @@ enum {
> > VIRTIO_IRQ = 1, /* 1 to 8 */
> > VIRTIO_COUNT = 8,
> > PCIE_IRQ = 0x20, /* 32 to 35 */
> > + IOMMU_IRQ = 60, /* 60 to 63 */
> > VIRT_PLATFORM_BUS_IRQ = 64, /* 64 to 95 */
> > };
> >
> > --
> > 2.34.1
> >
> >
best
- Tomasz
