The branch main has been updated by mmel: URL: https://cgit.FreeBSD.org/src/commit/?id=62e1227219f2f79a2ed7a672149dc3d774d25dd4
commit 62e1227219f2f79a2ed7a672149dc3d774d25dd4 Author: Michal Meloun <[email protected]> AuthorDate: 2025-11-06 18:56:10 +0000 Commit: Michal Meloun <[email protected]> CommitDate: 2025-12-03 06:47:41 +0000 ARM GICv3: Add support for non-coherent and/or bus address range limited DMA. Due to an implementation bug, the GICv3 and ITS master ports may be connected to a non-coherent bus. The new DT resolves this issue by marking these ports with the "dma-nocoherent" attribute. The older DT does not have this attribute, so we must match the affected SoC. Additionally, the RK356x family has GIC master ports on a 32-bit bus. Therefore, we must limit the address range for all tables and command buffers allocated for it. In this case, the DT does not have an attribute for this case, so the quirk should only be applied by SoC matching. MFC after: 4 weeks Reviewed by: imp Differential Revision: https://reviews.freebsd.org/D47488 --- sys/arm64/arm64/gic_v3.c | 4 ++ sys/arm64/arm64/gic_v3_fdt.c | 15 ++++- sys/arm64/arm64/gic_v3_var.h | 6 ++ sys/arm64/arm64/gicv3_its.c | 156 ++++++++++++++++++++++++++++++++++++------- 4 files changed, 157 insertions(+), 24 deletions(-) diff --git a/sys/arm64/arm64/gic_v3.c b/sys/arm64/arm64/gic_v3.c index 641b6d6dbc5e..d74cdc20dd20 100644 --- a/sys/arm64/arm64/gic_v3.c +++ b/sys/arm64/arm64/gic_v3.c @@ -494,6 +494,9 @@ gic_v3_read_ivar(device_t dev, device_t child, int which, uintptr_t *result) case GICV3_IVAR_REDIST: *result = (uintptr_t)&sc->gic_redists.pcpu[PCPU_GET(cpuid)]; return (0); + case GICV3_IVAR_FLAGS: + *result = sc->gic_flags; + return (0); case GIC_IVAR_SUPPORT_LPIS: *result = (gic_d_read(sc, 4, GICD_TYPER) & GICD_TYPER_LPIS) != 0; @@ -530,6 +533,7 @@ gic_v3_write_ivar(device_t dev, device_t child, int which, uintptr_t value) switch(which) { case GICV3_IVAR_NIRQS: case GICV3_IVAR_REDIST: + case GICV3_IVAR_FLAGS: case GIC_IVAR_HW_REV: case GIC_IVAR_BUS: return (EINVAL); diff --git a/sys/arm64/arm64/gic_v3_fdt.c b/sys/arm64/arm64/gic_v3_fdt.c index 4bea4040c0ba..871516145168 100644 --- a/sys/arm64/arm64/gic_v3_fdt.c +++ b/sys/arm64/arm64/gic_v3_fdt.c @@ -110,7 +110,7 @@ gic_v3_fdt_attach(device_t dev) { struct gic_v3_softc *sc; pcell_t redist_regions; - intptr_t xref; + phandle_t xref, node; int err; uint32_t *mbi_ranges; ssize_t ret; @@ -118,6 +118,19 @@ gic_v3_fdt_attach(device_t dev) sc = device_get_softc(dev); sc->dev = dev; sc->gic_bus = GIC_BUS_FDT; + node = ofw_bus_get_node(dev); + + /* + * Limit DMA shareability. "dma-noncoherent" was introduced in DT 6.15. + * For compatibility with previous versions, also use a match based on + * affected SoCs. + */ + if (OF_hasprop(node, "dma-noncoherent") || + ofw_bus_is_machine_compatible("rockchip,rk3566") || + ofw_bus_is_machine_compatible("rockchip,rk3568") || + ofw_bus_is_machine_compatible("rockchip,rk3588") || + ofw_bus_is_machine_compatible("rockchip,rk3588s")) + sc->gic_flags |= GIC_V3_FLAGS_FORCE_NOSHAREABLE; /* * Recover number of the Re-Distributor regions. diff --git a/sys/arm64/arm64/gic_v3_var.h b/sys/arm64/arm64/gic_v3_var.h index 2570834c2818..a490b69b7417 100644 --- a/sys/arm64/arm64/gic_v3_var.h +++ b/sys/arm64/arm64/gic_v3_var.h @@ -63,6 +63,8 @@ struct gic_redists { struct redist_pcpu *pcpu; }; +#define GIC_V3_FLAGS_FORCE_NOSHAREABLE 0x0000001 + struct gic_v3_softc { device_t dev; struct resource ** gic_res; @@ -92,6 +94,8 @@ struct gic_v3_softc { int nranges; struct arm_gic_range * ranges; + + uint32_t gic_flags; }; struct gic_v3_devinfo { @@ -108,9 +112,11 @@ MALLOC_DECLARE(M_GIC_V3); #define GICV3_IVAR_NIRQS 1000 /* 1001 was GICV3_IVAR_REDIST_VADDR */ #define GICV3_IVAR_REDIST 1002 +#define GICV3_IVAR_FLAGS 1003 __BUS_ACCESSOR(gicv3, nirqs, GICV3, NIRQS, u_int); __BUS_ACCESSOR(gicv3, redist, GICV3, REDIST, void *); +__BUS_ACCESSOR(gicv3, flags, GICV3, FLAGS, uint32_t); /* Device methods */ int gic_v3_attach(device_t dev); diff --git a/sys/arm64/arm64/gicv3_its.c b/sys/arm64/arm64/gicv3_its.c index 7821b1512083..06491f6ffe05 100644 --- a/sys/arm64/arm64/gicv3_its.c +++ b/sys/arm64/arm64/gicv3_its.c @@ -287,28 +287,50 @@ struct gicv3_its_softc { #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 #define ITS_FLAGS_LPI_PREALLOC 0x00000008 +#define ITS_FLAGS_FORCE_NOSHAREABLE 0x00000010 u_int sc_its_flags; bool trace_enable; vm_page_t ma; /* fake msi page */ + vm_paddr_t malloc_max_addr; /* max address for contigmalloc */ }; +typedef bool (its_quirk_detect_t)(device_t); typedef void (its_quirk_func_t)(device_t); -static its_quirk_func_t its_quirk_cavium_22375; + +static its_quirk_detect_t its_detect_cavium_22375; +static its_quirk_func_t its_quirk_cavium_22375; +#ifdef FDT +static its_quirk_detect_t its_detect_rk356x; +static its_quirk_func_t its_quirk_rk356x; +static its_quirk_detect_t its_detect_rk3588; +static its_quirk_func_t its_quirk_rk3588; +#endif static const struct { const char *desc; - uint32_t iidr; - uint32_t iidr_mask; + its_quirk_detect_t *detect; its_quirk_func_t *func; } its_quirks[] = { { /* Cavium ThunderX Pass 1.x */ .desc = "Cavium ThunderX errata: 22375, 24313", - .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, - GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), - .iidr_mask = ~GITS_IIDR_REVISION_MASK, + .detect = its_detect_cavium_22375, .func = its_quirk_cavium_22375, }, +#ifdef FDT + { + /* Rockchip RK356X implementation bugs */ + .desc = "RK356X ITS errata", + .detect = its_detect_rk356x, + .func = its_quirk_rk356x, + }, + { + /* Rockchip RK3588 implementation bugs */ + .desc = "RK3588 ITS errata", + .detect = its_detect_rk3588, + .func = its_quirk_rk3588, + }, +#endif }; #define gic_its_read_4(sc, reg) \ @@ -387,6 +409,13 @@ static device_method_t gicv3_its_methods[] = { static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, sizeof(struct gicv3_its_softc)); +/* Limit maximum address for memory mapped tables and buffers */ +static vm_paddr_t +gicv3_its_limit_max_addr(struct gicv3_its_softc *sc, vm_paddr_t addr) +{ + return (sc->malloc_max_addr > addr ? addr: sc->malloc_max_addr); +} + static void gicv3_its_cmdq_init(struct gicv3_its_softc *sc) { @@ -395,7 +424,8 @@ gicv3_its_cmdq_init(struct gicv3_its_softc *sc) /* Set up the command circular buffer */ sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS, - sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, + sc->sc_ds, M_WAITOK | M_ZERO, 0, + gicv3_its_limit_max_addr(sc, (1ul << 48) - 1), ITS_CMDQ_ALIGN, 0); sc->sc_its_cmd_next_idx = 0; @@ -404,8 +434,11 @@ gicv3_its_cmdq_init(struct gicv3_its_softc *sc) /* Set the base of the command buffer */ reg = GITS_CBASER_VALID | (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | - cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | - (ITS_CMDQ_SIZE / 4096 - 1); + cmd_paddr | (ITS_CMDQ_SIZE / 4096 - 1); + if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE) + reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; + else + reg |= GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT; gic_its_write_8(sc, GITS_CBASER, reg); /* Read back to check for fixed value fields */ @@ -532,10 +565,17 @@ gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) cache = 0; } else { devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); - cache = GITS_BASER_CACHE_RAWAWB; + if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE) + cache = GITS_BASER_CACHE_NC; + else + cache = GITS_BASER_CACHE_RAWAWB; } sc->sc_devbits = devbits; - share = GITS_BASER_SHARE_IS; + + if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE) + share = GITS_BASER_SHARE_NS; + else + share = GITS_BASER_SHARE_IS; for (i = 0; i < GITS_BASER_NUM; i++) { reg = gic_its_read_8(sc, GITS_BASER(i)); @@ -611,7 +651,8 @@ gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) /* Allocate the table */ table = contigmalloc_domainset(npages * PAGE_SIZE, M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, - (1ul << 48) - 1, PAGE_SIZE_64K, 0); + gicv3_its_limit_max_addr(sc, (1ul << 48) - 1), + PAGE_SIZE_64K, 0); sc->sc_its_ptab[i].ptab_vaddr = table; sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size; @@ -741,7 +782,8 @@ gicv3_its_conftable_init(struct gicv3_its_softc *sc) * PROPBASER register later in its_init_cpu_lpi(). */ conf_table = contigmalloc(LPI_CONFTAB_SIZE, - M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, + M_GICV3_ITS, M_WAITOK, 0, + gicv3_its_limit_max_addr(sc, LPI_CONFTAB_MAX_ADDR), LPI_CONFTAB_ALIGN, 0); } sc->sc_conf_base = conf_table; @@ -766,7 +808,9 @@ gicv3_its_pendtables_init(struct gicv3_its_softc *sc) sc->sc_pend_base[i] = contigmalloc( LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, - 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); + 0, + gicv3_its_limit_max_addr(sc, LPI_PENDTAB_MAX_ADDR), + LPI_PENDTAB_ALIGN, 0); /* Flush so the ITS can see the memory */ cpu_dcache_wb_range(sc->sc_pend_base[i], @@ -804,10 +848,12 @@ its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc) size = ilog2_long(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1; xbaser = vtophys(sc->sc_conf_base) | - (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | size; - + if (gicv3_get_flags(sc->dev) & GIC_V3_FLAGS_FORCE_NOSHAREABLE) + xbaser |= GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT; + else + xbaser |= GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT; gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); /* Check the cache attributes we set */ @@ -835,8 +881,11 @@ its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc) * Set the LPI pending table base */ xbaser = vtophys(sc->sc_pend_base[cpuid]) | - (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | - (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); + (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT); + if (sc->sc_its_flags & ITS_FLAGS_FORCE_NOSHAREABLE) + xbaser |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; + else + xbaser |= GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT; gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); @@ -1017,6 +1066,7 @@ gicv3_its_attach(device_t dev) sc->sc_irq_length = gicv3_get_nirqs(dev); sc->sc_irq_base = GIC_FIRST_LPI; sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; + sc->malloc_max_addr = ~0; rid = 0; sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, @@ -1034,7 +1084,7 @@ gicv3_its_attach(device_t dev) CPU_COPY(&all_cpus, &sc->sc_cpus); iidr = gic_its_read_4(sc, GITS_IIDR); for (i = 0; i < nitems(its_quirks); i++) { - if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { + if (its_quirks[i].detect(dev)) { if (bootverbose) { device_printf(dev, "Applying %s\n", its_quirks[i].desc); @@ -1128,6 +1178,21 @@ gicv3_its_detach(device_t dev) return (ENXIO); } +static bool +its_detect_cavium_22375(device_t dev) +{ + uint32_t iidr; + struct gicv3_its_softc *sc; + + sc = device_get_softc(dev); + iidr = gic_its_read_4(sc, GITS_IIDR); + if ((iidr & ~GITS_IIDR_REVISION_MASK) == + GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, GITS_IIDR_PROD_THUNDER, + GITS_IIDR_VAR_THUNDER_1, 0)) + return (true); + return(false); +} + static void its_quirk_cavium_22375(device_t dev) { @@ -1151,6 +1216,46 @@ its_quirk_cavium_22375(device_t dev) } } +#ifdef FDT +static bool +its_detect_rk356x(device_t dev) +{ + + if (ofw_bus_is_machine_compatible("rockchip,rk3566") || + ofw_bus_is_machine_compatible("rockchip,rk3568")) + return (true); + return(false); +} + +static void +its_quirk_rk356x(device_t dev) +{ + struct gicv3_its_softc *sc; + + sc = device_get_softc(dev); + sc->malloc_max_addr = (1ul << 32) - 1; +} + +static bool +its_detect_rk3588(device_t dev) +{ + + if (ofw_bus_is_machine_compatible("rockchip,rk3588") || + ofw_bus_is_machine_compatible("rockchip,rk3588s")) + return (true); + return(false); +} + +static void +its_quirk_rk3588(device_t dev) +{ + struct gicv3_its_softc *sc; + + sc = device_get_softc(dev); + sc->sc_its_flags |= ITS_FLAGS_FORCE_NOSHAREABLE; +} +#endif + static void gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) { @@ -1400,7 +1505,8 @@ its_device_alloc(struct gicv3_its_softc *sc, int devid) shareable = false; l2_table = contigmalloc_domainset(ptable->ptab_l2_size, - M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, + M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, + gicv3_its_limit_max_addr(sc, (1ul << 48) - 1), ptable->ptab_page_size, 0); if (!shareable) @@ -1461,7 +1567,8 @@ its_device_get(device_t dev, device_t child, u_int nvecs) itt_size = roundup2(MAX(nvecs, 2) * esize, 256); its_dev->itt = contigmalloc_domainset(itt_size, M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0, - LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0); + gicv3_its_limit_max_addr(sc, LPI_INT_TRANS_TAB_MAX_ADDR), + LPI_INT_TRANS_TAB_ALIGN, 0); if (its_dev->itt == NULL) { vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); free(its_dev, M_GICV3_ITS); @@ -2233,17 +2340,20 @@ static int gicv3_its_fdt_attach(device_t dev) { struct gicv3_its_softc *sc; - phandle_t xref; + phandle_t xref, node; int err; sc = device_get_softc(dev); sc->dev = dev; + node = ofw_bus_get_node(dev); err = gicv3_its_attach(dev); if (err != 0) return (err); + if (OF_hasprop(node, "dma-noncoherent")) + sc->sc_its_flags |= ITS_FLAGS_FORCE_NOSHAREABLE; /* Register this device as a interrupt controller */ - xref = OF_xref_from_node(ofw_bus_get_node(dev)); + xref = OF_xref_from_node(node); sc->sc_pic = intr_pic_register(dev, xref); err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
