Author: zbb Date: Wed Aug 19 10:36:36 2015 New Revision: 286919 URL: https://svnweb.freebsd.org/changeset/base/286919
Log: Add SMP support to GICv3 and ITS drivers Introduce supprot for SMP to GICv3 and ITS drivers. Obtained from: Semihalf Sponsored by: The FreeBSD Foundation Differential Revision: https://reviews.freebsd.org/D3299 Modified: head/sys/arm64/arm64/gic_v3.c head/sys/arm64/arm64/gic_v3_its.c head/sys/arm64/arm64/gic_v3_reg.h head/sys/arm64/arm64/gic_v3_var.h head/sys/arm64/include/cpu.h Modified: head/sys/arm64/arm64/gic_v3.c ============================================================================== --- head/sys/arm64/arm64/gic_v3.c Wed Aug 19 09:49:29 2015 (r286918) +++ head/sys/arm64/arm64/gic_v3.c Wed Aug 19 10:36:36 2015 (r286919) @@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$"); #include <sys/cpuset.h> #include <sys/lock.h> #include <sys/mutex.h> +#include <sys/smp.h> #include <vm/vm.h> #include <vm/pmap.h> @@ -61,6 +62,10 @@ static void gic_v3_dispatch(device_t, st static void gic_v3_eoi(device_t, u_int); static void gic_v3_mask_irq(device_t, u_int); static void gic_v3_unmask_irq(device_t, u_int); +#ifdef SMP +static void gic_v3_init_secondary(device_t); +static void gic_v3_ipi_send(device_t, cpuset_t, u_int); +#endif static device_method_t gic_v3_methods[] = { /* Device interface */ @@ -71,7 +76,10 @@ static device_method_t gic_v3_methods[] DEVMETHOD(pic_eoi, gic_v3_eoi), DEVMETHOD(pic_mask, gic_v3_mask_irq), DEVMETHOD(pic_unmask, gic_v3_unmask_irq), - +#ifdef SMP + DEVMETHOD(pic_init_secondary, gic_v3_init_secondary), + DEVMETHOD(pic_ipi_send, gic_v3_ipi_send), +#endif /* End */ DEVMETHOD_END }; @@ -95,6 +103,7 @@ enum gic_v3_xdist { /* Helper routines starting with gic_v3_ */ static int gic_v3_dist_init(struct gic_v3_softc *); +static int gic_v3_redist_alloc(struct gic_v3_softc *); static int gic_v3_redist_find(struct gic_v3_softc *); static int gic_v3_redist_init(struct gic_v3_softc *); static int gic_v3_cpu_init(struct gic_v3_softc *); @@ -105,11 +114,21 @@ typedef int (*gic_v3_initseq_t) (struct /* Primary CPU initialization sequence */ static gic_v3_initseq_t gic_v3_primary_init[] = { gic_v3_dist_init, + gic_v3_redist_alloc, gic_v3_redist_init, gic_v3_cpu_init, NULL }; +#ifdef SMP +/* Secondary CPU initialization sequence */ +static gic_v3_initseq_t gic_v3_secondary_init[] = { + gic_v3_redist_init, + gic_v3_cpu_init, + NULL +}; +#endif + /* * Device interface. */ @@ -213,7 +232,7 @@ gic_v3_detach(device_t dev) for (rid = 0; rid < (sc->gic_redists.nregions + 1); rid++) bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->gic_res[rid]); - for (i = 0; i < MAXCPU; i++) + for (i = 0; i < mp_ncpus; i++) free(sc->gic_redists.pcpu[i], M_GIC_V3); free(sc->gic_res, M_GIC_V3); @@ -258,13 +277,9 @@ gic_v3_dispatch(device_t dev, struct tra } if (active_irq <= GIC_LAST_SGI) { - /* - * TODO: Implement proper SGI handling. - * Mask it if such is received for some reason. - */ - device_printf(dev, - "Received unsupported interrupt type: SGI\n"); - PIC_MASK(dev, active_irq); + gic_icc_write(EOIR1, (uint64_t)active_irq); + arm_dispatch_intr(active_irq, frame); + continue; } } } @@ -283,7 +298,7 @@ gic_v3_mask_irq(device_t dev, u_int irq) sc = device_get_softc(dev); - if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_PPI) { /* PPIs in corresponding Re-Distributor */ + if (irq <= GIC_LAST_PPI) { /* SGIs and PPIs in corresponding Re-Distributor */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ICENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, REDIST); @@ -303,7 +318,7 @@ gic_v3_unmask_irq(device_t dev, u_int ir sc = device_get_softc(dev); - if (irq >= GIC_FIRST_PPI && irq <= GIC_LAST_PPI) { /* PPIs in corresponding Re-Distributor */ + if (irq <= GIC_LAST_PPI) { /* SGIs and PPIs in corresponding Re-Distributor */ gic_r_write(sc, 4, GICR_SGI_BASE_SIZE + GICD_ISENABLER(irq), GICD_I_MASK(irq)); gic_v3_wait_for_rwp(sc, REDIST); @@ -316,6 +331,101 @@ gic_v3_unmask_irq(device_t dev, u_int ir panic("%s: Unsupported IRQ number %u", __func__, irq); } +#ifdef SMP +static void +gic_v3_init_secondary(device_t dev) +{ + struct gic_v3_softc *sc; + gic_v3_initseq_t *init_func; + int err; + + sc = device_get_softc(dev); + + /* Train init sequence for boot CPU */ + for (init_func = gic_v3_secondary_init; *init_func != NULL; init_func++) { + err = (*init_func)(sc); + if (err != 0) { + device_printf(dev, + "Could not initialize GIC for CPU%u\n", + PCPU_GET(cpuid)); + return; + } + } + + /* + * Try to initialize ITS. + * If there is no driver attached this routine will fail but that + * does not mean failure here as only LPIs will not be functional + * on the current CPU. + */ + if (its_init_cpu(NULL) != 0) { + device_printf(dev, + "Could not initialize ITS for CPU%u. " + "No LPIs will arrive on this CPU\n", + PCPU_GET(cpuid)); + } + + /* + * ARM64TODO: Unmask timer PPIs. To be removed when appropriate + * mechanism is implemented. + * Activate the timer interrupts: virtual (27), secure (29), + * and non-secure (30). Use hardcoded values here as there + * should be no defines for them. + */ + gic_v3_unmask_irq(dev, 27); + gic_v3_unmask_irq(dev, 29); + gic_v3_unmask_irq(dev, 30); +} + +static void +gic_v3_ipi_send(device_t dev, cpuset_t cpuset, u_int ipi) +{ + u_int cpu; + uint64_t aff, tlist; + uint64_t val; + uint64_t aff_mask; + + /* Set affinity mask to match level 3, 2 and 1 */ + aff_mask = CPU_AFF1_MASK | CPU_AFF2_MASK | CPU_AFF3_MASK; + + /* Iterate through all CPUs in set */ + while (!CPU_EMPTY(&cpuset)) { + aff = tlist = 0; + for (cpu = 0; cpu < mp_ncpus; cpu++) { + /* Compose target list for single AFF3:AFF2:AFF1 set */ + if (CPU_ISSET(cpu, &cpuset)) { + if (!tlist) { + /* + * Save affinity of the first CPU to + * send IPI to for later comparison. + */ + aff = CPU_AFFINITY(cpu); + tlist |= (1UL << CPU_AFF0(aff)); + CPU_CLR(cpu, &cpuset); + } + /* Check for same Affinity level 3, 2 and 1 */ + if ((aff & aff_mask) == (CPU_AFFINITY(cpu) & aff_mask)) { + tlist |= (1UL << CPU_AFF0(CPU_AFFINITY(cpu))); + /* Clear CPU in cpuset from target list */ + CPU_CLR(cpu, &cpuset); + } + } + } + if (tlist) { + KASSERT((tlist & ~GICI_SGI_TLIST_MASK) == 0, + ("Target list too long for GICv3 IPI")); + /* Send SGI to CPUs in target list */ + val = tlist; + val |= (uint64_t)CPU_AFF3(aff) << GICI_SGI_AFF3_SHIFT; + val |= (uint64_t)CPU_AFF2(aff) << GICI_SGI_AFF2_SHIFT; + val |= (uint64_t)CPU_AFF1(aff) << GICI_SGI_AFF1_SHIFT; + val |= (uint64_t)(ipi & GICI_SGI_IPI_MASK) << GICI_SGI_IPI_SHIFT; + gic_icc_write(SGI1R, val); + } + } +} +#endif + /* * Helper routines */ @@ -463,6 +573,22 @@ gic_v3_dist_init(struct gic_v3_softc *sc /* Re-Distributor */ static int +gic_v3_redist_alloc(struct gic_v3_softc *sc) +{ + u_int cpuid; + + /* Allocate struct resource for all CPU's Re-Distributor registers */ + for (cpuid = 0; cpuid < mp_ncpus; cpuid++) + if (CPU_ISSET(cpuid, &all_cpus) != 0) + sc->gic_redists.pcpu[cpuid] = + malloc(sizeof(*sc->gic_redists.pcpu[0]), + M_GIC_V3, M_WAITOK); + else + sc->gic_redists.pcpu[cpuid] = NULL; + return (0); +} + +static int gic_v3_redist_find(struct gic_v3_softc *sc) { struct resource r_res; @@ -475,10 +601,6 @@ gic_v3_redist_find(struct gic_v3_softc * cpuid = PCPU_GET(cpuid); - /* Allocate struct resource for this CPU's Re-Distributor registers */ - sc->gic_redists.pcpu[cpuid] = - malloc(sizeof(*sc->gic_redists.pcpu[0]), M_GIC_V3, M_WAITOK); - aff = CPU_AFFINITY(cpuid); /* Affinity in format for comparison with typer */ aff = (CPU_AFF3(aff) << 24) | (CPU_AFF2(aff) << 16) | @@ -502,7 +624,6 @@ gic_v3_redist_find(struct gic_v3_softc * default: device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); - free(sc->gic_redists.pcpu[cpuid], M_GIC_V3); return (ENODEV); } @@ -531,7 +652,6 @@ gic_v3_redist_find(struct gic_v3_softc * } while ((typer & GICR_TYPER_LAST) == 0); } - free(sc->gic_redists.pcpu[cpuid], M_GIC_V3); device_printf(sc->dev, "No Re-Distributor found for CPU%u\n", cpuid); return (ENXIO); } Modified: head/sys/arm64/arm64/gic_v3_its.c ============================================================================== --- head/sys/arm64/arm64/gic_v3_its.c Wed Aug 19 09:49:29 2015 (r286918) +++ head/sys/arm64/arm64/gic_v3_its.c Wed Aug 19 10:36:36 2015 (r286919) @@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$"); #include <sys/pcpu.h> #include <sys/lock.h> #include <sys/mutex.h> +#include <sys/smp.h> #include <dev/pci/pcireg.h> #include <dev/pci/pcivar.h> @@ -88,7 +89,6 @@ MALLOC_DEFINE(M_GIC_V3_ITS, "GICv3 ITS", static int its_alloc_tables(struct gic_v3_its_softc *); static void its_free_tables(struct gic_v3_its_softc *); static void its_init_commandq(struct gic_v3_its_softc *); -static int its_init_cpu(struct gic_v3_its_softc *); static void its_init_cpu_collection(struct gic_v3_its_softc *); static uint32_t its_get_devid(device_t); @@ -105,8 +105,8 @@ static uint32_t its_get_devbits(device_t static void lpi_init_conftable(struct gic_v3_its_softc *); static void lpi_bitmap_init(struct gic_v3_its_softc *); -static void lpi_init_cpu(struct gic_v3_its_softc *); static int lpi_config_cpu(struct gic_v3_its_softc *); +static void lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *); const char *its_ptab_cache[] = { [GITS_BASER_CACHE_NCNB] = "(NC,NB)", @@ -223,8 +223,12 @@ gic_v3_its_attach(device_t dev) } /* 3. Allocate collections. One per-CPU */ - sc->its_cols = malloc(sizeof(*sc->its_cols) * MAXCPU, - M_GIC_V3_ITS, (M_WAITOK | M_ZERO)); + for (int cpu = 0; cpu < mp_ncpus; cpu++) + if (CPU_ISSET(cpu, &all_cpus) != 0) + sc->its_cols[cpu] = malloc(sizeof(*sc->its_cols[0]), + M_GIC_V3_ITS, (M_WAITOK | M_ZERO)); + else + sc->its_cols[cpu] = NULL; /* 4. Enable ITS in GITS_CTLR */ gits_tmp = gic_its_read(sc, 4, GITS_CTLR); @@ -236,10 +240,13 @@ gic_v3_its_attach(device_t dev) /* 6. LPIs bitmap init */ lpi_bitmap_init(sc); - /* 7. CPU init */ + /* 7. Allocate pending tables for all CPUs */ + lpi_alloc_cpu_pendtables(sc); + + /* 8. CPU init */ (void)its_init_cpu(sc); - /* 8. Init ITS devices list */ + /* 9. Init ITS devices list */ TAILQ_INIT(&sc->its_dev_list); arm_register_msi_pic(dev); @@ -280,7 +287,8 @@ gic_v3_its_detach(device_t dev) /* ITTs */ its_free_tables(sc); /* Collections */ - free(sc->its_cols, M_GIC_V3_ITS); + for (cpuid = 0; cpuid < mp_ncpus; cpuid++) + free(sc->its_cols[cpuid], M_GIC_V3_ITS); /* LPI config table */ parent = device_get_parent(sc->dev); gic_sc = device_get_softc(parent); @@ -288,10 +296,13 @@ gic_v3_its_detach(device_t dev) contigfree((void *)gic_sc->gic_redists.lpis.conf_base, LPI_CONFTAB_SIZE, M_GIC_V3_ITS); } - if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) { - contigfree((void *)gic_sc->gic_redists.lpis.pend_base[cpuid], - roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS); - } + for (cpuid = 0; cpuid < mp_ncpus; cpuid++) + if ((void *)gic_sc->gic_redists.lpis.pend_base[cpuid] != NULL) { + contigfree( + (void *)gic_sc->gic_redists.lpis.pend_base[cpuid], + roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), + M_GIC_V3_ITS); + } /* Resource... */ bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->its_res); @@ -525,13 +536,31 @@ its_init_commandq(struct gic_v3_its_soft gic_its_write(sc, 8, GITS_CWRITER, 0x0); } -static int +int its_init_cpu(struct gic_v3_its_softc *sc) { device_t parent; struct gic_v3_softc *gic_sc; /* + * NULL in place of the softc pointer means that + * this function was called during GICv3 secondary initialization. + */ + if (sc == NULL) { + if (device_is_attached(its_sc->dev)) { + /* + * XXX ARM64TODO: This is part of the workaround that + * saves ITS software context for further use in + * mask/unmask and here. This should be removed as soon + * as the upper layer is capable of passing the ITS + * context to this function. + */ + sc = its_sc; + } else + return (ENXIO); + } + + /* * Check for LPIs support on this Re-Distributor. */ parent = device_get_parent(sc->dev); @@ -544,8 +573,8 @@ its_init_cpu(struct gic_v3_its_softc *sc return (ENXIO); } - /* Initialize LPIs for this CPU */ - lpi_init_cpu(sc); + /* Configure LPIs for this CPU */ + lpi_config_cpu(sc); /* Initialize collections */ its_init_cpu_collection(sc); @@ -582,11 +611,12 @@ its_init_cpu_collection(struct gic_v3_it target = GICR_TYPER_CPUNUM(typer); } - sc->its_cols[cpuid].col_target = target; - sc->its_cols[cpuid].col_id = cpuid; + sc->its_cols[cpuid]->col_target = target; + sc->its_cols[cpuid]->col_id = cpuid; + + its_cmd_mapc(sc, sc->its_cols[cpuid], 1); + its_cmd_invall(sc, sc->its_cols[cpuid]); - its_cmd_mapc(sc, &sc->its_cols[cpuid], 1); - its_cmd_invall(sc, &sc->its_cols[cpuid]); } static void @@ -633,7 +663,7 @@ lpi_init_conftable(struct gic_v3_its_sof } static void -lpi_init_cpu(struct gic_v3_its_softc *sc) +lpi_alloc_cpu_pendtables(struct gic_v3_its_softc *sc) { device_t parent; struct gic_v3_softc *gic_sc; @@ -647,25 +677,31 @@ lpi_init_cpu(struct gic_v3_its_softc *sc * LPI Pending Table settings. * This has to be done for each Re-Distributor, hence for each CPU. */ - cpuid = PCPU_GET(cpuid); + for (cpuid = 0; cpuid < mp_ncpus; cpuid++) { - pend_base = (vm_offset_t)contigmalloc( - roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS, - (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); - - /* Clean D-cache so that ITS can see zeroed pages */ - cpu_dcache_wb_range((vm_offset_t)pend_base, - roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K)); + /* Limit allocation to active CPUs only */ + if (CPU_ISSET(cpuid, &all_cpus) == 0) + continue; - if (bootverbose) { - device_printf(sc->dev, - "LPI Pending Table for CPU%u at PA: 0x%lx\n", - cpuid, vtophys(pend_base)); - } + pend_base = (vm_offset_t)contigmalloc( + roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K), M_GIC_V3_ITS, + (M_WAITOK | M_ZERO), 0, ~0UL, PAGE_SIZE_64K, 0); + + /* Clean D-cache so that ITS can see zeroed pages */ + cpu_dcache_wb_range((vm_offset_t)pend_base, + roundup2(LPI_PENDTAB_SIZE, PAGE_SIZE_64K)); - gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base; + if (bootverbose) { + device_printf(sc->dev, + "LPI Pending Table for CPU%u at PA: 0x%lx\n", + cpuid, vtophys(pend_base)); + } - lpi_config_cpu(sc); + gic_sc->gic_redists.lpis.pend_base[cpuid] = pend_base; + } + + /* Ensure visibility of pend_base addresses on other CPUs */ + wmb(); } static int @@ -683,6 +719,9 @@ lpi_config_cpu(struct gic_v3_its_softc * gic_sc = device_get_softc(parent); cpuid = PCPU_GET(cpuid); + /* Ensure data observability on a current CPU */ + rmb(); + conf_base = gic_sc->gic_redists.lpis.conf_base; pend_base = gic_sc->gic_redists.lpis.pend_base[cpuid]; @@ -1379,7 +1418,7 @@ its_device_alloc_locked(struct gic_v3_it * to be bound to the CPU that performs the configuration. */ cpuid = PCPU_GET(cpuid); - newdev->col = &sc->its_cols[cpuid]; + newdev->col = sc->its_cols[cpuid]; TAILQ_INSERT_TAIL(&sc->its_dev_list, newdev, entry); Modified: head/sys/arm64/arm64/gic_v3_reg.h ============================================================================== --- head/sys/arm64/arm64/gic_v3_reg.h Wed Aug 19 09:49:29 2015 (r286918) +++ head/sys/arm64/arm64/gic_v3_reg.h Wed Aug 19 10:36:36 2015 (r286919) @@ -356,6 +356,12 @@ /* * CPU interface */ +#define GICI_SGI_TLIST_MASK (0xffffUL) +#define GICI_SGI_AFF1_SHIFT (16UL) +#define GICI_SGI_AFF2_SHIFT (32UL) +#define GICI_SGI_AFF3_SHIFT (48UL) +#define GICI_SGI_IPI_MASK (0xfUL) +#define GICI_SGI_IPI_SHIFT (24UL) /* * Registers list (ICC_xyz_EL1): Modified: head/sys/arm64/arm64/gic_v3_var.h ============================================================================== --- head/sys/arm64/arm64/gic_v3_var.h Wed Aug 19 09:49:29 2015 (r286918) +++ head/sys/arm64/arm64/gic_v3_var.h Wed Aug 19 10:36:36 2015 (r286919) @@ -221,7 +221,7 @@ struct gic_v3_its_softc { struct its_cmd * its_cmdq_base; /* ITS command queue base */ struct its_cmd * its_cmdq_write; /* ITS command queue write ptr */ struct its_ptab its_ptabs[GITS_BASER_NUM];/* ITS private tables */ - struct its_col * its_cols; /* Per-CPU collections */ + struct its_col * its_cols[MAXCPU];/* Per-CPU collections */ uint64_t its_flags; @@ -253,6 +253,8 @@ int gic_v3_its_alloc_msix(device_t, devi int gic_v3_its_alloc_msi(device_t, device_t, int, int *); int gic_v3_its_map_msix(device_t, device_t, int, uint64_t *, uint32_t *); +int its_init_cpu(struct gic_v3_its_softc *); + void lpi_unmask_irq(device_t, uint32_t); void lpi_mask_irq(device_t, uint32_t); /* Modified: head/sys/arm64/include/cpu.h ============================================================================== --- head/sys/arm64/include/cpu.h Wed Aug 19 09:49:29 2015 (r286918) +++ head/sys/arm64/include/cpu.h Wed Aug 19 10:36:36 2015 (r286919) @@ -57,7 +57,12 @@ #define CPU_AFF1(mpidr) (u_int)(((mpidr) >> 8) & 0xff) #define CPU_AFF2(mpidr) (u_int)(((mpidr) >> 16) & 0xff) #define CPU_AFF3(mpidr) (u_int)(((mpidr) >> 32) & 0xff) -#define CPU_AFF_MASK 0xff00ffffffUL /* Mask affinity fields in MPIDR_EL1 */ +#define CPU_AFF0_MASK 0xffUL +#define CPU_AFF1_MASK 0xff00UL +#define CPU_AFF2_MASK 0xff0000UL +#define CPU_AFF3_MASK 0xff00000000UL +#define CPU_AFF_MASK (CPU_AFF0_MASK | CPU_AFF1_MASK | \ + CPU_AFF2_MASK| CPU_AFF3_MASK) /* Mask affinity fields in MPIDR_EL1 */ #ifdef _KERNEL _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
