Author: kib Date: Thu Dec 3 11:14:14 2015 New Revision: 291688 URL: https://svnweb.freebsd.org/changeset/base/291688
Log: For amd64 non-PCID machines, and for i386 machines with support for the PG_G global pte flag, pmap_invalidate_all() fails to flush global TLB entries [*]. This is because TLB shootdown handler for such configs reloads CR3, and on i386 pmap_invalidate_all() does the same for the initiating CPU. Note that current code does not issue total invalidation requests for the kernel_pmap. Rename amd64 function invltlb_globpcid() to invltlb_glob(), it is not specific for PCID for quite some time, and implement the same functionality for i386. Use the function instead of invltlb() in shootdown handlers and in i386 pmap_invalidate_all(), but only for the kernel pmap (which maps pages with the PG_G attribute set), which takes care of PG_G TLB entries on flush. To detect the affected pmap in i386 TLB shootdown handler, pmap should be passed to the smp_masked_invltlb() function, which makes amd64 and i386 TLB shootdown code almost identical. Merge the code under x86/. Noted by: jhb [*] Reviewed by: cem, jhb, pho Tested by: pho Sponsored by: The FreeBSD Foundation Differential revision: https://reviews.freebsd.org/D4346 Modified: head/sys/amd64/amd64/mp_machdep.c head/sys/amd64/amd64/pmap.c head/sys/amd64/include/cpufunc.h head/sys/amd64/include/smp.h head/sys/i386/i386/mp_machdep.c head/sys/i386/i386/pmap.c head/sys/i386/include/pmap.h head/sys/i386/include/smp.h head/sys/x86/x86/mp_x86.c Modified: head/sys/amd64/amd64/mp_machdep.c ============================================================================== --- head/sys/amd64/amd64/mp_machdep.c Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/amd64/amd64/mp_machdep.c Thu Dec 3 11:14:14 2015 (r291688) @@ -87,11 +87,6 @@ extern struct pcpu __pcpu[]; char *doublefault_stack; char *nmi_stack; -/* Variables needed for SMP tlb shootdown. */ -static vm_offset_t smp_tlb_addr1, smp_tlb_addr2; -static pmap_t smp_tlb_pmap; -volatile int smp_tlb_wait; - extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); /* @@ -410,121 +405,6 @@ start_ap(int apic_id) return 0; /* return FAILURE */ } -/* - * Flush the TLB on other CPU's - */ - -static void -smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap, - vm_offset_t addr1, vm_offset_t addr2) -{ - int cpu, ncpu, othercpus; - - othercpus = mp_ncpus - 1; /* does not shootdown self */ - - /* - * Check for other cpus. Return if none. - */ - if (CPU_ISFULLSET(&mask)) { - if (othercpus < 1) - return; - } else { - CPU_CLR(PCPU_GET(cpuid), &mask); - if (CPU_EMPTY(&mask)) - return; - } - - if (!(read_rflags() & PSL_I)) - panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_ipi_mtx); - smp_tlb_addr1 = addr1; - smp_tlb_addr2 = addr2; - smp_tlb_pmap = pmap; - smp_tlb_wait = 0; - if (CPU_ISFULLSET(&mask)) { - ncpu = othercpus; - ipi_all_but_self(vector); - } else { - ncpu = 0; - while ((cpu = CPU_FFS(&mask)) != 0) { - cpu--; - CPU_CLR(cpu, &mask); - CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, - cpu, vector); - ipi_send_cpu(cpu, vector); - ncpu++; - } - } - while (smp_tlb_wait < ncpu) - ia32_pause(); - mtx_unlock_spin(&smp_ipi_mtx); -} - -void -smp_masked_invltlb(cpuset_t mask, pmap_t pmap) -{ - - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_global++; -#endif - } -} - -void -smp_masked_invlpg(cpuset_t mask, vm_offset_t addr) -{ - - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLPG, NULL, addr, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_page++; -#endif - } -} - -void -smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2) -{ - - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, NULL, - addr1, addr2); -#ifdef COUNT_XINVLTLB_HITS - ipi_range++; - ipi_range_size += (addr2 - addr1) / PAGE_SIZE; -#endif - } -} - -void -smp_cache_flush(void) -{ - - if (smp_started) { - smp_targeted_tlb_shootdown(all_cpus, IPI_INVLCACHE, NULL, - 0, 0); - } -} - -/* - * Handlers for TLB related IPIs - */ -void -invltlb_handler(void) -{ -#ifdef COUNT_XINVLTLB_HITS - xhits_gbl[PCPU_GET(cpuid)]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; -#endif /* COUNT_IPIS */ - - invltlb(); - atomic_add_int(&smp_tlb_wait, 1); -} - void invltlb_invpcid_handler(void) { @@ -556,7 +436,7 @@ invltlb_pcid_handler(void) #endif /* COUNT_IPIS */ if (smp_tlb_pmap == kernel_pmap) { - invltlb_globpcid(); + invltlb_glob(); } else { /* * The current pmap might not be equal to @@ -572,38 +452,3 @@ invltlb_pcid_handler(void) } atomic_add_int(&smp_tlb_wait, 1); } - -void -invlpg_handler(void) -{ -#ifdef COUNT_XINVLTLB_HITS - xhits_pg[PCPU_GET(cpuid)]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; -#endif /* COUNT_IPIS */ - - invlpg(smp_tlb_addr1); - atomic_add_int(&smp_tlb_wait, 1); -} - -void -invlrng_handler(void) -{ - vm_offset_t addr; - -#ifdef COUNT_XINVLTLB_HITS - xhits_rng[PCPU_GET(cpuid)]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlrng_counts[PCPU_GET(cpuid)])++; -#endif /* COUNT_IPIS */ - - addr = smp_tlb_addr1; - do { - invlpg(addr); - addr += PAGE_SIZE; - } while (addr < smp_tlb_addr2); - - atomic_add_int(&smp_tlb_wait, 1); -} Modified: head/sys/amd64/amd64/pmap.c ============================================================================== --- head/sys/amd64/amd64/pmap.c Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/amd64/amd64/pmap.c Thu Dec 3 11:14:14 2015 (r291688) @@ -1321,7 +1321,7 @@ pmap_update_pde_invalidate(pmap_t pmap, * Promotion: flush every 4KB page mapping from the TLB, * including any global (PG_G) mappings. */ - invltlb_globpcid(); + invltlb_glob(); } } #ifdef SMP @@ -1482,7 +1482,7 @@ pmap_invalidate_all(pmap_t pmap) bzero(&d, sizeof(d)); invpcid(&d, INVPCID_CTXGLOB); } else { - invltlb_globpcid(); + invltlb_glob(); } mask = &all_cpus; } else { @@ -1653,7 +1653,7 @@ pmap_invalidate_all(pmap_t pmap) bzero(&d, sizeof(d)); invpcid(&d, INVPCID_CTXGLOB); } else { - invltlb_globpcid(); + invltlb_glob(); } } else if (pmap == PCPU_GET(curpmap)) { if (pmap_pcid_enabled) { Modified: head/sys/amd64/include/cpufunc.h ============================================================================== --- head/sys/amd64/include/cpufunc.h Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/amd64/include/cpufunc.h Thu Dec 3 11:14:14 2015 (r291688) @@ -505,7 +505,7 @@ invltlb(void) * Operations that Invalidate TLBs and Paging-Structure Caches. */ static __inline void -invltlb_globpcid(void) +invltlb_glob(void) { uint64_t cr4; Modified: head/sys/amd64/include/smp.h ============================================================================== --- head/sys/amd64/include/smp.h Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/amd64/include/smp.h Thu Dec 3 11:14:14 2015 (r291688) @@ -25,6 +25,8 @@ #include <x86/apicvar.h> #include <machine/pcb.h> +struct pmap; + /* global symbols in mpboot.S */ extern char mptramp_start[]; extern char mptramp_end[]; @@ -53,6 +55,7 @@ extern u_int ipi_global; extern u_int ipi_page; extern u_int ipi_range; extern u_int ipi_range_size; +extern struct pmap *smp_tlb_pmap; extern volatile int smp_tlb_wait; @@ -86,8 +89,6 @@ inthand_t IDTVEC(justreturn), /* interrupt CPU with minimum overhead */ IDTVEC(rendezvous); /* handle CPU rendezvous */ -struct pmap; - /* functions in mp_machdep.c */ void assign_cpu_ids(void); void cpu_add(u_int apic_id, char boot_cpu); Modified: head/sys/i386/i386/mp_machdep.c ============================================================================== --- head/sys/i386/i386/mp_machdep.c Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/i386/i386/mp_machdep.c Thu Dec 3 11:14:14 2015 (r291688) @@ -132,11 +132,6 @@ __FBSDID("$FreeBSD$"); extern struct pcpu __pcpu[]; -/* Variables needed for SMP tlb shootdown. */ -vm_offset_t smp_tlb_addr1; -vm_offset_t smp_tlb_addr2; -volatile int smp_tlb_wait; - /* * Local data and functions. */ @@ -487,201 +482,3 @@ start_ap(int apic_id) } return 0; /* return FAILURE */ } - -/* - * Flush the TLB on all other CPU's - */ -static void -smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2) -{ - u_int ncpu; - - ncpu = mp_ncpus - 1; /* does not shootdown self */ - if (ncpu < 1) - return; /* no other cpus */ - if (!(read_eflags() & PSL_I)) - panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_ipi_mtx); - smp_tlb_addr1 = addr1; - smp_tlb_addr2 = addr2; - smp_tlb_wait = 0; - ipi_all_but_self(vector); - while (smp_tlb_wait < ncpu) - ia32_pause(); - mtx_unlock_spin(&smp_ipi_mtx); -} - -static void -smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, vm_offset_t addr1, vm_offset_t addr2) -{ - int cpu, ncpu, othercpus; - - othercpus = mp_ncpus - 1; - if (CPU_ISFULLSET(&mask)) { - if (othercpus < 1) - return; - } else { - CPU_CLR(PCPU_GET(cpuid), &mask); - if (CPU_EMPTY(&mask)) - return; - } - if (!(read_eflags() & PSL_I)) - panic("%s: interrupts disabled", __func__); - mtx_lock_spin(&smp_ipi_mtx); - smp_tlb_addr1 = addr1; - smp_tlb_addr2 = addr2; - atomic_store_rel_int(&smp_tlb_wait, 0); - if (CPU_ISFULLSET(&mask)) { - ncpu = othercpus; - ipi_all_but_self(vector); - } else { - ncpu = 0; - while ((cpu = CPU_FFS(&mask)) != 0) { - cpu--; - CPU_CLR(cpu, &mask); - CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, cpu, - vector); - ipi_send_cpu(cpu, vector); - ncpu++; - } - } - while (smp_tlb_wait < ncpu) - ia32_pause(); - mtx_unlock_spin(&smp_ipi_mtx); -} - -void -smp_invltlb(void) -{ - - if (smp_started) { - smp_tlb_shootdown(IPI_INVLTLB, 0, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_global++; -#endif - } -} - -void -smp_invlpg(vm_offset_t addr) -{ - - if (smp_started) { - smp_tlb_shootdown(IPI_INVLPG, addr, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_page++; -#endif - } -} - -void -smp_invlpg_range(vm_offset_t addr1, vm_offset_t addr2) -{ - - if (smp_started) { - smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2); -#ifdef COUNT_XINVLTLB_HITS - ipi_range++; - ipi_range_size += (addr2 - addr1) / PAGE_SIZE; -#endif - } -} - -void -smp_masked_invltlb(cpuset_t mask) -{ - - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_masked_global++; -#endif - } -} - -void -smp_masked_invlpg(cpuset_t mask, vm_offset_t addr) -{ - - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0); -#ifdef COUNT_XINVLTLB_HITS - ipi_masked_page++; -#endif - } -} - -void -smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2) -{ - - if (smp_started) { - smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2); -#ifdef COUNT_XINVLTLB_HITS - ipi_masked_range++; - ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE; -#endif - } -} - -void -smp_cache_flush(void) -{ - - if (smp_started) - smp_tlb_shootdown(IPI_INVLCACHE, 0, 0); -} - -/* - * Handlers for TLB related IPIs - */ -void -invltlb_handler(void) -{ - uint64_t cr3; -#ifdef COUNT_XINVLTLB_HITS - xhits_gbl[PCPU_GET(cpuid)]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; -#endif /* COUNT_IPIS */ - - cr3 = rcr3(); - load_cr3(cr3); - atomic_add_int(&smp_tlb_wait, 1); -} - -void -invlpg_handler(void) -{ -#ifdef COUNT_XINVLTLB_HITS - xhits_pg[PCPU_GET(cpuid)]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; -#endif /* COUNT_IPIS */ - - invlpg(smp_tlb_addr1); - - atomic_add_int(&smp_tlb_wait, 1); -} - -void -invlrng_handler(void) -{ - vm_offset_t addr; -#ifdef COUNT_XINVLTLB_HITS - xhits_rng[PCPU_GET(cpuid)]++; -#endif /* COUNT_XINVLTLB_HITS */ -#ifdef COUNT_IPIS - (*ipi_invlrng_counts[PCPU_GET(cpuid)])++; -#endif /* COUNT_IPIS */ - - addr = smp_tlb_addr1; - do { - invlpg(addr); - addr += PAGE_SIZE; - } while (addr < smp_tlb_addr2); - - atomic_add_int(&smp_tlb_wait, 1); -} Modified: head/sys/i386/i386/pmap.c ============================================================================== --- head/sys/i386/i386/pmap.c Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/i386/i386/pmap.c Thu Dec 3 11:14:14 2015 (r291688) @@ -655,7 +655,7 @@ pmap_set_pg(void) va = KERNBASE + KERNLOAD; while (va < endva) { pdir_pde(PTD, va) |= pgeflag; - invltlb(); /* Play it safe, invltlb() every time */ + invltlb_glob(); /* Play it safe, invltlb() every time */ va += NBPDR; } } else { @@ -664,7 +664,7 @@ pmap_set_pg(void) pte = vtopte(va); if (*pte) *pte |= pgeflag; - invltlb(); /* Play it safe, invltlb() every time */ + invltlb_glob(); /* Play it safe, invltlb() every time */ va += PAGE_SIZE; } } @@ -973,6 +973,22 @@ pmap_update_pde_invalidate(vm_offset_t v load_cr4(cr4 | CR4_PGE); } } + +void +invltlb_glob(void) +{ + uint64_t cr4; + + if (pgeflag == 0) { + invltlb(); + } else { + cr4 = rcr4(); + load_cr4(cr4 & ~CR4_PGE); + load_cr4(cr4 | CR4_PGE); + } +} + + #ifdef SMP /* * For SMP, these functions have to use the IPI mechanism for coherence. @@ -996,13 +1012,13 @@ pmap_update_pde_invalidate(vm_offset_t v void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { - cpuset_t other_cpus; + cpuset_t *mask, other_cpus; u_int cpuid; sched_pin(); if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { invlpg(va); - smp_invlpg(va); + mask = &all_cpus; } else { cpuid = PCPU_GET(cpuid); other_cpus = all_cpus; @@ -1010,16 +1026,16 @@ pmap_invalidate_page(pmap_t pmap, vm_off if (CPU_ISSET(cpuid, &pmap->pm_active)) invlpg(va); CPU_AND(&other_cpus, &pmap->pm_active); - if (!CPU_EMPTY(&other_cpus)) - smp_masked_invlpg(other_cpus, va); + mask = &other_cpus; } + smp_masked_invlpg(*mask, va); sched_unpin(); } void pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - cpuset_t other_cpus; + cpuset_t *mask, other_cpus; vm_offset_t addr; u_int cpuid; @@ -1027,7 +1043,7 @@ pmap_invalidate_range(pmap_t pmap, vm_of if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); - smp_invlpg_range(sva, eva); + mask = &all_cpus; } else { cpuid = PCPU_GET(cpuid); other_cpus = all_cpus; @@ -1036,22 +1052,25 @@ pmap_invalidate_range(pmap_t pmap, vm_of for (addr = sva; addr < eva; addr += PAGE_SIZE) invlpg(addr); CPU_AND(&other_cpus, &pmap->pm_active); - if (!CPU_EMPTY(&other_cpus)) - smp_masked_invlpg_range(other_cpus, sva, eva); + mask = &other_cpus; } + smp_masked_invlpg_range(*mask, sva, eva); sched_unpin(); } void pmap_invalidate_all(pmap_t pmap) { - cpuset_t other_cpus; + cpuset_t *mask, other_cpus; u_int cpuid; sched_pin(); - if (pmap == kernel_pmap || !CPU_CMP(&pmap->pm_active, &all_cpus)) { + if (pmap == kernel_pmap) { + invltlb_glob(); + mask = &all_cpus; + } else if (!CPU_CMP(&pmap->pm_active, &all_cpus)) { invltlb(); - smp_invltlb(); + mask = &all_cpus; } else { cpuid = PCPU_GET(cpuid); other_cpus = all_cpus; @@ -1059,9 +1078,9 @@ pmap_invalidate_all(pmap_t pmap) if (CPU_ISSET(cpuid, &pmap->pm_active)) invltlb(); CPU_AND(&other_cpus, &pmap->pm_active); - if (!CPU_EMPTY(&other_cpus)) - smp_masked_invltlb(other_cpus); + mask = &other_cpus; } + smp_masked_invltlb(*mask, pmap); sched_unpin(); } @@ -1193,7 +1212,9 @@ PMAP_INLINE void pmap_invalidate_all(pmap_t pmap) { - if (pmap == kernel_pmap || !CPU_EMPTY(&pmap->pm_active)) + if (pmap == kernel_pmap) + invltlb_glob(); + else if (!CPU_EMPTY(&pmap->pm_active)) invltlb(); } Modified: head/sys/i386/include/pmap.h ============================================================================== --- head/sys/i386/include/pmap.h Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/i386/include/pmap.h Thu Dec 3 11:14:14 2015 (r291688) @@ -394,6 +394,8 @@ void pmap_invalidate_cache_pages(vm_page void pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force); +void invltlb_glob(void); + #endif /* _KERNEL */ #endif /* !LOCORE */ Modified: head/sys/i386/include/smp.h ============================================================================== --- head/sys/i386/include/smp.h Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/i386/include/smp.h Thu Dec 3 11:14:14 2015 (r291688) @@ -53,10 +53,6 @@ extern u_int ipi_global; extern u_int ipi_page; extern u_int ipi_range; extern u_int ipi_range_size; -extern u_int ipi_masked_global; -extern u_int ipi_masked_page; -extern u_int ipi_masked_range; -extern u_int ipi_masked_range_size; struct cpu_info { int cpu_present:1; @@ -105,13 +101,10 @@ void ipi_selected(cpuset_t cpus, u_int i u_int mp_bootaddress(u_int); void set_interrupt_apic_ids(void); void smp_cache_flush(void); -void smp_invlpg(vm_offset_t addr); void smp_masked_invlpg(cpuset_t mask, vm_offset_t addr); -void smp_invlpg_range(vm_offset_t startva, vm_offset_t endva); void smp_masked_invlpg_range(cpuset_t mask, vm_offset_t startva, vm_offset_t endva); -void smp_invltlb(void); -void smp_masked_invltlb(cpuset_t mask); +void smp_masked_invltlb(cpuset_t mask, struct pmap *pmap); void mem_range_AP_init(void); void topo_probe(void); void ipi_send_cpu(int cpu, u_int ipi); Modified: head/sys/x86/x86/mp_x86.c ============================================================================== --- head/sys/x86/x86/mp_x86.c Thu Dec 3 11:05:35 2015 (r291687) +++ head/sys/x86/x86/mp_x86.c Thu Dec 3 11:14:14 2015 (r291688) @@ -713,19 +713,6 @@ SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_p SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, ""); SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size, 0, ""); - -u_int ipi_masked_global; -u_int ipi_masked_page; -u_int ipi_masked_range; -u_int ipi_masked_range_size; -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW, - &ipi_masked_global, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW, - &ipi_masked_page, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW, - &ipi_masked_range, 0, ""); -SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW, - &ipi_masked_range_size, 0, ""); #endif /* COUNT_XINVLTLB_HITS */ /* @@ -1090,3 +1077,165 @@ mp_ipi_intrcnt(void *dummy) } SYSINIT(mp_ipi_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, mp_ipi_intrcnt, NULL); #endif + +/* + * Flush the TLB on other CPU's + */ + +/* Variables needed for SMP tlb shootdown. */ +static vm_offset_t smp_tlb_addr1, smp_tlb_addr2; +pmap_t smp_tlb_pmap; +volatile int smp_tlb_wait; + +#ifdef __amd64__ +#define read_eflags() read_rflags() +#endif + +static void +smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap, + vm_offset_t addr1, vm_offset_t addr2) +{ + int cpu, ncpu, othercpus; + + othercpus = mp_ncpus - 1; /* does not shootdown self */ + + /* + * Check for other cpus. Return if none. + */ + if (CPU_ISFULLSET(&mask)) { + if (othercpus < 1) + return; + } else { + CPU_CLR(PCPU_GET(cpuid), &mask); + if (CPU_EMPTY(&mask)) + return; + } + + if (!(read_eflags() & PSL_I)) + panic("%s: interrupts disabled", __func__); + mtx_lock_spin(&smp_ipi_mtx); + smp_tlb_addr1 = addr1; + smp_tlb_addr2 = addr2; + smp_tlb_pmap = pmap; + smp_tlb_wait = 0; + if (CPU_ISFULLSET(&mask)) { + ncpu = othercpus; + ipi_all_but_self(vector); + } else { + ncpu = 0; + while ((cpu = CPU_FFS(&mask)) != 0) { + cpu--; + CPU_CLR(cpu, &mask); + CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__, + cpu, vector); + ipi_send_cpu(cpu, vector); + ncpu++; + } + } + while (smp_tlb_wait < ncpu) + ia32_pause(); + mtx_unlock_spin(&smp_ipi_mtx); +} + +void +smp_masked_invltlb(cpuset_t mask, pmap_t pmap) +{ + + if (smp_started) { + smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_global++; +#endif + } +} + +void +smp_masked_invlpg(cpuset_t mask, vm_offset_t addr) +{ + + if (smp_started) { + smp_targeted_tlb_shootdown(mask, IPI_INVLPG, NULL, addr, 0); +#ifdef COUNT_XINVLTLB_HITS + ipi_page++; +#endif + } +} + +void +smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2) +{ + + if (smp_started) { + smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, NULL, + addr1, addr2); +#ifdef COUNT_XINVLTLB_HITS + ipi_range++; + ipi_range_size += (addr2 - addr1) / PAGE_SIZE; +#endif + } +} + +void +smp_cache_flush(void) +{ + + if (smp_started) { + smp_targeted_tlb_shootdown(all_cpus, IPI_INVLCACHE, NULL, + 0, 0); + } +} + +/* + * Handlers for TLB related IPIs + */ +void +invltlb_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_gbl[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invltlb_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + if (smp_tlb_pmap == kernel_pmap) + invltlb_glob(); + else + invltlb(); + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlpg_handler(void) +{ +#ifdef COUNT_XINVLTLB_HITS + xhits_pg[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlpg_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + invlpg(smp_tlb_addr1); + atomic_add_int(&smp_tlb_wait, 1); +} + +void +invlrng_handler(void) +{ + vm_offset_t addr; + +#ifdef COUNT_XINVLTLB_HITS + xhits_rng[PCPU_GET(cpuid)]++; +#endif /* COUNT_XINVLTLB_HITS */ +#ifdef COUNT_IPIS + (*ipi_invlrng_counts[PCPU_GET(cpuid)])++; +#endif /* COUNT_IPIS */ + + addr = smp_tlb_addr1; + do { + invlpg(addr); + addr += PAGE_SIZE; + } while (addr < smp_tlb_addr2); + + atomic_add_int(&smp_tlb_wait, 1); +} _______________________________________________ svn-src-head@freebsd.org mailing list https://lists.freebsd.org/mailman/listinfo/svn-src-head To unsubscribe, send any mail to "svn-src-head-unsubscr...@freebsd.org"
