Module Name: src
Committed By: mrg
Date: Mon Mar 8 08:59:06 UTC 2010
Modified Files:
src/sys/arch/sparc64/sparc64: cache.h ipifuncs.c locore.s pmap.c
Log Message:
blast_dcache() SMP friendly:
- blast_dcache() becomes sp_blast_dcache(dcache_size, dcache_line_size)
- new smp_blast_dcache(sparc64_cpuset_t) that blasts the D$ on this cpuset
- sparc64_ipi_blast_dcache() to support the above
- in pmap_remove_all(), when freeing mmu contexts for this pmap, mark
the set of cpus to blast the d$ on as well and convert the
blast_dcache() call into smp_blast_dcache() on the cpus who ran this
pmap, or, sp_blast_dcache(dcache_size, dcache_line_size)
- convert the remaining blast_dcache() in machdep.c to sp_blast_dcache()
- in pmap_destroy()/pmap_remove_all() take the pmap_lock() always since
we assert it is held always.
with these changes, NFS builds on the U60 seem to be stable now, and
the USIII machines also can often complete a single build.sh run now,
diskful or diskless.
reviewed by mlelstv and partially by martin, tested by martin and myself,
with some ideas from chuq as well.
To generate a diff of this commit:
cvs rdiff -u -r1.16 -r1.17 src/sys/arch/sparc64/sparc64/cache.h
cvs rdiff -u -r1.34 -r1.35 src/sys/arch/sparc64/sparc64/ipifuncs.c
cvs rdiff -u -r1.325 -r1.326 src/sys/arch/sparc64/sparc64/locore.s
cvs rdiff -u -r1.257 -r1.258 src/sys/arch/sparc64/sparc64/pmap.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/sparc64/sparc64/cache.h
diff -u src/sys/arch/sparc64/sparc64/cache.h:1.16 src/sys/arch/sparc64/sparc64/cache.h:1.17
--- src/sys/arch/sparc64/sparc64/cache.h:1.16 Sat Mar 6 08:08:29 2010
+++ src/sys/arch/sparc64/sparc64/cache.h Mon Mar 8 08:59:06 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: cache.h,v 1.16 2010/03/06 08:08:29 mrg Exp $ */
+/* $NetBSD: cache.h,v 1.17 2010/03/08 08:59:06 mrg Exp $ */
/*
* Copyright (c) 1996
@@ -72,10 +72,17 @@
* set-associative -- each bank is 8K. No conflict there.)
*/
+/* Various cache size/line sizes */
+extern int ecache_min_line_size;
+extern int dcache_line_size;
+extern int dcache_size;
+extern int icache_line_size;
+extern int icache_size;
+
/* The following are for I$ and D$ flushes and are in locore.s */
void dcache_flush_page_us(paddr_t); /* flush page from D$ */
void dcache_flush_page_usiii(paddr_t); /* flush page from D$ */
-void blast_dcache(void); /* Clear entire D$ */
+void sp_blast_dcache(int, int); /* Clear entire D$ */
void blast_icache_us(void); /* Clear entire I$ */
void blast_icache_usiii(void); /* Clear entire I$ */
@@ -137,17 +144,14 @@
#ifdef MULTIPROCESSOR
void smp_tlb_flush_pte(vaddr_t, struct pmap *);
-#define tlb_flush_pte(va,pm) smp_tlb_flush_pte(va, pm)
void smp_dcache_flush_page_all(paddr_t pa);
+void smp_blast_dcache(sparc64_cpuset_t);
+#define tlb_flush_pte(va,pm ) smp_tlb_flush_pte(va, pm)
#define dcache_flush_page_all(pa) smp_dcache_flush_page_all(pa)
+#define blast_dcache() smp_blast_dcache(cpus_active)
#else
-#define tlb_flush_pte(va,pm) sp_tlb_flush_pte(va, (pm)->pm_ctx[0])
+#define tlb_flush_pte(va,pm) sp_tlb_flush_pte(va, (pm)->pm_ctx[0])
#define dcache_flush_page_all(pa) dcache_flush_page(pa)
+#define blast_dcache() sp_blast_dcache(dcache_size, \
+ dcache_line_size)
#endif
-
-/* Various cache size/line sizes */
-extern int ecache_min_line_size;
-extern int dcache_line_size;
-extern int dcache_size;
-extern int icache_line_size;
-extern int icache_size;
Index: src/sys/arch/sparc64/sparc64/ipifuncs.c
diff -u src/sys/arch/sparc64/sparc64/ipifuncs.c:1.34 src/sys/arch/sparc64/sparc64/ipifuncs.c:1.35
--- src/sys/arch/sparc64/sparc64/ipifuncs.c:1.34 Wed Feb 24 09:49:36 2010
+++ src/sys/arch/sparc64/sparc64/ipifuncs.c Mon Mar 8 08:59:06 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: ipifuncs.c,v 1.34 2010/02/24 09:49:36 mrg Exp $ */
+/* $NetBSD: ipifuncs.c,v 1.35 2010/03/08 08:59:06 mrg Exp $ */
/*-
* Copyright (c) 2004 The NetBSD Foundation, Inc.
@@ -27,7 +27,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.34 2010/02/24 09:49:36 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: ipifuncs.c,v 1.35 2010/03/08 08:59:06 mrg Exp $");
#include "opt_ddb.h"
@@ -73,6 +73,7 @@
void sparc64_ipi_flush_pte_usiii(void *);
void sparc64_ipi_dcache_flush_page_us(void *);
void sparc64_ipi_dcache_flush_page_usiii(void *);
+void sparc64_ipi_blast_dcache(void *);
/*
* Process cpu stop-self event.
@@ -428,6 +429,18 @@
}
/*
+ * Flush the D$ on this set of CPUs.
+ */
+void
+smp_blast_dcache(sparc64_cpuset_t activecpus)
+{
+
+ sparc64_multicast_ipi(activecpus, sparc64_ipi_blast_dcache,
+ dcache_size, dcache_line_size);
+ sp_blast_dcache(dcache_size, dcache_line_size);
+}
+
+/*
* Print an error message.
*/
void
Index: src/sys/arch/sparc64/sparc64/locore.s
diff -u src/sys/arch/sparc64/sparc64/locore.s:1.325 src/sys/arch/sparc64/sparc64/locore.s:1.326
--- src/sys/arch/sparc64/sparc64/locore.s:1.325 Sun Mar 7 01:52:44 2010
+++ src/sys/arch/sparc64/sparc64/locore.s Mon Mar 8 08:59:06 2010
@@ -1,6 +1,7 @@
-/* $NetBSD: locore.s,v 1.325 2010/03/07 01:52:44 mrg Exp $ */
+/* $NetBSD: locore.s,v 1.326 2010/03/08 08:59:06 mrg Exp $ */
/*
+ * Copyright (c) 2006-2010 Matthew R. Green
* Copyright (c) 1996-2002 Eduardo Horvath
* Copyright (c) 1996 Paul Kranenburg
* Copyright (c) 1996
@@ -4918,14 +4919,12 @@
wrpr %o3, %pstate
/*
- * blast_dcache()
+ * sp_blast_dcache(int dcache_size, int dcache_line_size)
*
* Clear out all of D$ regardless of contents
- * Does not modify %o0
- *
*/
.align 8
-ENTRY(blast_dcache)
+ENTRY(sp_blast_dcache)
/*
* We turn off interrupts for the duration to prevent RED exceptions.
*/
@@ -4934,18 +4933,14 @@
#endif
rdpr %pstate, %o3
- sethi %hi(dcache_size), %o1
- ld [%o1 + %lo(dcache_size)], %o1
- sethi %hi(dcache_line_size), %o5
- ld [%o5 + %lo(dcache_line_size)], %o5
- sub %o1, %o5, %o1
+ sub %o0, %o1, %o0
andn %o3, PSTATE_IE, %o4 ! Turn off PSTATE_IE bit
wrpr %o4, 0, %pstate
1:
- stxa %g0, [%o1] ASI_DCACHE_TAG
+ stxa %g0, [%o0] ASI_DCACHE_TAG
membar #Sync
- brnz,pt %o1, 1b
- sub %o1, %o5, %o1
+ brnz,pt %o0, 1b
+ sub %o0, %o1, %o0
sethi %hi(KERNBASE), %o2
flush %o2
@@ -4959,6 +4954,33 @@
wrpr %o3, %pstate
#endif
+#ifdef MULTIPROCESSOR
+/*
+ * void sparc64_ipi_blast_dcache(int dcache_size, int dcache_line_size)
+ *
+ * Clear out all of D$ regardless of contents
+ *
+ * On entry:
+ * %g2 = dcache_size
+ * %g3 = dcache_line_size
+ */
+ .align 8
+ENTRY(sparc64_ipi_blast_dcache)
+ sub %g2, %g3, %g2
+1:
+ stxa %g0, [%g2] ASI_DCACHE_TAG
+ membar #Sync
+ brnz,pt %g2, 1b
+ sub %g2, %g3, %g2
+
+ sethi %hi(KERNBASE), %g5
+ flush %g5
+ membar #Sync
+
+ ba,a ret_from_intr_vector
+ nop
+#endif /* MULTIPROCESSOR */
+
/*
* blast_icache_us()
* blast_icache_usiii()
Index: src/sys/arch/sparc64/sparc64/pmap.c
diff -u src/sys/arch/sparc64/sparc64/pmap.c:1.257 src/sys/arch/sparc64/sparc64/pmap.c:1.258
--- src/sys/arch/sparc64/sparc64/pmap.c:1.257 Sat Mar 6 08:08:30 2010
+++ src/sys/arch/sparc64/sparc64/pmap.c Mon Mar 8 08:59:06 2010
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.257 2010/03/06 08:08:30 mrg Exp $ */
+/* $NetBSD: pmap.c,v 1.258 2010/03/08 08:59:06 mrg Exp $ */
/*
*
* Copyright (C) 1996-1999 Eduardo Horvath.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.257 2010/03/06 08:08:30 mrg Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.258 2010/03/08 08:59:06 mrg Exp $");
#undef NO_VCACHE /* Don't forget the locked TLB in dostart */
#define HWREF
@@ -1365,16 +1365,16 @@
return;
}
DPRINTF(PDB_DESTROY, ("pmap_destroy: freeing pmap %p\n", pm));
-#ifdef MULTIPROCESSOR
mutex_enter(&pmap_lock);
+#ifdef MULTIPROCESSOR
for (ci = cpus; ci != NULL; ci = ci->ci_next) {
if (CPUSET_HAS(cpus_active, ci->ci_index))
ctx_free(pm, ci);
}
- mutex_exit(&pmap_lock);
#else
ctx_free(pm, curcpu());
#endif
+ mutex_exit(&pmap_lock);
/* we could be a little smarter and leave pages zeroed */
for (pg = TAILQ_FIRST(&pm->pm_obj.memq); pg != NULL; pg = nextpg) {
@@ -1863,6 +1863,7 @@
{
#ifdef MULTIPROCESSOR
struct cpu_info *ci;
+ sparc64_cpuset_t pmap_cpus_active;
#endif
if (pm == pmap_kernel()) {
@@ -1870,18 +1871,28 @@
}
write_user_windows();
pm->pm_refs = 0;
-#ifdef MULTIPROCESSOR
+
mutex_enter(&pmap_lock);
+#ifdef MULTIPROCESSOR
+ CPUSET_CLEAR(pmap_cpus_active);
for (ci = cpus; ci != NULL; ci = ci->ci_next) {
- if (CPUSET_HAS(cpus_active, ci->ci_index))
+ if (CPUSET_HAS(cpus_active, ci->ci_index)) {
+ if (pm->pm_ctx[ci->ci_index] > 0)
+ CPUSET_ADD(pmap_cpus_active, ci->ci_index);
ctx_free(pm, ci);
+ }
}
- mutex_exit(&pmap_lock);
#else
ctx_free(pm, curcpu());
#endif
+ mutex_exit(&pmap_lock);
+
REMOVE_STAT(flushes);
- blast_dcache();
+#ifdef MULTIPROCESSOR
+ smp_blast_dcache(pmap_cpus_active);
+#else
+ sp_blast_dcache(dcache_size, dcache_line_size);
+#endif
}
/*
