Module Name: src
Committed By: maxv
Date: Wed May 29 16:54:42 UTC 2019
Modified Files:
src/sys/arch/amd64/amd64: cpufunc.S genassym.cf
src/sys/arch/amd64/include: pmap.h
src/sys/arch/x86/include: cpufunc.h pmap.h specialreg.h
src/sys/arch/x86/x86: cpu.c svs.c x86_machdep.c
Log Message:
Add PCID support in SVS. This avoids TLB flushes during kernel<->user
transitions, which greatly reduces the performance penalty introduced by
SVS.
We use two ASIDs, 0 (kern) and 1 (user), and use invpcid to flush pages
in both ASIDs.
The read-only machdep.svs.pcid={0,1} sysctl is added, and indicates whether
SVS+PCID is in use.
To generate a diff of this commit:
cvs rdiff -u -r1.40 -r1.41 src/sys/arch/amd64/amd64/cpufunc.S
cvs rdiff -u -r1.75 -r1.76 src/sys/arch/amd64/amd64/genassym.cf
cvs rdiff -u -r1.60 -r1.61 src/sys/arch/amd64/include/pmap.h
cvs rdiff -u -r1.30 -r1.31 src/sys/arch/x86/include/cpufunc.h
cvs rdiff -u -r1.100 -r1.101 src/sys/arch/x86/include/pmap.h
cvs rdiff -u -r1.146 -r1.147 src/sys/arch/x86/include/specialreg.h
cvs rdiff -u -r1.170 -r1.171 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.28 -r1.29 src/sys/arch/x86/x86/svs.c
cvs rdiff -u -r1.126 -r1.127 src/sys/arch/x86/x86/x86_machdep.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/amd64/amd64/cpufunc.S
diff -u src/sys/arch/amd64/amd64/cpufunc.S:1.40 src/sys/arch/amd64/amd64/cpufunc.S:1.41
--- src/sys/arch/amd64/amd64/cpufunc.S:1.40 Sun May 19 08:17:02 2019
+++ src/sys/arch/amd64/amd64/cpufunc.S Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.S,v 1.40 2019/05/19 08:17:02 maxv Exp $ */
+/* $NetBSD: cpufunc.S,v 1.41 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2008 The NetBSD Foundation, Inc.
@@ -41,6 +41,7 @@
#include <machine/segments.h>
#include "opt_xen.h"
+#include "opt_svs.h"
#include "assym.h"
@@ -65,6 +66,17 @@ END(x86_mfence)
#ifndef XENPV
ENTRY(invlpg)
+#ifdef SVS
+ movb _C_LABEL(svs_pcid),%al
+ testb %al,%al
+ jz 1f
+ pushq %rdi
+ pushq $PMAP_PCID_USER
+ movq $INVPCID_ADDRESS,%rax
+ invpcid (%rsp),%rax
+ addq $16,%rsp
+1: /* FALLTHROUGH */
+#endif
invlpg (%rdi)
ret
END(invlpg)
@@ -112,7 +124,7 @@ END(ltr)
ENTRY(tlbflushg)
movq %cr4, %rax
testq $CR4_PGE, %rax
- jz 1f
+ jz tlbflush
movq %rax, %rdx
andq $~CR4_PGE, %rdx
movq %rdx, %cr4
@@ -121,8 +133,19 @@ ENTRY(tlbflushg)
END(tlbflushg)
ENTRY(tlbflush)
-1:
- movq %cr3, %rax
+#ifdef SVS
+ movb _C_LABEL(svs_pcid),%al
+ testb %al,%al
+ jz 1f
+ xorq %rax,%rax
+ pushq %rax
+ pushq %rax
+ movq $INVPCID_ALL_NONGLOBAL,%rax
+ invpcid (%rsp),%rax
+ addq $16,%rsp
+ ret
+#endif
+1: movq %cr3, %rax
movq %rax, %cr3
ret
END(tlbflush)
Index: src/sys/arch/amd64/amd64/genassym.cf
diff -u src/sys/arch/amd64/amd64/genassym.cf:1.75 src/sys/arch/amd64/amd64/genassym.cf:1.76
--- src/sys/arch/amd64/amd64/genassym.cf:1.75 Mon May 27 17:32:36 2019
+++ src/sys/arch/amd64/amd64/genassym.cf Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-# $NetBSD: genassym.cf,v 1.75 2019/05/27 17:32:36 maxv Exp $
+# $NetBSD: genassym.cf,v 1.76 2019/05/29 16:54:41 maxv Exp $
#
# Copyright (c) 1998, 2006, 2007, 2008 The NetBSD Foundation, Inc.
@@ -276,6 +276,14 @@ define ACPI_SUSPEND_CR3 offsetof(struct
define ACPI_SUSPEND_CR4 offsetof(struct cpu_info, ci_suspend_cr4)
define ACPI_SUSPEND_CR8 offsetof(struct cpu_info, ci_suspend_cr8)
+define INVPCID_ADDRESS INVPCID_ADDRESS
+define INVPCID_CONTEXT INVPCID_CONTEXT
+define INVPCID_ALL INVPCID_ALL
+define INVPCID_ALL_NONGLOBAL INVPCID_ALL_NONGLOBAL
+
+define PMAP_PCID_USER PMAP_PCID_USER
+define PMAP_PCID_KERN PMAP_PCID_KERN
+
if NIOAPIC > 0
define PIC_IOAPIC offsetof(struct pic, pic_ioapic)
define IOAPIC_SC_REG offsetof(struct ioapic_softc, sc_reg)
Index: src/sys/arch/amd64/include/pmap.h
diff -u src/sys/arch/amd64/include/pmap.h:1.60 src/sys/arch/amd64/include/pmap.h:1.61
--- src/sys/arch/amd64/include/pmap.h:1.60 Sat Mar 9 09:09:56 2019
+++ src/sys/arch/amd64/include/pmap.h Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.60 2019/03/09 09:09:56 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.61 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -179,6 +179,7 @@ void svs_lwp_switch(struct lwp *, struct
void svs_pdir_switch(struct pmap *);
void svs_init(void);
extern bool svs_enabled;
+extern bool svs_pcid;
#include <x86/pmap.h>
Index: src/sys/arch/x86/include/cpufunc.h
diff -u src/sys/arch/x86/include/cpufunc.h:1.30 src/sys/arch/x86/include/cpufunc.h:1.31
--- src/sys/arch/x86/include/cpufunc.h:1.30 Sat May 11 13:40:26 2019
+++ src/sys/arch/x86/include/cpufunc.h Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpufunc.h,v 1.30 2019/05/11 13:40:26 christos Exp $ */
+/* $NetBSD: cpufunc.h,v 1.31 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1998, 2007, 2019 The NetBSD Foundation, Inc.
@@ -65,6 +65,30 @@ void invlpg(vaddr_t);
void wbinvd(void);
void breakpoint(void);
+#define INVPCID_ADDRESS 0
+#define INVPCID_CONTEXT 1
+#define INVPCID_ALL 2
+#define INVPCID_ALL_NONGLOBAL 3
+
+static inline void
+invpcid(register_t op, uint64_t pcid, vaddr_t va)
+{
+ struct {
+ uint64_t pcid;
+ uint64_t addr;
+ } desc = {
+ .pcid = pcid,
+ .addr = va
+ };
+
+ asm volatile (
+ "invpcid %[desc],%[op]"
+ :
+ : [desc] "m" (desc), [op] "r" (op)
+ : "memory"
+ );
+}
+
static inline uint64_t
rdtsc(void)
{
Index: src/sys/arch/x86/include/pmap.h
diff -u src/sys/arch/x86/include/pmap.h:1.100 src/sys/arch/x86/include/pmap.h:1.101
--- src/sys/arch/x86/include/pmap.h:1.100 Sun Mar 10 16:30:01 2019
+++ src/sys/arch/x86/include/pmap.h Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.h,v 1.100 2019/03/10 16:30:01 maxv Exp $ */
+/* $NetBSD: pmap.h,v 1.101 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 1997 Charles D. Cranor and Washington University.
@@ -210,6 +210,9 @@ struct pcpu_area {
extern struct pcpu_area *pcpuarea;
+#define PMAP_PCID_KERN 0
+#define PMAP_PCID_USER 1
+
/*
* pmap data structures: see pmap.c for details of locking.
*/
Index: src/sys/arch/x86/include/specialreg.h
diff -u src/sys/arch/x86/include/specialreg.h:1.146 src/sys/arch/x86/include/specialreg.h:1.147
--- src/sys/arch/x86/include/specialreg.h:1.146 Sat May 18 08:17:39 2019
+++ src/sys/arch/x86/include/specialreg.h Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: specialreg.h,v 1.146 2019/05/18 08:17:39 maxv Exp $ */
+/* $NetBSD: specialreg.h,v 1.147 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2014-2019 The NetBSD Foundation, Inc.
@@ -88,6 +88,13 @@
#define CCR1_RPL 0x01 /* enables RPLSET and RPLVAL# pins */
/*
+ * CR3
+ */
+#define CR3_PCID __BITS(11,0)
+#define CR3_PA __BITS(62,12)
+#define CR3_NO_TLB_FLUSH __BIT(63)
+
+/*
* CR4
*/
#define CR4_VME 0x00000001 /* virtual 8086 mode extension enable */
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.170 src/sys/arch/x86/x86/cpu.c:1.171
--- src/sys/arch/x86/x86/cpu.c:1.170 Mon May 27 18:36:37 2019
+++ src/sys/arch/x86/x86/cpu.c Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $ */
+/* $NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.171 2019/05/29 16:54:41 maxv Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -613,6 +613,12 @@ cpu_init(struct cpu_info *ci)
if (cpu_feature[5] & CPUID_SEF_SMAP)
cr4 |= CR4_SMAP;
+#ifdef SVS
+ /* If PCID is supported, enable it */
+ if (svs_pcid)
+ cr4 |= CR4_PCIDE;
+#endif
+
if (cr4) {
cr4 |= rcr4();
lcr4(cr4);
Index: src/sys/arch/x86/x86/svs.c
diff -u src/sys/arch/x86/x86/svs.c:1.28 src/sys/arch/x86/x86/svs.c:1.29
--- src/sys/arch/x86/x86/svs.c:1.28 Mon May 27 18:36:37 2019
+++ src/sys/arch/x86/x86/svs.c Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $ */
+/* $NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $ */
/*
* Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.29 2019/05/29 16:54:41 maxv Exp $");
#include "opt_svs.h"
@@ -228,6 +228,10 @@ __KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28
*/
bool svs_enabled __read_mostly = false;
+bool svs_pcid __read_mostly = false;
+
+static uint64_t svs_pcid_kcr3 __read_mostly;
+static uint64_t svs_pcid_ucr3 __read_mostly;
struct svs_utls {
paddr_t kpdirpa;
@@ -405,6 +409,19 @@ svs_utls_init(struct cpu_info *ci)
}
static void
+svs_pcid_init(struct cpu_info *ci)
+{
+ if (!svs_pcid) {
+ return;
+ }
+
+ svs_pcid_ucr3 = __SHIFTIN(PMAP_PCID_USER, CR3_PCID) | CR3_NO_TLB_FLUSH;
+ svs_pcid_kcr3 = __SHIFTIN(PMAP_PCID_KERN, CR3_PCID) | CR3_NO_TLB_FLUSH;
+
+ ci->ci_svs_updirpa |= svs_pcid_ucr3;
+}
+
+static void
svs_range_add(struct cpu_info *ci, vaddr_t va, size_t size, bool global)
{
size_t i, n;
@@ -454,6 +471,8 @@ cpu_svs_init(struct cpu_info *ci)
svs_rsp0_init(ci);
svs_utls_init(ci);
+
+ svs_pcid_init(ci);
}
void
@@ -523,11 +542,14 @@ svs_lwp_switch(struct lwp *oldlwp, struc
utls->scratch = 0;
/*
- * Enter the user rsp0. We don't need to flush the TLB here, since
- * the user page tables are not loaded.
+ * Enter the user rsp0. If we're using PCID we must flush the user VA,
+ * if we aren't it will be flushed during the next CR3 reload.
*/
pte = ci->ci_svs_rsp0_pte;
*pte = L1_BASE[pl1_i(va)];
+ if (svs_pcid) {
+ invpcid(INVPCID_ADDRESS, PMAP_PCID_USER, ci->ci_svs_rsp0);
+ }
}
static inline pt_entry_t
@@ -558,7 +580,7 @@ svs_pdir_switch(struct pmap *pmap)
/* Update the info in the UTLS page */
utls = (struct svs_utls *)ci->ci_svs_utls;
- utls->kpdirpa = pmap_pdirpa(pmap, 0);
+ utls->kpdirpa = pmap_pdirpa(pmap, 0) | svs_pcid_kcr3;
mutex_enter(&ci->ci_svs_mtx);
@@ -569,6 +591,10 @@ svs_pdir_switch(struct pmap *pmap)
}
mutex_exit(&ci->ci_svs_mtx);
+
+ if (svs_pcid) {
+ invpcid(INVPCID_CONTEXT, PMAP_PCID_USER, 0);
+ }
}
static void
@@ -641,5 +667,11 @@ svs_init(void)
}
}
+ if ((cpu_info_primary.ci_feat_val[1] & CPUID2_PCID) &&
+ (cpu_info_primary.ci_feat_val[5] & CPUID_SEF_INVPCID)) {
+ svs_pcid = true;
+ lcr4(rcr4() | CR4_PCIDE);
+ }
+
svs_enable();
}
Index: src/sys/arch/x86/x86/x86_machdep.c
diff -u src/sys/arch/x86/x86/x86_machdep.c:1.126 src/sys/arch/x86/x86/x86_machdep.c:1.127
--- src/sys/arch/x86/x86/x86_machdep.c:1.126 Sun May 19 08:17:02 2019
+++ src/sys/arch/x86/x86/x86_machdep.c Wed May 29 16:54:41 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: x86_machdep.c,v 1.126 2019/05/19 08:17:02 maxv Exp $ */
+/* $NetBSD: x86_machdep.c,v 1.127 2019/05/29 16:54:41 maxv Exp $ */
/*-
* Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi,
@@ -31,7 +31,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.126 2019/05/19 08:17:02 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: x86_machdep.c,v 1.127 2019/05/29 16:54:41 maxv Exp $");
#include "opt_modular.h"
#include "opt_physmem.h"
@@ -1293,12 +1293,18 @@ SYSCTL_SETUP(sysctl_machdep_setup, "sysc
CTLTYPE_NODE, "svs", NULL,
NULL, 0, NULL, 0,
CTL_MACHDEP, CTL_CREATE);
- sysctl_createv(clog, 0, &svs_rnode, &svs_rnode,
+ sysctl_createv(clog, 0, &svs_rnode, NULL,
CTLFLAG_PERMANENT,
CTLTYPE_BOOL, "enabled",
SYSCTL_DESCR("Whether the kernel uses SVS"),
NULL, 0, &svs_enabled, 0,
CTL_CREATE, CTL_EOL);
+ sysctl_createv(clog, 0, &svs_rnode, NULL,
+ CTLFLAG_PERMANENT,
+ CTLTYPE_BOOL, "pcid",
+ SYSCTL_DESCR("Whether SVS uses PCID"),
+ NULL, 0, &svs_pcid, 0,
+ CTL_CREATE, CTL_EOL);
#endif
#ifndef XEN
