Module Name: src
Committed By: maxv
Date: Mon May 27 18:36:37 UTC 2019
Modified Files:
src/sys/arch/x86/x86: cpu.c pmap.c svs.c
Log Message:
Change the effect of SVS on the TLB. Keep CR4_PGE set when SVS is enabled,
but don't use PTE_G on the kernel PTEs in general.
Add PTE_G on only a few pages, that are already leaked to userland and do
not contain secrets.
This slightly improves syscall performance.
To generate a diff of this commit:
cvs rdiff -u -r1.169 -r1.170 src/sys/arch/x86/x86/cpu.c
cvs rdiff -u -r1.332 -r1.333 src/sys/arch/x86/x86/pmap.c
cvs rdiff -u -r1.27 -r1.28 src/sys/arch/x86/x86/svs.c
Please note that diffs are not public domain; they are subject to the
copyright notices on the relevant files.
Modified files:
Index: src/sys/arch/x86/x86/cpu.c
diff -u src/sys/arch/x86/x86/cpu.c:1.169 src/sys/arch/x86/x86/cpu.c:1.170
--- src/sys/arch/x86/x86/cpu.c:1.169 Mon May 27 17:32:36 2019
+++ src/sys/arch/x86/x86/cpu.c Mon May 27 18:36:37 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: cpu.c,v 1.169 2019/05/27 17:32:36 maxv Exp $ */
+/* $NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $ */
/*
* Copyright (c) 2000-2012 NetBSD Foundation, Inc.
@@ -62,7 +62,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.169 2019/05/27 17:32:36 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.170 2019/05/27 18:36:37 maxv Exp $");
#include "opt_ddb.h"
#include "opt_mpbios.h" /* for MPDEBUG */
@@ -586,9 +586,6 @@ cpu_init(struct cpu_info *ci)
/* If global TLB caching is supported, enable it */
if (cpu_feature[0] & CPUID_PGE)
-#ifdef SVS
- if (!svs_enabled)
-#endif
cr4 |= CR4_PGE;
/*
Index: src/sys/arch/x86/x86/pmap.c
diff -u src/sys/arch/x86/x86/pmap.c:1.332 src/sys/arch/x86/x86/pmap.c:1.333
--- src/sys/arch/x86/x86/pmap.c:1.332 Mon May 27 17:32:36 2019
+++ src/sys/arch/x86/x86/pmap.c Mon May 27 18:36:37 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: pmap.c,v 1.332 2019/05/27 17:32:36 maxv Exp $ */
+/* $NetBSD: pmap.c,v 1.333 2019/05/27 18:36:37 maxv Exp $ */
/*
* Copyright (c) 2008, 2010, 2016, 2017 The NetBSD Foundation, Inc.
@@ -130,7 +130,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.332 2019/05/27 17:32:36 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: pmap.c,v 1.333 2019/05/27 18:36:37 maxv Exp $");
#include "opt_user_ldt.h"
#include "opt_lockdebug.h"
@@ -1117,12 +1117,17 @@ pmap_bootstrap(vaddr_t kva_start)
#if !defined(XENPV)
/*
* Begin to enable global TLB entries if they are supported: add PTE_G
- * attribute to already mapped kernel pages.
+ * attribute to already mapped kernel pages. Do that only if SVS is
+ * disabled.
*
* The G bit has no effect until the CR4_PGE bit is set in CR4, which
* happens later in cpu_init().
*/
+#ifdef SVS
+ if (!svs_enabled && (cpu_feature[0] & CPUID_PGE)) {
+#else
if (cpu_feature[0] & CPUID_PGE) {
+#endif
pmap_pg_g = PTE_G;
pmap_remap_global();
}
Index: src/sys/arch/x86/x86/svs.c
diff -u src/sys/arch/x86/x86/svs.c:1.27 src/sys/arch/x86/x86/svs.c:1.28
--- src/sys/arch/x86/x86/svs.c:1.27 Mon May 27 17:32:36 2019
+++ src/sys/arch/x86/x86/svs.c Mon May 27 18:36:37 2019
@@ -1,4 +1,4 @@
-/* $NetBSD: svs.c,v 1.27 2019/05/27 17:32:36 maxv Exp $ */
+/* $NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $ */
/*
* Copyright (c) 2018-2019 The NetBSD Foundation, Inc.
@@ -30,7 +30,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.27 2019/05/27 17:32:36 maxv Exp $");
+__KERNEL_RCSID(0, "$NetBSD: svs.c,v 1.28 2019/05/27 18:36:37 maxv Exp $");
#include "opt_svs.h"
@@ -268,7 +268,7 @@ svs_tree_add(struct cpu_info *ci, vaddr_
}
static void
-svs_page_add(struct cpu_info *ci, vaddr_t va)
+svs_page_add(struct cpu_info *ci, vaddr_t va, bool global)
{
pd_entry_t *srcpde, *dstpde, pde;
size_t idx, pidx;
@@ -289,9 +289,10 @@ svs_page_add(struct cpu_info *ci, vaddr_
panic("%s: L2 page not mapped", __func__);
}
if (srcpde[idx] & PTE_PS) {
+ KASSERT(!global);
pa = srcpde[idx] & PTE_2MFRAME;
pa += (paddr_t)(va % NBPD_L2);
- pde = (srcpde[idx] & ~(PTE_G|PTE_PS|PTE_2MFRAME)) | pa;
+ pde = (srcpde[idx] & ~(PTE_PS|PTE_2MFRAME)) | pa;
if (pmap_valid_entry(dstpde[pidx])) {
panic("%s: L1 page already mapped", __func__);
@@ -311,7 +312,17 @@ svs_page_add(struct cpu_info *ci, vaddr_
if (pmap_valid_entry(dstpde[pidx])) {
panic("%s: L1 page already mapped", __func__);
}
- dstpde[pidx] = srcpde[idx] & ~(PTE_G);
+ dstpde[pidx] = srcpde[idx];
+
+ /*
+ * If we want a global translation, mark both the src and dst with
+ * PTE_G.
+ */
+ if (global) {
+ srcpde[idx] |= PTE_G;
+ dstpde[pidx] |= PTE_G;
+ tlbflushg();
+ }
}
static void
@@ -394,14 +405,14 @@ svs_utls_init(struct cpu_info *ci)
}
static void
-svs_range_add(struct cpu_info *ci, vaddr_t va, size_t size)
+svs_range_add(struct cpu_info *ci, vaddr_t va, size_t size, bool global)
{
size_t i, n;
KASSERT(size % PAGE_SIZE == 0);
n = size / PAGE_SIZE;
for (i = 0; i < n; i++) {
- svs_page_add(ci, va + i * PAGE_SIZE);
+ svs_page_add(ci, va + i * PAGE_SIZE, global);
}
}
@@ -434,12 +445,12 @@ cpu_svs_init(struct cpu_info *ci)
mutex_init(&ci->ci_svs_mtx, MUTEX_DEFAULT, IPL_VM);
- svs_page_add(ci, (vaddr_t)&pcpuarea->idt);
- svs_page_add(ci, (vaddr_t)&pcpuarea->ldt);
+ svs_page_add(ci, (vaddr_t)&pcpuarea->idt, true);
+ svs_page_add(ci, (vaddr_t)&pcpuarea->ldt, true);
svs_range_add(ci, (vaddr_t)&pcpuarea->ent[cid],
- offsetof(struct pcpu_entry, rsp0));
+ offsetof(struct pcpu_entry, rsp0), true);
svs_range_add(ci, (vaddr_t)&__text_user_start,
- (vaddr_t)&__text_user_end - (vaddr_t)&__text_user_start);
+ (vaddr_t)&__text_user_end - (vaddr_t)&__text_user_start, false);
svs_rsp0_init(ci);
svs_utls_init(ci);
