The branch main has been updated by alc:
URL:
https://cgit.FreeBSD.org/src/commit/?id=5b8c01d13a0970b11f47503fcd627d249a6e638a
commit 5b8c01d13a0970b11f47503fcd627d249a6e638a
Author: Alan Cox <a...@freebsd.org>
AuthorDate: 2024-07-25 06:57:53 +0000
Commit: Alan Cox <a...@freebsd.org>
CommitDate: 2024-07-26 05:38:46 +0000
amd64 pmap: Optimize PKU lookups when creating superpage mappings
Modify pmap_pkru_same() to update the prototype PTE at the same time as
checking the address range. This eliminates the need for calling
pmap_pkru_get() in addition to pmap_pkru_same(). pmap_pkru_same() was
already doing most of the work of pmap_pkru_get().
Reviewed by: kib
Differential Revision: https://reviews.freebsd.org/D46135
---
sys/amd64/amd64/pmap.c | 29 +++++++++++++++++------------
1 file changed, 17 insertions(+), 12 deletions(-)
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index dcf9b4f5a4f3..778d07689ff0 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -573,7 +573,8 @@ struct pmap_pkru_range {
};
static uma_zone_t pmap_pkru_ranges_zone;
-static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
+static bool pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva,
+ pt_entry_t *pte);
static pt_entry_t pmap_pkru_get(pmap_t pmap, vm_offset_t va);
static void pmap_pkru_on_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva);
static void *pkru_dup_range(void *ctx, void *data);
@@ -7071,11 +7072,9 @@ pmap_enter_largepage(pmap_t pmap, vm_offset_t va,
pt_entry_t newpte, int flags,
PG_V = pmap_valid_bit(pmap);
restart:
- if (!pmap_pkru_same(pmap, va, va + pagesizes[psind]))
- return (KERN_PROTECTION_FAILURE);
pten = newpte;
- if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86)
- pten |= pmap_pkru_get(pmap, va);
+ if (!pmap_pkru_same(pmap, va, va + pagesizes[psind], &pten))
+ return (KERN_PROTECTION_FAILURE);
if (psind == 2) { /* 1G */
pml4e = pmap_pml4e(pmap, va);
@@ -7529,14 +7528,10 @@ pmap_enter_pde(pmap_t pmap, vm_offset_t va, pd_entry_t
newpde, u_int flags,
* and let vm_fault() cope. Check after pde allocation, since
* it could sleep.
*/
- if (!pmap_pkru_same(pmap, va, va + NBPDR)) {
+ if (!pmap_pkru_same(pmap, va, va + NBPDR, &newpde)) {
pmap_abort_ptp(pmap, va, pdpg);
return (KERN_PROTECTION_FAILURE);
}
- if (va < VM_MAXUSER_ADDRESS && pmap->pm_type == PT_X86) {
- newpde &= ~X86_PG_PKU_MASK;
- newpde |= pmap_pkru_get(pmap, va);
- }
/*
* If there are existing mappings, either abort or remove them.
@@ -11460,13 +11455,21 @@ pmap_pkru_deassign_all(pmap_t pmap)
rangeset_remove_all(&pmap->pm_pkru);
}
+/*
+ * Returns true if the PKU setting is the same across the specified address
+ * range, and false otherwise. When returning true, updates the referenced PTE
+ * to reflect the PKU setting.
+ */
static bool
-pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
+pmap_pkru_same(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, pt_entry_t *pte)
{
struct pmap_pkru_range *next_ppr, *ppr;
vm_offset_t va;
+ u_int keyidx;
PMAP_LOCK_ASSERT(pmap, MA_OWNED);
+ KASSERT(pmap->pm_type != PT_X86 || (*pte & X86_PG_PKU_MASK) == 0,
+ ("pte %p has unexpected PKU %ld", pte, *pte & X86_PG_PKU_MASK));
if (pmap->pm_type != PT_X86 ||
(cpu_stdext_feature2 & CPUID_STDEXT2_PKU) == 0 ||
sva >= VM_MAXUSER_ADDRESS)
@@ -11478,14 +11481,16 @@ pmap_pkru_same(pmap_t pmap, vm_offset_t sva,
vm_offset_t eva)
return (ppr == NULL ||
ppr->pkru_rs_el.re_start >= eva);
}
+ keyidx = ppr->pkru_keyidx;
while ((va = ppr->pkru_rs_el.re_end) < eva) {
next_ppr = rangeset_next(&pmap->pm_pkru, va);
if (next_ppr == NULL ||
va != next_ppr->pkru_rs_el.re_start ||
- ppr->pkru_keyidx != next_ppr->pkru_keyidx)
+ keyidx != next_ppr->pkru_keyidx)
return (false);
ppr = next_ppr;
}
+ *pte |= X86_PG_PKU(keyidx);
return (true);
}
