The branch main has been updated by andrew:
URL:
https://cgit.FreeBSD.org/src/commit/?id=813738fabaaea43503724b8371faf5bab73a3047
commit 813738fabaaea43503724b8371faf5bab73a3047
Author: Andrew Turner <and...@freebsd.org>
AuthorDate: 2022-03-10 14:39:03 +0000
Commit: Andrew Turner <and...@freebsd.org>
CommitDate: 2022-03-15 17:25:28 +0000
Fix arm64 TLB invalidation with non-4k pages
When using 16k or 64k pages atop will shift the address by more than
the needed amount for a tlbi instruction. Replace this with a new macro
to shift the address by 12 and use PAGE_SIZE in the for loop to let the
code work with any page size.
Reviewed by: alc, markj
Sponsored by: The FreeBSD Foundation
Differential Revision: https://reviews.freebsd.org/D34516
---
sys/arm64/arm64/pmap.c | 20 ++++++++++++--------
1 file changed, 12 insertions(+), 8 deletions(-)
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 12b3bed7cb72..0ab27e0a7b53 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -360,6 +360,10 @@ void (*pmap_invalidate_vpipt_icache)(void);
#define COOKIE_TO_ASID(cookie) ((int)(cookie))
#define COOKIE_TO_EPOCH(cookie) ((int)((u_long)(cookie) >> 32))
+#define TLBI_VA_SHIFT 12
+#define TLBI_VA(addr) ((addr) >> TLBI_VA_SHIFT)
+#define TLBI_VA_L3_INCR (L3_SIZE >> TLBI_VA_SHIFT)
+
static int superpages_enabled = 1;
SYSCTL_INT(_vm_pmap, OID_AUTO, superpages_enabled,
CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &superpages_enabled, 0,
@@ -1248,11 +1252,11 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va, bool
final_only)
PMAP_ASSERT_STAGE1(pmap);
dsb(ishst);
+ r = TLBI_VA(va);
if (pmap == kernel_pmap) {
- r = atop(va);
pmap_invalidate_kernel(r, final_only);
} else {
- r = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie)) | atop(va);
+ r |= ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
pmap_invalidate_user(r, final_only);
}
dsb(ish);
@@ -1273,15 +1277,15 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva,
vm_offset_t eva,
dsb(ishst);
if (pmap == kernel_pmap) {
- start = atop(sva);
- end = atop(eva);
- for (r = start; r < end; r++)
+ start = TLBI_VA(sva);
+ end = TLBI_VA(eva);
+ for (r = start; r < end; r += TLBI_VA_L3_INCR)
pmap_invalidate_kernel(r, final_only);
} else {
start = end = ASID_TO_OPERAND(COOKIE_TO_ASID(pmap->pm_cookie));
- start |= atop(sva);
- end |= atop(eva);
- for (r = start; r < end; r++)
+ start |= TLBI_VA(sva);
+ end |= TLBI_VA(eva);
+ for (r = start; r < end; r += TLBI_VA_L3_INCR)
pmap_invalidate_user(r, final_only);
}
dsb(ish);
