The branch main has been updated by andrew:
URL:
https://cgit.FreeBSD.org/src/commit/?id=3041b636463d521e3c2bbea7673da6afceec34e5
commit 3041b636463d521e3c2bbea7673da6afceec34e5
Author: Andrew Turner <and...@freebsd.org>
AuthorDate: 2025-01-24 11:36:18 +0000
Commit: Andrew Turner <and...@freebsd.org>
CommitDate: 2025-01-24 12:09:27 +0000
arm64: Support mapping a 52-bit physical adddress
When FEAT_LPA2 is enabled the physical address space increases from
48-bits to 52-bits. The top two address bits are moved to the now
unused shareability field.
Update the kernel to support this new larger address space.
Reviewed by: alc, kib
Sponsored by: Arm Ltd
Differential Revision: https://reviews.freebsd.org/D46624
---
sys/arm64/arm64/pmap.c | 2 +-
sys/arm64/include/armreg.h | 2 +-
sys/arm64/include/hypervisor.h | 1 +
sys/arm64/include/pmap.h | 2 ++
sys/arm64/include/pte.h | 52 ++++++++++++++++++++++++++++++++++++------
sys/arm64/vmm/vmm_arm64.c | 10 +++++---
6 files changed, 57 insertions(+), 12 deletions(-)
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 23eedff2e7cd..5c1e5bb63e4d 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -355,7 +355,7 @@ static u_int physmap_idx;
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"VM/pmap parameters");
-static bool pmap_lpa_enabled __read_mostly = false;
+bool pmap_lpa_enabled __read_mostly = false;
pt_entry_t pmap_sh_attr __read_mostly = ATTR_SH(ATTR_SH_IS);
#if PAGE_SIZE == PAGE_SIZE_4K
diff --git a/sys/arm64/include/armreg.h b/sys/arm64/include/armreg.h
index d586d3568bd7..2a2c8b23e0a4 100644
--- a/sys/arm64/include/armreg.h
+++ b/sys/arm64/include/armreg.h
@@ -2066,7 +2066,7 @@
#define PAR_NS_SHIFT 9
#define PAR_NS_MASK (0x3 << PAR_NS_SHIFT)
#define PAR_PA_SHIFT 12
-#define PAR_PA_MASK 0x0000fffffffff000
+#define PAR_PA_MASK 0x000ffffffffff000
#define PAR_ATTR_SHIFT 56
#define PAR_ATTR_MASK (0xff << PAR_ATTR_SHIFT)
/* When PAR_F == 1 (aborted) */
diff --git a/sys/arm64/include/hypervisor.h b/sys/arm64/include/hypervisor.h
index 15fc36014626..a32e1000d911 100644
--- a/sys/arm64/include/hypervisor.h
+++ b/sys/arm64/include/hypervisor.h
@@ -281,6 +281,7 @@
#define VTCR_EL2_PS_42BIT (0x3UL << VTCR_EL2_PS_SHIFT)
#define VTCR_EL2_PS_44BIT (0x4UL << VTCR_EL2_PS_SHIFT)
#define VTCR_EL2_PS_48BIT (0x5UL << VTCR_EL2_PS_SHIFT)
+#define VTCR_EL2_PS_52BIT (0x6UL << VTCR_EL2_PS_SHIFT)
#define VTCR_EL2_DS_SHIFT 32
#define VTCR_EL2_DS (0x1UL << VTCR_EL2_DS_SHIFT)
diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h
index d92069ee42fd..75de9e342c72 100644
--- a/sys/arm64/include/pmap.h
+++ b/sys/arm64/include/pmap.h
@@ -101,6 +101,8 @@ extern struct pmap kernel_pmap_store;
#define kernel_pmap (&kernel_pmap_store)
#define pmap_kernel() kernel_pmap
+extern bool pmap_lpa_enabled;
+
#define PMAP_ASSERT_LOCKED(pmap) \
mtx_assert(&(pmap)->pm_mtx, MA_OWNED)
#define PMAP_LOCK(pmap) mtx_lock(&(pmap)->pm_mtx)
diff --git a/sys/arm64/include/pte.h b/sys/arm64/include/pte.h
index 02eba21448ba..ae6a8694f6c4 100644
--- a/sys/arm64/include/pte.h
+++ b/sys/arm64/include/pte.h
@@ -54,13 +54,6 @@ typedef uint64_t pt_entry_t; /* page
table entry */
#define ATTR_MASK_L UINT64_C(0x0000000000000fff)
#define ATTR_MASK (ATTR_MASK_H | ATTR_MASK_L)
-#define BASE_MASK ~ATTR_MASK
-#define BASE_ADDR(x) ((x) & BASE_MASK)
-
-#define PTE_TO_PHYS(pte) BASE_ADDR(pte)
-/* Convert a phys addr to the output address field of a PTE */
-#define PHYS_TO_PTE(pa) (pa)
-
/* Bits 58:55 are reserved for software */
#define ATTR_SW_UNUSED1 (1UL << 58)
#define ATTR_SW_NO_PROMOTE (1UL << 57)
@@ -81,13 +74,35 @@ typedef uint64_t pt_entry_t; /* page
table entry */
#define ATTR_CONTIGUOUS (1UL << 52)
#define ATTR_DBM (1UL << 51)
#define ATTR_S1_GP (1UL << 50)
+
+/*
+ * Largest possible output address field for a level 3 page. Block
+ * entries will use fewer low address bits, but these are res0 so
+ * should be safe to include.
+ *
+ * This is also safe to use for the next-level table address for
+ * table entries as they encode a physical address in the same way.
+ */
+#if PAGE_SIZE == PAGE_SIZE_4K
+#define ATTR_ADDR UINT64_C(0x0003fffffffff000)
+#elif PAGE_SIZE == PAGE_SIZE_16K
+#define ATTR_ADDR UINT64_C(0x0003ffffffffc000)
+#else
+#error Unsupported page size
+#endif
+
#define ATTR_S1_nG (1 << 11)
#define ATTR_AF (1 << 10)
+/* When TCR_EL1.DS == 0 */
#define ATTR_SH(x) ((x) << 8)
#define ATTR_SH_MASK ATTR_SH(3)
#define ATTR_SH_NS 0 /* Non-shareable */
#define ATTR_SH_OS 2 /* Outer-shareable */
#define ATTR_SH_IS 3 /* Inner-shareable */
+/* When TCR_EL1.DS == 1 */
+#define ATTR_OA_51_50_SHIFT 8
+#define ATTR_OA_51_50_MASK (3 << ATTR_OA_51_50_SHIFT)
+#define ATTR_OA_51_50_DELTA (50 - 8) /* Delta from address
to pte */
#define ATTR_S1_AP_RW_BIT (1 << 7)
#define ATTR_S1_AP(x) ((x) << 6)
@@ -124,6 +139,29 @@ typedef uint64_t pt_entry_t; /* page
table entry */
*/
#define ATTR_PROMOTE (ATTR_MASK & ~(ATTR_CONTIGUOUS | ATTR_AF))
+/* Read the output address or next-level table address from a PTE */
+#define PTE_TO_PHYS(x) ({ \
+ pt_entry_t _pte = (x); \
+ vm_paddr_t _pa; \
+ _pa = _pte & ATTR_ADDR; \
+ if (pmap_lpa_enabled) \
+ _pa |= (_pte & ATTR_OA_51_50_MASK) << ATTR_OA_51_50_DELTA; \
+ _pa; \
+})
+
+/*
+ * Convert a physical address to an output address or next-level
+ * table address in a PTE
+ */
+#define PHYS_TO_PTE(x) ({ \
+ vm_paddr_t _pa = (x); \
+ pt_entry_t _pte; \
+ _pte = _pa & ATTR_ADDR; \
+ if (pmap_lpa_enabled) \
+ _pte |= (_pa >> ATTR_OA_51_50_DELTA) & ATTR_OA_51_50_MASK; \
+ _pte; \
+})
+
#if PAGE_SIZE == PAGE_SIZE_4K
#define L0_SHIFT 39
#define L1_SHIFT 30
diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c
index 80d985241c69..43b2ba7802d7 100644
--- a/sys/arm64/vmm/vmm_arm64.c
+++ b/sys/arm64/vmm/vmm_arm64.c
@@ -381,8 +381,6 @@ vmmops_modinit(int ipinum)
* shareable
*/
el2_regs.vtcr_el2 = VTCR_EL2_RES1;
- el2_regs.vtcr_el2 |=
- min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_48BIT);
el2_regs.vtcr_el2 |= VTCR_EL2_IRGN0_WBWA | VTCR_EL2_ORGN0_WBWA;
el2_regs.vtcr_el2 |= VTCR_EL2_T0SZ(64 - vmm_virt_bits);
el2_regs.vtcr_el2 |= vmm_vtcr_el2_sl(vmm_pmap_levels);
@@ -402,8 +400,14 @@ vmmops_modinit(int ipinum)
* the shareability field changes to become address bits when this
* is set.
*/
- if ((READ_SPECIALREG(tcr_el1) & TCR_DS) != 0)
+ if ((READ_SPECIALREG(tcr_el1) & TCR_DS) != 0) {
el2_regs.vtcr_el2 |= VTCR_EL2_DS;
+ el2_regs.vtcr_el2 |=
+ min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_52BIT);
+ } else {
+ el2_regs.vtcr_el2 |=
+ min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_48BIT);
+ }
smp_rendezvous(NULL, arm_setup_vectors, NULL, &el2_regs);
