The branch main has been updated by andrew:
URL:
https://cgit.FreeBSD.org/src/commit/?id=d52c31904218c79424435b69b9ec4098885d800f
commit d52c31904218c79424435b69b9ec4098885d800f
Author: Andrew Turner <and...@freebsd.org>
AuthorDate: 2024-09-05 12:12:04 +0000
Commit: Andrew Turner <and...@freebsd.org>
CommitDate: 2024-09-05 12:12:17 +0000
arm64: Make shareability attributes dynamic
When LPA2 is enabled the shareability attribute in the page table are
replaces with output address bits. To support a larger physical address
space make this attribute dynamic so we only set it when appropriate.
Reviewed by: alc, kib
Sponsored by: Arm Ltd
Differential Revision: https://reviews.freebsd.org/D46394
---
sys/arm64/arm64/efirt_machdep.c | 2 +-
sys/arm64/arm64/locore.S | 44 ++++++++++++++++++++++++++++++++++++--
sys/arm64/arm64/minidump_machdep.c | 4 ++--
sys/arm64/arm64/pmap.c | 24 +++++++++++----------
sys/arm64/include/hypervisor.h | 2 ++
sys/arm64/include/pmap.h | 2 ++
sys/arm64/vmm/vmm_arm64.c | 8 +++++++
7 files changed, 70 insertions(+), 16 deletions(-)
diff --git a/sys/arm64/arm64/efirt_machdep.c b/sys/arm64/arm64/efirt_machdep.c
index 7c1d12b0b9b4..47e0a209d8b1 100644
--- a/sys/arm64/arm64/efirt_machdep.c
+++ b/sys/arm64/arm64/efirt_machdep.c
@@ -214,7 +214,7 @@ efi_create_1t1_map(struct efi_md *map, int ndesc, int
descsz)
p->md_phys, mode, p->md_pages);
}
- l3_attr = ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_IDX(mode) |
+ l3_attr = ATTR_AF | pmap_sh_attr | ATTR_S1_IDX(mode) |
ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_nG | L3_PAGE;
if (mode == VM_MEMATTR_DEVICE || p->md_attr & EFI_MD_ATTR_XP)
l3_attr |= ATTR_S1_XN;
diff --git a/sys/arm64/arm64/locore.S b/sys/arm64/arm64/locore.S
index e37ea5aa3b48..7c60a2c5bf0a 100644
--- a/sys/arm64/arm64/locore.S
+++ b/sys/arm64/arm64/locore.S
@@ -86,6 +86,7 @@ ENTRY(_start)
* x27 = TTBR0 table
* x26 = Kernel L1 table
* x24 = TTBR1 table
+ * x22 = PTE shareability attributes
*/
/* Enable the mmu */
@@ -135,6 +136,10 @@ virtdone:
str x27, [x0, #BP_KERN_TTBR0]
str x23, [x0, #BP_BOOT_EL]
+ /* Set this before it's used in kasan_init_early */
+ adrp x1, pmap_sh_attr
+ str x22, [x1, :lo12:pmap_sh_attr]
+
#ifdef KASAN
/* Save bootparams */
mov x19, x0
@@ -476,6 +481,30 @@ LENTRY(create_pagetables)
cmp x6, x27
b.lo 1b
+ /*
+ * Find the shareability attribute we should use. If FEAT_LPA2 is
+ * enabled then the shareability field is moved from the page table
+ * to tcr_el1 and the bits in the page table are reused by the
+ * address field.
+ */
+#if PAGE_SIZE == PAGE_SIZE_4K
+#define LPA2_MASK ID_AA64MMFR0_TGran4_MASK
+#define LPA2_VAL ID_AA64MMFR0_TGran4_LPA2
+#elif PAGE_SIZE == PAGE_SIZE_16K
+#define LPA2_MASK ID_AA64MMFR0_TGran16_MASK
+#define LPA2_VAL ID_AA64MMFR0_TGran16_LPA2
+#else
+#error Unsupported page size
+#endif
+ mrs x6, id_aa64mmfr0_el1
+ mov x7, LPA2_VAL
+ and x6, x6, LPA2_MASK
+ cmp x6, x7
+ ldr x22, =(ATTR_SH(ATTR_SH_IS))
+ csel x22, xzr, x22, eq
+#undef LPA2_MASK
+#undef LPA2_VAL
+
/*
* Build the TTBR1 maps.
*/
@@ -747,11 +776,13 @@ LENTRY(build_l2_block_pagetable)
/* Build the L2 block entry */
orr x12, x7, #L2_BLOCK
- orr x12, x12, #(ATTR_AF | ATTR_SH(ATTR_SH_IS))
+ orr x12, x12, #(ATTR_AF)
orr x12, x12, #(ATTR_S1_UXN)
#ifdef __ARM_FEATURE_BTI_DEFAULT
orr x12, x12, #(ATTR_S1_GP)
#endif
+ /* Set the shareability attribute */
+ orr x12, x12, x22
/* Only use the output address bits */
lsr x9, x9, #L2_SHIFT
@@ -823,11 +854,13 @@ LENTRY(build_l3_page_pagetable)
/* Build the L3 page entry */
orr x12, x7, #L3_PAGE
- orr x12, x12, #(ATTR_AF | ATTR_SH(ATTR_SH_IS))
+ orr x12, x12, #(ATTR_AF)
orr x12, x12, #(ATTR_S1_UXN)
#ifdef __ARM_FEATURE_BTI_DEFAULT
orr x12, x12, #(ATTR_S1_GP)
#endif
+ /* Set the shareability attribute */
+ orr x12, x12, x22
/* Only use the output address bits */
lsr x9, x9, #L3_SHIFT
@@ -886,6 +919,13 @@ LENTRY(start_mmu)
* to 1 only if the ASIDBits field equals 0b0010.
*/
ldr x2, tcr
+
+ /* If x22 contains a non-zero value then LPA2 is not implemented */
+ cbnz x22, .Lno_lpa2
+ ldr x3, =(TCR_DS)
+ orr x2, x2, x3
+.Lno_lpa2:
+
mrs x3, id_aa64mmfr0_el1
/* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */
diff --git a/sys/arm64/arm64/minidump_machdep.c
b/sys/arm64/arm64/minidump_machdep.c
index ac0a2c2dc96e..749c96545506 100644
--- a/sys/arm64/arm64/minidump_machdep.c
+++ b/sys/arm64/arm64/minidump_machdep.c
@@ -311,7 +311,7 @@ cpu_minidumpsys(struct dumperinfo *di, const struct
minidumpstate *state)
for (j = 0; j < Ln_ENTRIES; j++) {
tmpbuffer[j] = (pa + i * L2_SIZE +
j * PAGE_SIZE) | ATTR_AF |
- ATTR_SH(ATTR_SH_IS) | L3_PAGE;
+ pmap_sh_attr | L3_PAGE;
}
error = blk_write(di, (char *)&tmpbuffer, 0,
PAGE_SIZE);
@@ -330,7 +330,7 @@ cpu_minidumpsys(struct dumperinfo *di, const struct
minidumpstate *state)
/* Generate fake l3 entries based upon the l1 entry */
for (i = 0; i < Ln_ENTRIES; i++) {
tmpbuffer[i] = (pa + i * PAGE_SIZE) |
- ATTR_AF | ATTR_SH(ATTR_SH_IS) | L3_PAGE;
+ ATTR_AF | pmap_sh_attr | L3_PAGE;
}
error = blk_write(di, (char *)&tmpbuffer, 0, PAGE_SIZE);
if (error)
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 5f09f4cbcf1b..dc02e732568f 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -185,7 +185,7 @@
#else
#define ATTR_KERN_GP 0
#endif
-#define PMAP_SAN_PTE_BITS (ATTR_AF | ATTR_SH(ATTR_SH_IS) |
ATTR_S1_XN | \
+#define PMAP_SAN_PTE_BITS (ATTR_AF | ATTR_S1_XN | pmap_sh_attr | \
ATTR_KERN_GP | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
ATTR_S1_AP(ATTR_S1_AP_RW))
struct pmap_large_md_page {
@@ -355,6 +355,8 @@ static u_int physmap_idx;
static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
"VM/pmap parameters");
+pt_entry_t pmap_sh_attr __read_mostly = ATTR_SH(ATTR_SH_IS);
+
#if PAGE_SIZE == PAGE_SIZE_4K
#define L1_BLOCKS_SUPPORTED 1
#else
@@ -1150,7 +1152,7 @@ pmap_bootstrap_l2_block(struct pmap_bootstrap_state
*state, int i)
MPASS((state->pa & L2_OFFSET) == 0);
MPASS(state->l2[l2_slot] == 0);
pmap_store(&state->l2[l2_slot], PHYS_TO_PTE(state->pa) |
- ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_XN | ATTR_KERN_GP |
+ ATTR_AF | pmap_sh_attr | ATTR_S1_XN | ATTR_KERN_GP |
ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | contig | L2_BLOCK);
}
MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
@@ -1200,7 +1202,7 @@ pmap_bootstrap_l3_page(struct pmap_bootstrap_state
*state, int i)
MPASS((state->pa & L3_OFFSET) == 0);
MPASS(state->l3[l3_slot] == 0);
pmap_store(&state->l3[l3_slot], PHYS_TO_PTE(state->pa) |
- ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_XN | ATTR_KERN_GP |
+ ATTR_AF | pmap_sh_attr | ATTR_S1_XN | ATTR_KERN_GP |
ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | contig | L3_PAGE);
}
MPASS(state->va == (state->pa - dmap_phys_base + DMAP_MIN_ADDRESS));
@@ -1243,7 +1245,7 @@ pmap_bootstrap_dmap(void)
pmap_store(
&bs_state.l1[pmap_l1_index(bs_state.va)],
PHYS_TO_PTE(bs_state.pa) | ATTR_AF |
- ATTR_SH(ATTR_SH_IS) |
+ pmap_sh_attr |
ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) |
ATTR_S1_XN | ATTR_KERN_GP | L1_BLOCK);
}
@@ -2112,7 +2114,7 @@ pmap_kenter(vm_offset_t sva, vm_size_t size, vm_paddr_t
pa, int mode)
KASSERT((size & PAGE_MASK) == 0,
("pmap_kenter: Mapping is not page-sized"));
- attr = ATTR_AF | ATTR_SH(ATTR_SH_IS) | ATTR_S1_AP(ATTR_S1_AP_RW) |
+ attr = ATTR_AF | pmap_sh_attr | ATTR_S1_AP(ATTR_S1_AP_RW) |
ATTR_S1_XN | ATTR_KERN_GP | ATTR_S1_IDX(mode);
old_l3e = 0;
va = sva;
@@ -2327,7 +2329,7 @@ pmap_qenter(vm_offset_t sva, vm_page_t *ma, int count)
("pmap_qenter: Invalid level %d", lvl));
m = ma[i];
- attr = ATTR_AF | ATTR_SH(ATTR_SH_IS) |
+ attr = ATTR_AF | pmap_sh_attr |
ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_XN |
ATTR_KERN_GP | ATTR_S1_IDX(m->md.pv_memattr) | L3_PAGE;
pte = pmap_l2_to_l3(pde, va);
@@ -5124,7 +5126,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m,
vm_prot_t prot,
if ((m->oflags & VPO_UNMANAGED) == 0)
VM_PAGE_OBJECT_BUSY_ASSERT(m);
pa = VM_PAGE_TO_PHYS(m);
- new_l3 = (pt_entry_t)(PHYS_TO_PTE(pa) | ATTR_AF | ATTR_SH(ATTR_SH_IS) |
+ new_l3 = (pt_entry_t)(PHYS_TO_PTE(pa) | ATTR_AF | pmap_sh_attr |
L3_PAGE);
new_l3 |= pmap_pte_memattr(pmap, m->md.pv_memattr);
new_l3 |= pmap_pte_prot(pmap, prot);
@@ -5468,7 +5470,7 @@ pmap_enter_l2_rx(pmap_t pmap, vm_offset_t va, vm_page_t
m, vm_prot_t prot,
KASSERT(ADDR_IS_CANONICAL(va),
("%s: Address not in canonical form: %lx", __func__, va));
- new_l2 = (pd_entry_t)(VM_PAGE_TO_PTE(m) | ATTR_SH(ATTR_SH_IS) |
+ new_l2 = (pd_entry_t)(VM_PAGE_TO_PTE(m) | pmap_sh_attr |
ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) |
L2_BLOCK);
if ((m->oflags & VPO_UNMANAGED) == 0)
@@ -5697,7 +5699,7 @@ pmap_enter_l3c_rx(pmap_t pmap, vm_offset_t va, vm_page_t
m, vm_page_t *ml3p,
KASSERT(ADDR_IS_CANONICAL(va),
("%s: Address not in canonical form: %lx", __func__, va));
- l3e = VM_PAGE_TO_PTE(m) | ATTR_SH(ATTR_SH_IS) |
+ l3e = VM_PAGE_TO_PTE(m) | pmap_sh_attr |
ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) |
ATTR_CONTIGUOUS | L3_PAGE;
if ((m->oflags & VPO_UNMANAGED) == 0)
@@ -6094,7 +6096,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
vm_page_t m,
pmap_resident_count_inc(pmap, 1);
pa = VM_PAGE_TO_PHYS(m);
- l3_val = PHYS_TO_PTE(pa) | ATTR_SH(ATTR_SH_IS) |
+ l3_val = PHYS_TO_PTE(pa) | pmap_sh_attr |
ATTR_S1_IDX(m->md.pv_memattr) | ATTR_S1_AP(ATTR_S1_AP_RO) | L3_PAGE;
l3_val |= pmap_pte_bti(pmap, va);
if ((prot & VM_PROT_EXECUTE) == 0 ||
@@ -7744,7 +7746,7 @@ pmap_mapbios(vm_paddr_t pa, vm_size_t size)
/* Insert L2_BLOCK */
l2 = pmap_l1_to_l2(pde, va);
old_l2e |= pmap_load_store(l2,
- PHYS_TO_PTE(pa) | ATTR_AF | ATTR_SH(ATTR_SH_IS) |
+ PHYS_TO_PTE(pa) | ATTR_AF | pmap_sh_attr |
ATTR_S1_XN | ATTR_KERN_GP |
ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK) | L2_BLOCK);
diff --git a/sys/arm64/include/hypervisor.h b/sys/arm64/include/hypervisor.h
index 4c501e2722a9..1a27a8dd919b 100644
--- a/sys/arm64/include/hypervisor.h
+++ b/sys/arm64/include/hypervisor.h
@@ -241,6 +241,8 @@
#define VTCR_EL2_PS_42BIT (0x3UL << VTCR_EL2_PS_SHIFT)
#define VTCR_EL2_PS_44BIT (0x4UL << VTCR_EL2_PS_SHIFT)
#define VTCR_EL2_PS_48BIT (0x5UL << VTCR_EL2_PS_SHIFT)
+#define VTCR_EL2_DS_SHIFT 32
+#define VTCR_EL2_DS (0x1UL << VTCR_EL2_DS_SHIFT)
/* VTTBR_EL2 - Virtualization Translation Table Base Register */
#define VTTBR_VMID_MASK 0xffff000000000000
diff --git a/sys/arm64/include/pmap.h b/sys/arm64/include/pmap.h
index d604f705e596..c9552ebc326a 100644
--- a/sys/arm64/include/pmap.h
+++ b/sys/arm64/include/pmap.h
@@ -127,6 +127,8 @@ extern struct pmap kernel_pmap_store;
extern vm_offset_t virtual_avail;
extern vm_offset_t virtual_end;
+extern pt_entry_t pmap_sh_attr;
+
/*
* Macros to test if a mapping is mappable with an L1 Section mapping
* or an L2 Large Page mapping.
diff --git a/sys/arm64/vmm/vmm_arm64.c b/sys/arm64/vmm/vmm_arm64.c
index 164ff65cfe2c..80d985241c69 100644
--- a/sys/arm64/vmm/vmm_arm64.c
+++ b/sys/arm64/vmm/vmm_arm64.c
@@ -396,6 +396,14 @@ vmmops_modinit(int ipinum)
#ifdef SMP
el2_regs.vtcr_el2 |= VTCR_EL2_SH0_IS;
#endif
+ /*
+ * If FEAT_LPA2 is enabled in the host then we need to enable it here
+ * so the page tables created by pmap.c are correct. The meaning of
+ * the shareability field changes to become address bits when this
+ * is set.
+ */
+ if ((READ_SPECIALREG(tcr_el1) & TCR_DS) != 0)
+ el2_regs.vtcr_el2 |= VTCR_EL2_DS;
smp_rendezvous(NULL, arm_setup_vectors, NULL, &el2_regs);
