The branch main has been updated by andrew:
URL:
https://cgit.FreeBSD.org/src/commit/?id=4c6c27d3fb4ad15931aae2eaf8e624aed99a3fd9
commit 4c6c27d3fb4ad15931aae2eaf8e624aed99a3fd9
Author: Andrew Turner <and...@freebsd.org>
AuthorDate: 2025-08-08 08:09:29 +0000
Commit: Andrew Turner <and...@freebsd.org>
CommitDate: 2025-08-08 08:09:38 +0000
arm64: Support TBI in userspace
To allow for Hardware-assisted AddressSanitizer (HWASAN) and future
work to enable MTE we need to enable TBI in userspace. As address space
that previously would have faulted will now not it could be considered
an ABI change so only enable for processes with a late enough revision.
Relnotes: yes
Sponsored by: Arm Ltd
Differential Revision: https://reviews.freebsd.org/D51637
---
sys/arm64/arm64/efirt_machdep.c | 19 +++++++++++++++++++
sys/arm64/arm64/elf_machdep.c | 17 +++++++++++++++--
sys/arm64/arm64/exec_machdep.c | 31 +++++++++++++++++++++++++++++++
sys/arm64/arm64/genassym.c | 1 +
sys/arm64/arm64/pmap.c | 19 ++++++++++++++-----
sys/arm64/arm64/ptrauth.c | 4 ++++
sys/arm64/arm64/swtch.S | 12 ++++++++++++
sys/arm64/arm64/vm_machdep.c | 3 +++
sys/arm64/include/cpu.h | 3 +++
sys/arm64/include/elf.h | 3 +++
sys/arm64/include/proc.h | 7 ++++++-
sys/arm64/include/vmparam.h | 3 ++-
sys/sys/param.h | 2 +-
tests/sys/kern/ptrace_test.c | 2 +-
14 files changed, 115 insertions(+), 11 deletions(-)
diff --git a/sys/arm64/arm64/efirt_machdep.c b/sys/arm64/arm64/efirt_machdep.c
index 0f46e44f5d6a..0301eb91c9ef 100644
--- a/sys/arm64/arm64/efirt_machdep.c
+++ b/sys/arm64/arm64/efirt_machdep.c
@@ -241,6 +241,7 @@ fail:
int
efi_arch_enter(void)
{
+ uint64_t tcr;
CRITICAL_ASSERT(curthread);
curthread->td_md.md_efirt_dis_pf = vm_fault_disable_pagefaults();
@@ -249,7 +250,17 @@ efi_arch_enter(void)
* Temporarily switch to EFI's page table. However, we leave curpmap
* unchanged in order to prevent its ASID from being reclaimed before
* we switch back to its page table in efi_arch_leave().
+ *
+ * UEFI sdoesn't care about TBI, so enable it. It's more likely
+ * userspace will have TBI on as it's only disabled for backwards
+ * compatibility.
*/
+ tcr = READ_SPECIALREG(tcr_el1);
+ if ((tcr & MD_TCR_FIELDS) != TCR_TBI0) {
+ tcr &= ~MD_TCR_FIELDS;
+ tcr |= TCR_TBI0;
+ WRITE_SPECIALREG(tcr_el1, tcr);
+ }
set_ttbr0(efi_ttbr0);
if (PCPU_GET(bcast_tlbi_workaround) != 0)
invalidate_local_icache();
@@ -260,6 +271,7 @@ efi_arch_enter(void)
void
efi_arch_leave(void)
{
+ uint64_t proc_tcr, tcr;
/*
* Restore the pcpu pointer. Some UEFI implementations trash it and
@@ -271,6 +283,13 @@ efi_arch_leave(void)
__asm __volatile(
"mrs x18, tpidr_el1 \n"
);
+ proc_tcr = curthread->td_proc->p_md.md_tcr;
+ tcr = READ_SPECIALREG(tcr_el1);
+ if ((tcr & MD_TCR_FIELDS) != proc_tcr) {
+ tcr &= ~MD_TCR_FIELDS;
+ tcr |= proc_tcr;
+ WRITE_SPECIALREG(tcr_el1, tcr);
+ }
set_ttbr0(pmap_to_ttbr0(PCPU_GET(curpmap)));
if (PCPU_GET(bcast_tlbi_workaround) != 0)
invalidate_local_icache();
diff --git a/sys/arm64/arm64/elf_machdep.c b/sys/arm64/arm64/elf_machdep.c
index 970dba0ca7d9..13af5c5065d6 100644
--- a/sys/arm64/arm64/elf_machdep.c
+++ b/sys/arm64/arm64/elf_machdep.c
@@ -65,7 +65,13 @@ u_long __read_frequently linux_elf_hwcap2;
u_long __read_frequently linux_elf_hwcap3;
u_long __read_frequently linux_elf_hwcap4;
-struct arm64_addr_mask elf64_addr_mask;
+struct arm64_addr_mask elf64_addr_mask = {
+ .code = TBI_ADDR_MASK,
+ .data = TBI_ADDR_MASK,
+};
+#ifdef COMPAT_FREEBSD14
+struct arm64_addr_mask elf64_addr_mask_14;
+#endif
static void arm64_exec_protect(struct image_params *, int);
@@ -136,7 +142,14 @@ get_arm64_addr_mask(struct regset *rs, struct thread *td,
void *buf,
if (buf != NULL) {
KASSERT(*sizep == sizeof(elf64_addr_mask),
("%s: invalid size", __func__));
- memcpy(buf, &elf64_addr_mask, sizeof(elf64_addr_mask));
+#ifdef COMPAT_FREEBSD14
+ /* running an old binary use the old address mask */
+ if (td->td_proc->p_osrel < TBI_VERSION)
+ memcpy(buf, &elf64_addr_mask_14,
+ sizeof(elf64_addr_mask_14));
+ else
+#endif
+ memcpy(buf, &elf64_addr_mask, sizeof(elf64_addr_mask));
}
*sizep = sizeof(elf64_addr_mask);
diff --git a/sys/arm64/arm64/exec_machdep.c b/sys/arm64/arm64/exec_machdep.c
index 751329affd91..7c50dc93fdb4 100644
--- a/sys/arm64/arm64/exec_machdep.c
+++ b/sys/arm64/arm64/exec_machdep.c
@@ -51,6 +51,7 @@
#include <vm/vm_map.h>
#include <machine/armreg.h>
+#include <machine/elf.h>
#include <machine/kdb.h>
#include <machine/md_var.h>
#include <machine/pcb.h>
@@ -411,6 +412,7 @@ exec_setregs(struct thread *td, struct image_params *imgp,
uintptr_t stack)
{
struct trapframe *tf = td->td_frame;
struct pcb *pcb = td->td_pcb;
+ uint64_t new_tcr, tcr;
memset(tf, 0, sizeof(struct trapframe));
@@ -433,6 +435,35 @@ exec_setregs(struct thread *td, struct image_params *imgp,
uintptr_t stack)
*/
bzero(&pcb->pcb_dbg_regs, sizeof(pcb->pcb_dbg_regs));
+ /* If the process is new enough enable TBI */
+ if (td->td_proc->p_osrel >= TBI_VERSION)
+ new_tcr = TCR_TBI0;
+ else
+ new_tcr = 0;
+ td->td_proc->p_md.md_tcr = new_tcr;
+
+ /* TODO: should create a pmap function for this... */
+ tcr = READ_SPECIALREG(tcr_el1);
+ if ((tcr & MD_TCR_FIELDS) != new_tcr) {
+ uint64_t asid;
+
+ tcr &= ~MD_TCR_FIELDS;
+ tcr |= new_tcr;
+ WRITE_SPECIALREG(tcr_el1, tcr);
+ isb();
+
+ /*
+ * TCR_EL1.TBI0 is permitted to be cached in the TLB, so
+ * we need to perform a TLB invalidation.
+ */
+ asid = READ_SPECIALREG(ttbr0_el1) & TTBR_ASID_MASK;
+ __asm __volatile(
+ "tlbi aside1is, %0 \n"
+ "dsb ish \n"
+ "isb \n"
+ : : "r" (asid));
+ }
+
/* Generate new pointer authentication keys */
ptrauth_exec(td);
}
diff --git a/sys/arm64/arm64/genassym.c b/sys/arm64/arm64/genassym.c
index e3977798b046..22696796e69d 100644
--- a/sys/arm64/arm64/genassym.c
+++ b/sys/arm64/arm64/genassym.c
@@ -64,6 +64,7 @@ ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault));
ASSYM(PCB_FLAGS, offsetof(struct pcb, pcb_flags));
ASSYM(P_PID, offsetof(struct proc, p_pid));
+ASSYM(P_MD_TCR, offsetof(struct proc, p_md.md_tcr));
ASSYM(SF_UC, offsetof(struct sigframe, sf_uc));
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 07955866bd1c..ec89c4573799 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -469,7 +469,7 @@ static pv_entry_t pmap_pvh_remove(struct md_page *pvh,
pmap_t pmap,
vm_offset_t va);
static void pmap_abort_ptp(pmap_t pmap, vm_offset_t va, vm_page_t mpte);
-static bool pmap_activate_int(pmap_t pmap);
+static bool pmap_activate_int(struct thread *td, pmap_t pmap);
static void pmap_alloc_asid(pmap_t pmap);
static int pmap_change_props_locked(vm_offset_t va, vm_size_t size,
vm_prot_t prot, int mode, bool skip_unmapped);
@@ -9113,7 +9113,7 @@ pmap_init_cnp(void *dummy __unused)
SYSINIT(pmap_init_cnp, SI_SUB_SMP, SI_ORDER_ANY, pmap_init_cnp, NULL);
static bool
-pmap_activate_int(pmap_t pmap)
+pmap_activate_int(struct thread *td, pmap_t pmap)
{
struct asid_set *set;
int epoch;
@@ -9152,6 +9152,15 @@ pmap_activate_int(pmap_t pmap)
pmap_alloc_asid(pmap);
if (pmap->pm_stage == PM_STAGE1) {
+ uint64_t new_tcr, tcr;
+
+ new_tcr = td->td_proc->p_md.md_tcr;
+ tcr = READ_SPECIALREG(tcr_el1);
+ if ((tcr & MD_TCR_FIELDS) != new_tcr) {
+ tcr &= ~MD_TCR_FIELDS;
+ tcr |= new_tcr;
+ WRITE_SPECIALREG(tcr_el1, tcr);
+ }
set_ttbr0(pmap_to_ttbr0(pmap));
if (PCPU_GET(bcast_tlbi_workaround) != 0)
invalidate_local_icache();
@@ -9165,7 +9174,7 @@ pmap_activate_vm(pmap_t pmap)
PMAP_ASSERT_STAGE2(pmap);
- (void)pmap_activate_int(pmap);
+ (void)pmap_activate_int(NULL, pmap);
}
void
@@ -9176,7 +9185,7 @@ pmap_activate(struct thread *td)
pmap = vmspace_pmap(td->td_proc->p_vmspace);
PMAP_ASSERT_STAGE1(pmap);
critical_enter();
- (void)pmap_activate_int(pmap);
+ (void)pmap_activate_int(td, pmap);
critical_exit();
}
@@ -9202,7 +9211,7 @@ pmap_switch(struct thread *new)
* to a user process.
*/
- if (pmap_activate_int(vmspace_pmap(new->td_proc->p_vmspace))) {
+ if (pmap_activate_int(new, vmspace_pmap(new->td_proc->p_vmspace))) {
/*
* Stop userspace from training the branch predictor against
* other processes. This will call into a CPU specific
diff --git a/sys/arm64/arm64/ptrauth.c b/sys/arm64/arm64/ptrauth.c
index 767b7e115479..dbe0c69b8d60 100644
--- a/sys/arm64/arm64/ptrauth.c
+++ b/sys/arm64/arm64/ptrauth.c
@@ -149,6 +149,10 @@ ptrauth_enable(const struct cpu_feat *feat __unused,
enable_ptrauth = true;
elf64_addr_mask.code |= PAC_ADDR_MASK;
elf64_addr_mask.data |= PAC_ADDR_MASK;
+#ifdef COMPAT_FREEBSD14
+ elf64_addr_mask_14.code |= PAC_ADDR_MASK_14;
+ elf64_addr_mask_14.data |= PAC_ADDR_MASK_14;
+#endif
}
diff --git a/sys/arm64/arm64/swtch.S b/sys/arm64/arm64/swtch.S
index 7b6010a5f51f..a461fded929c 100644
--- a/sys/arm64/arm64/swtch.S
+++ b/sys/arm64/arm64/swtch.S
@@ -37,6 +37,8 @@
#include <machine/asm.h>
#include <machine/armreg.h>
+#include <machine/proc.h>
+
.macro clear_step_flag pcbflags, tmp
tbz \pcbflags, #PCB_SINGLE_STEP_SHIFT, 999f
mrs \tmp, mdscr_el1
@@ -239,6 +241,16 @@ ENTRY(fork_trampoline)
msr daifset, #(DAIF_D | DAIF_INTR)
ldr x0, [x18, #PC_CURTHREAD]
+
+ /* Set the per-process tcr_el1 fields */
+ ldr x10, [x0, #TD_PROC]
+ ldr x10, [x10, #P_MD_TCR]
+ mrs x11, tcr_el1
+ and x11, x11, #(~MD_TCR_FIELDS)
+ orr x11, x11, x10
+ msr tcr_el1, x11
+ /* No isb as the eret below is the context-synchronising event */
+
bl ptrauth_enter_el0
/* Restore sp, lr, elr, and spsr */
diff --git a/sys/arm64/arm64/vm_machdep.c b/sys/arm64/arm64/vm_machdep.c
index 38a126ff602f..0134feb65b6a 100644
--- a/sys/arm64/arm64/vm_machdep.c
+++ b/sys/arm64/arm64/vm_machdep.c
@@ -120,6 +120,9 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread
*td2, int flags)
td2->td_md.md_spinlock_count = 1;
td2->td_md.md_saved_daif = PSR_DAIF_DEFAULT;
+ /* Copy the TCR_EL1 value */
+ td2->td_proc->p_md.md_tcr = td1->td_proc->p_md.md_tcr;
+
#if defined(PERTHREAD_SSP)
/* Set the new canary */
arc4random_buf(&td2->td_md.md_canary, sizeof(td2->td_md.md_canary));
diff --git a/sys/arm64/include/cpu.h b/sys/arm64/include/cpu.h
index 935e3754bf25..59cda36f275e 100644
--- a/sys/arm64/include/cpu.h
+++ b/sys/arm64/include/cpu.h
@@ -226,6 +226,9 @@ extern uint64_t __cpu_affinity[];
struct arm64_addr_mask;
extern struct arm64_addr_mask elf64_addr_mask;
+#ifdef COMPAT_FREEBSD14
+extern struct arm64_addr_mask elf64_addr_mask_14;
+#endif
typedef void (*cpu_reset_hook_t)(void);
extern cpu_reset_hook_t cpu_reset_hook;
diff --git a/sys/arm64/include/elf.h b/sys/arm64/include/elf.h
index d6328c143585..81ee7392f866 100644
--- a/sys/arm64/include/elf.h
+++ b/sys/arm64/include/elf.h
@@ -93,6 +93,9 @@ __ElfType(Auxinfo);
#define ET_DYN_LOAD_ADDR 0x100000
#endif
+/* First __FreeBSD_version that supports Top Byte Ignore (TBI) */
+#define TBI_VERSION 1500058
+
/* HWCAP */
#define HWCAP_FP (1 << 0)
#define HWCAP_ASIMD (1 << 1)
diff --git a/sys/arm64/include/proc.h b/sys/arm64/include/proc.h
index dc2fa2df654d..184743d4cc80 100644
--- a/sys/arm64/include/proc.h
+++ b/sys/arm64/include/proc.h
@@ -35,6 +35,7 @@
#ifndef _MACHINE_PROC_H_
#define _MACHINE_PROC_H_
+#ifndef LOCORE
struct ptrauth_key {
uint64_t pa_key_lo;
uint64_t pa_key_hi;
@@ -73,8 +74,12 @@ struct mdthread {
};
struct mdproc {
- long md_dummy;
+ uint64_t md_tcr; /* TCR_EL1 fields to update */
};
+#endif /* !LOCORE */
+
+/* Fields that can be set in md_tcr */
+#define MD_TCR_FIELDS TCR_TBI0
#define KINFO_PROC_SIZE 1088
#define KINFO_PROC32_SIZE 816
diff --git a/sys/arm64/include/vmparam.h b/sys/arm64/include/vmparam.h
index c30ca1b2bff4..781602306436 100644
--- a/sys/arm64/include/vmparam.h
+++ b/sys/arm64/include/vmparam.h
@@ -209,7 +209,8 @@
#define KMSAN_ORIG_MAX_ADDRESS (0xffff028000000000UL)
/* The address bits that hold a pointer authentication code */
-#define PAC_ADDR_MASK (0xff7f000000000000UL)
+#define PAC_ADDR_MASK (0x007f000000000000UL)
+#define PAC_ADDR_MASK_14 (0xff7f000000000000UL)
/* The top-byte ignore address bits */
#define TBI_ADDR_MASK 0xff00000000000000UL
diff --git a/sys/sys/param.h b/sys/sys/param.h
index de3697d6dedd..72061bd8134e 100644
--- a/sys/sys/param.h
+++ b/sys/sys/param.h
@@ -74,7 +74,7 @@
* cannot include sys/param.h and should only be updated here.
*/
#undef __FreeBSD_version
-#define __FreeBSD_version 1500057
+#define __FreeBSD_version 1500058
/*
* __FreeBSD_kernel__ indicates that this system uses the kernel of FreeBSD,
diff --git a/tests/sys/kern/ptrace_test.c b/tests/sys/kern/ptrace_test.c
index d36dfe951e20..fee0bd2ffa38 100644
--- a/tests/sys/kern/ptrace_test.c
+++ b/tests/sys/kern/ptrace_test.c
@@ -3239,7 +3239,7 @@ ATF_TC_BODY(ptrace__PT_REGSET, tc)
ATF_REQUIRE(ptrace(PT_GETREGSET, wpid, (caddr_t)&vec,
NT_ARM_ADDR_MASK) != -1);
REQUIRE_EQ(addr_mask.code, addr_mask.data);
- ATF_REQUIRE(addr_mask.code == 0 ||
+ ATF_REQUIRE(addr_mask.code == 0xff00000000000000ul ||
addr_mask.code == 0xff7f000000000000UL);
#endif
