For the arm64 kernel, when it processes hardware memory errors for synchronize notifications(do_sea()), if the errors is consumed within the kernel, the current processing is panic. However, it is not optimal.
Take copy_from/to_user for example, If ld* triggers a memory error, even in kernel mode, only the associated process is affected. Killing the user process and isolating the corrupt page is a better choice. Add new fixup type EX_TYPE_KACCESS_SEA to identify insn that can recover from memory errors triggered by access to kernel memory, and this fixup type is used in __arch_copy_to_user(), This make the regular copy_to_user() will handle kernel memory errors. [Ruidong: modify subject and rename EX_TYPE_KACCESS_ERR_ZERO_MEM_ERR to EX_TYPE_KACCESS_SEA] Signed-off-by: Tong Tiangen <[email protected]> Signed-off-by: Ruidong Tian <[email protected]> --- arch/arm64/include/asm/asm-extable.h | 5 +++++ arch/arm64/include/asm/asm-uaccess.h | 4 ++++ arch/arm64/include/asm/extable.h | 1 + arch/arm64/lib/copy_to_user.S | 10 +++++----- arch/arm64/mm/extable.c | 28 ++++++++++++++++++++++++++ arch/arm64/mm/fault.c | 30 ++++++++++++++++++++-------- 6 files changed, 65 insertions(+), 13 deletions(-) diff --git a/arch/arm64/include/asm/asm-extable.h b/arch/arm64/include/asm/asm-extable.h index 06b19023939b..8450ec5a3af6 100644 --- a/arch/arm64/include/asm/asm-extable.h +++ b/arch/arm64/include/asm/asm-extable.h @@ -10,6 +10,7 @@ #define EX_TYPE_ACCESS_ERR_ZERO 2 #define EX_TYPE_UACCESS_CPY 3 #define EX_TYPE_LOAD_UNALIGNED_ZEROPAD 4 +#define EX_TYPE_KACCESS_SEA 5 /* Data fields for EX_TYPE_ACCESS_ERR_ZERO */ #define EX_DATA_REG_ERR_SHIFT 0 @@ -76,6 +77,10 @@ __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_UACCESS_CPY, \uaccess_is_write) .endm + .macro _asm_extable_kaccess_sea, insn, fixup + __ASM_EXTABLE_RAW(\insn, \fixup, EX_TYPE_KACCESS_SEA, 0) + .endm + #else /* __ASSEMBLER__ */ #include <linux/stringify.h> diff --git a/arch/arm64/include/asm/asm-uaccess.h b/arch/arm64/include/asm/asm-uaccess.h index 12aa6a283249..27bf8edbf597 100644 --- a/arch/arm64/include/asm/asm-uaccess.h +++ b/arch/arm64/include/asm/asm-uaccess.h @@ -57,6 +57,10 @@ alternative_else_nop_endif .endm #endif +#define KERNEL_SEA(l, x...) \ +9999: x; \ + _asm_extable_kaccess_sea 9999b, l + #define USER(l, x...) \ 9999: x; \ _asm_extable_uaccess 9999b, l diff --git a/arch/arm64/include/asm/extable.h b/arch/arm64/include/asm/extable.h index 9dc39612bdf5..47c851d7df4f 100644 --- a/arch/arm64/include/asm/extable.h +++ b/arch/arm64/include/asm/extable.h @@ -48,4 +48,5 @@ bool ex_handler_bpf(const struct exception_table_entry *ex, #endif /* !CONFIG_BPF_JIT */ bool fixup_exception(struct pt_regs *regs, unsigned long esr); +bool fixup_exception_me(struct pt_regs *regs); #endif diff --git a/arch/arm64/lib/copy_to_user.S b/arch/arm64/lib/copy_to_user.S index 819f2e3fc7a9..6103f5b0a2d0 100644 --- a/arch/arm64/lib/copy_to_user.S +++ b/arch/arm64/lib/copy_to_user.S @@ -20,7 +20,7 @@ * x0 - bytes not copied */ .macro ldrb1 reg, ptr, val - ldrb \reg, [\ptr], \val + KERNEL_SEA(9998f, ldrb \reg, [\ptr], \val) .endm .macro strb1 reg, ptr, val @@ -28,7 +28,7 @@ .endm .macro ldrh1 reg, ptr, val - ldrh \reg, [\ptr], \val + KERNEL_SEA(9998f, ldrh \reg, [\ptr], \val) .endm .macro strh1 reg, ptr, val @@ -36,7 +36,7 @@ .endm .macro ldr1 reg, ptr, val - ldr \reg, [\ptr], \val + KERNEL_SEA(9998f, ldr \reg, [\ptr], \val) .endm .macro str1 reg, ptr, val @@ -44,7 +44,7 @@ .endm .macro ldp1 reg1, reg2, ptr, val - ldp \reg1, \reg2, [\ptr], \val + KERNEL_SEA(9998f, ldp \reg1, \reg2, [\ptr], \val) .endm .macro stp1 reg1, reg2, ptr, val @@ -74,7 +74,7 @@ SYM_FUNC_START(__arch_copy_to_user) 9997: cmp dst, dstin b.ne 9998f // Before being absolutely sure we couldn't copy anything, try harder - ldrb tmp1w, [srcin] +KERNEL_SEA(9998f, ldrb tmp1w, [srcin]) USER(9998f, sttrb tmp1w, [dst]) add dst, dst, #1 9998: sub x0, end, dst // bytes not copied diff --git a/arch/arm64/mm/extable.c b/arch/arm64/mm/extable.c index 76b18780f1f9..20a7a9eeed94 100644 --- a/arch/arm64/mm/extable.c +++ b/arch/arm64/mm/extable.c @@ -109,7 +109,35 @@ bool fixup_exception(struct pt_regs *regs, unsigned long esr) return ex_handler_uaccess_cpy(ex, regs, esr); case EX_TYPE_LOAD_UNALIGNED_ZEROPAD: return ex_handler_load_unaligned_zeropad(ex, regs); + /* + * Kernel address faults (e.g. copy_to_user reading from kernel src). + * Do not fixup here: a translation fault on a kernel address is a + * kernel bug (e.g. NULL pointer dereference) and must oops. + * Only SEA (hardware memory error) should be fixed up, which is + * handled by fixup_exception_me() through the do_sea path. + */ + case EX_TYPE_KACCESS_SEA: + return false; } BUG(); } + +bool fixup_exception_me(struct pt_regs *regs) +{ + const struct exception_table_entry *ex; + + ex = search_exception_tables(instruction_pointer(regs)); + if (!ex) + return false; + + switch (ex->type) { + case EX_TYPE_ACCESS_ERR_ZERO: + return ex_handler_access_err_zero(ex, regs); + case EX_TYPE_KACCESS_SEA: + regs->pc = get_ex_fixup(ex); + return true; + } + + return false; +} diff --git a/arch/arm64/mm/fault.c b/arch/arm64/mm/fault.c index 0f3c5c7ca054..b775c0928a53 100644 --- a/arch/arm64/mm/fault.c +++ b/arch/arm64/mm/fault.c @@ -858,21 +858,35 @@ static int do_bad(unsigned long far, unsigned long esr, struct pt_regs *regs) return 1; /* "fault" */ } +/* + * APEI claimed this as a firmware-first notification. + * Some processing deferred to task_work before ret_to_user(). + */ +static int do_apei_claim_sea(struct pt_regs *regs) +{ + int ret; + + ret = apei_claim_sea(regs); + if (ret) + return ret; + + if (!user_mode(regs)) { + if (!fixup_exception_me(regs)) + return -ENOENT; + } + + return ret; +} + static int do_sea(unsigned long far, unsigned long esr, struct pt_regs *regs) { const struct fault_info *inf; unsigned long siaddr; - inf = esr_to_fault_info(esr); - - if (user_mode(regs) && apei_claim_sea(regs) == 0) { - /* - * APEI claimed this as a firmware-first notification. - * Some processing deferred to task_work before ret_to_user(). - */ + if (do_apei_claim_sea(regs) == 0) return 0; - } + inf = esr_to_fault_info(esr); if (esr & ESR_ELx_FnV) { siaddr = 0; } else { -- 2.39.3
